valknar/llmx
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: local tokenizer (#5508)

Browse Source
This commit is contained in:
jif-oai
2025-10-22 16:01:02 +01:00
committed by GitHub
parent 00b1e130b3
commit fd0673e457
4 changed files with 218 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
codex-rs/Cargo.toml
View File

@@ -37,6 +37,7 @@ members = [
"utils/readiness",
"utils/pty",
"utils/string",
"utils/tokenizer",
]
resolver = "2"
@@ -82,6 +83,7 @@ codex-utils-json-to-toml = { path = "utils/json-to-toml" }
codex-utils-pty = { path = "utils/pty" }
codex-utils-readiness = { path = "utils/readiness" }
codex-utils-string = { path = "utils/string" }
codex-utils-tokenizer = { path = "utils/tokenizer" }
core_test_support = { path = "core/tests/common" }
mcp-types = { path = "mcp-types" }
mcp_test_support = { path = "mcp-server/tests/common" }
@@ -246,7 +248,7 @@ unwrap_used = "deny"
# cargo-shear cannot see the platform-specific openssl-sys usage, so we
# silence the false positive here instead of deleting a real dependency.
[workspace.metadata.cargo-shear]
ignored = ["openssl-sys", "codex-utils-readiness"]
ignored = ["openssl-sys", "codex-utils-readiness", "codex-utils-tokenizer"]
[profile.release]
lto = "fat"