feat: local tokenizer (#5508)

2025-10-22 16:01:02 +01:00
parent 00b1e130b3
commit fd0673e457
4 changed files with 218 additions and 3 deletions
--- a/codex-rs/utils/tokenizer/Cargo.toml
+++ b/codex-rs/utils/tokenizer/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+edition.workspace = true
+name = "codex-utils-tokenizer"
+version.workspace = true
+
+[lints]
+workspace = true
+
+[dependencies]
+anyhow = { workspace = true }
+thiserror = { workspace = true }
+tiktoken-rs = "0.7"
+
+[dev-dependencies]
+pretty_assertions = { workspace = true }