feat(encoding): add claude_v3 and claude_v4_7 encodings ported from ctok

.agents/skills/verify-ai-tokenizer/features/entry-points.md

@@ -1,6 +1,6 @@
 # Package entry points
 
-The package exports `.`, `./sdk`, `./encoding` and `./encoding/{cl100k_base,o200k_base,p50k_base,claude}`, each with ESM, CJS and type entries. A consumer can import every subpath from ESM, require it from CJS, and typecheck the README snippets with TypeScript.
+The package exports `.`, `./sdk`, `./encoding` and `./encoding/{cl100k_base,o200k_base,p50k_base,claude,claude_v3,claude_v4_7,claude_v4_8,claude_v5}`, each with ESM, CJS and type entries. A consumer can import every subpath from ESM, require it from CJS, and typecheck the README snippets with TypeScript.
 
 ## Sub-features
 

.agents/skills/verify-ai-tokenizer/journeys/entry-points.cjs

@@ -3,7 +3,7 @@ const results = [];
 const check = (id, action, expected, observed, pass) => results.push({ id, pass: Boolean(pass), action, expected, observed });
 const outcome = (fn) => { try { return { value: fn() }; } catch (e) { return { threw: String(e && e.message ? e.message : e) }; } };
 
-const names = ["cl100k_base", "o200k_base", "p50k_base", "claude"];
+const names = ["cl100k_base", "o200k_base", "p50k_base", "claude", "claude_v3", "claude_v4_7", "claude_v4_8", "claude_v5"];
 const specs = ["ai-tokenizer", "ai-tokenizer/sdk", "ai-tokenizer/encoding", ...names.map((n) => `ai-tokenizer/encoding/${n}`)];
 for (const s of specs) {
   const r = outcome(() => require.resolve(s));

.agents/skills/verify-ai-tokenizer/journeys/entry-points.mjs

@@ -6,9 +6,13 @@ import * as cl100k_base from "ai-tokenizer/encoding/cl100k_base";
 import * as o200k_base from "ai-tokenizer/encoding/o200k_base";
 import * as p50k_base from "ai-tokenizer/encoding/p50k_base";
 import * as claude from "ai-tokenizer/encoding/claude";
+import * as claude_v3 from "ai-tokenizer/encoding/claude_v3";
+import * as claude_v4_7 from "ai-tokenizer/encoding/claude_v4_7";
+import * as claude_v4_8 from "ai-tokenizer/encoding/claude_v4_8";
+import * as claude_v5 from "ai-tokenizer/encoding/claude_v5";
 import { check, finish, outcome, sameJSON } from "./check.mjs";
 
-const specs = ["ai-tokenizer", "ai-tokenizer/sdk", "ai-tokenizer/encoding", ...["cl100k_base", "o200k_base", "p50k_base", "claude"].map((n) => `ai-tokenizer/encoding/${n}`)];
+const specs = ["ai-tokenizer", "ai-tokenizer/sdk", "ai-tokenizer/encoding", ...["cl100k_base", "o200k_base", "p50k_base", "claude", "claude_v3", "claude_v4_7", "claude_v4_8", "claude_v5"].map((n) => `ai-tokenizer/encoding/${n}`)];
 for (const s of specs) {
   const r = outcome(() => import.meta.resolve(s));
   check(`resolve/${s}`, `import.meta.resolve("${s}")`, "a dist/*.js file inside node_modules/ai-tokenizer", r, /node_modules\/ai-tokenizer\/dist\/.+\.js$/.test(r.value ?? ""));
@@ -18,7 +22,7 @@ check("default-export", 'import Tokenizer, { Tokenizer } from "ai-tokenizer"', "
 check("models", 'import { models } from "ai-tokenizer"', "a non-empty record with encoding names", Object.keys(models).length, Object.keys(models).length > 0 && Object.values(models).every((m) => m.encoding in encoding));
 check("sdk-count", 'import { count } from "ai-tokenizer/sdk"', "function", typeof count, typeof count === "function");
 
-const subpaths = { cl100k_base, o200k_base, p50k_base, claude };
+const subpaths = { cl100k_base, o200k_base, p50k_base, claude, claude_v3, claude_v4_7, claude_v4_8, claude_v5 };
 for (const [name, mod] of Object.entries(subpaths)) {
   check(`encoding/${name}/name`, `import * as ${name} from "ai-tokenizer/encoding/${name}"`, name, mod.name, mod.name === name);
   const a = outcome(() => new Tokenizer(mod).encode("hello world"));

LICENSE

@@ -17,4 +17,33 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+---
+
+Third-party notice
+
+src/claude/ and src/encoding/claude_v*.ts are ported from ctok
+(https://github.com/sanderland/ctok) under this license:
+
+MIT License
+
+Copyright (c) 2026 Sander Land
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -91,6 +91,39 @@ const total = tokenizer.count("some text input");
 > [!WARNING]
 > Import encodings selectively based on what you need. Each encoding is 2-8MB uncompressed.
 
+## Claude encodings
+
+Anthropic changed its tokenizer at Claude Opus 4.7. Four encodings count Claude tokens offline. Each one is named after the first model that counts that way. They match the `count_tokens` API exactly on all 93 texts in `src/fixtures/claude-count-tokens.json`, measured on Claude Opus 4.6, Opus 4.7, Opus 4.8 and Opus 5.5. The older `claude` encoding stays available unchanged.
+
+| Encoding | Models | Frame |
+| --- | --- | --- |
+| `claude_v3` | Claude 3 through Claude Opus 4.6, Sonnet 4.6 and Haiku 4.5 | 7 |
+| `claude_v4_7` | Claude Opus 4.7 | 11 |
+| `claude_v4_8` | Claude Opus 4.8, Sonnet 5 and Fable 5 | 6 |
+| `claude_v5` | Claude Opus 5, Opus 5.5, Sonnet 5.5, Haiku 5.5 and Fable 5.1 | 6 on Opus 5, 8 on the 5.5 and 5.1 models |
+
+`claude_v4_7`, `claude_v4_8` and `claude_v5` share one vocabulary. They differ in how the message frame treats the content.
+
+```ts
+import { Tokenizer } from "ai-tokenizer";
+import * as claude_v5 from "ai-tokenizer/encoding/claude_v5";
+
+const tokenizer = new Tokenizer(claude_v5);
+tokenizer.count("hello, world"); // 4
+```
+
+- `count()` counts the content of one user message. `count_tokens` also charges a fixed message frame, shown in the table above. The frame comes from the model config, not from the encoding.
+- The content count includes what the frame does to the content. On `claude_v3` and `claude_v4_7` the frame absorbs one leading space. On every encoding, newlines at the end of the content join the frame's own newline token, so a short run of them is free.
+- `claude_v5` drops trailing ASCII whitespace (space, tab, CR, LF, VT, FF) from the text it counts. These models drop it at the end of each user message and at the end of the system prompt. They do not drop it in assistant messages or in `tool_result` content. The API joins the text blocks of a message with no separator and trims only the last block, so trailing whitespace in earlier blocks still counts. A last block that holds only whitespace is not trimmed: `["a", "   "]` costs what `"a   "` costs on `claude_v4_8`. To count a user message with several text blocks, count the joined text with `claude_v5`, or with `claude_v4_8` when the last block is only whitespace.
+- `count("")` is 0. The API rejects empty content.
+- Token ids are synthetic, because Anthropic does not publish its ids. They are stable for one version of this package only.
+- `decode()` returns the normalized text, so it is lossy. Plain ASCII prose comes back unchanged. Unicode normalization, folded quotes and spaces, removed control characters, the absorbed leading space, some case changes and trailing whitespace do not.
+- Counts use Unicode 15.1 data, like the reference. A JavaScript engine with Unicode data older than 15.1 can count text with newer characters differently.
+- Some Latin letters added in Unicode 16 (for example U+A7CC, U+A7CD and U+A7DA) count a few tokens off. ctok has the same gap.
+- A single space at the end of the text, right after one of the ideographic tone marks U+302A to U+302F, counts 1 token too many on `claude_v3`, `claude_v4_7` and `claude_v4_8` (`"x \u302e "` is 5 on the API and 6 here). ctok has the same gap. `claude_v5` is right, because it trims that space.
+
+The encodings are a TypeScript port of [ctok](https://github.com/sanderland/ctok) by Sander Land (MIT License), at commit [`8c43ffd`](https://github.com/sanderland/ctok/tree/8c43ffd3c7be467622a540f025516e49ef78d8b1). `scripts/generate-claude-encodings.ts` regenerates the data from that commit, and `scripts/compare-claude-ctok.ts` compares the port with ctok.
+
 ## Accuracy
 
 Validated against actual API responses with pseudo-random messages:

bench/claude.ts

@@ -0,0 +1,84 @@
+/**
+ * Claude encoding timings: 1M characters of prose, 100k of one repeated
+ * character and 100k combining marks in the wrong order, each also at twice
+ * the size. A 2x/1x ratio near 2 is linear.
+ *
+ *   bun bench/claude.ts
+ */
+import { Tokenizer } from "../src/index";
+import * as claude_v3 from "../src/encoding/claude_v3";
+import * as claude_v4_7 from "../src/encoding/claude_v4_7";
+
+const WORDS =
+  "the of and to in is you that it he was for on are as with his they at be this have from or one had by word but not what all were we when your can said there use an each which she do how their if will up other about out many then them these so some her would make like him into time has look two more write go see number no way could people my than first water been call who oil its now find long down day did get come made may part".split(
+    " "
+  );
+
+let seed = 7;
+function rand(n: number): number {
+  seed = (Math.imul(seed, 1103515245) + 12345) >>> 0;
+  return (seed >>> 8) % n;
+}
+
+function prose(chars: number): string {
+  const parts: string[] = [];
+  let length = 0;
+  while (length < chars) {
+    const words = Array.from(
+      { length: 6 + rand(14) },
+      () => WORDS[rand(WORDS.length)]!
+    );
+    words[0] = words[0]![0]!.toUpperCase() + words[0]!.slice(1);
+    if (rand(4) === 0) words[rand(words.length)] += ",";
+    const sentence = words.join(" ") + [".", "!", "?", "."][rand(4)]!;
+    const sep = rand(6) === 0 ? "\n\n" : " ";
+    parts.push(sentence + sep);
+    length += sentence.length + sep.length;
+  }
+  return parts.join("").slice(0, chars);
+}
+
+function median(fn: () => void): number {
+  const times: number[] = [];
+  for (let i = 0; i < 3; i++) {
+    const start = performance.now();
+    fn();
+    times.push(performance.now() - start);
+  }
+  return times.sort((a, b) => a - b)[1]!;
+}
+
+const inputs: Array<[string, (scale: number) => string]> = [
+  ["prose 1M", (k) => prose(1_000_000 * k)],
+  ...["a", " ", "\n", "!", "日"].map(
+    (ch): [string, (scale: number) => string] => [
+      `${JSON.stringify(ch)} x 100k`,
+      (k) => ch.repeat(100_000 * k),
+    ]
+  ),
+  [
+    "marks x 100k",
+    (k) => "a" + "\u0301".repeat(50_000 * k) + "\u0316".repeat(50_000 * k),
+  ],
+];
+
+for (const encoding of [claude_v3, claude_v4_7]) {
+  const tokenizer = new Tokenizer(encoding);
+  const start = performance.now();
+  tokenizer.count("warm up");
+  console.log(
+    `${encoding.name}: first count (builds the trie) ${(performance.now() - start).toFixed(0)} ms`
+  );
+  for (const [label, make] of inputs) {
+    const one = make(1);
+    const two = make(2);
+    let count1 = 0;
+    let count2 = 0;
+    const t1 = median(() => (count1 = tokenizer.count(one)));
+    const t2 = median(() => (count2 = tokenizer.count(two)));
+    const tokens = `${count1}/${count2} tokens`;
+    console.log(
+      `  ${label.padEnd(14)} ${tokens.padEnd(22)} 1x ${t1.toFixed(0).padStart(5)} ms  2x ${t2.toFixed(0).padStart(5)} ms  2x/1x ${(t2 / t1).toFixed(2)}`
+    );
+  }
+}

package.json

@@ -34,7 +34,11 @@
       "encoding/cl100k_base": ["./dist/encoding/cl100k_base.d.ts"],
       "encoding/o200k_base": ["./dist/encoding/o200k_base.d.ts"],
       "encoding/p50k_base": ["./dist/encoding/p50k_base.d.ts"],
-      "encoding/claude": ["./dist/encoding/claude.d.ts"]
+      "encoding/claude": ["./dist/encoding/claude.d.ts"],
+      "encoding/claude_v3": ["./dist/encoding/claude_v3.d.ts"],
+      "encoding/claude_v4_7": ["./dist/encoding/claude_v4_7.d.ts"],
+      "encoding/claude_v4_8": ["./dist/encoding/claude_v4_8.d.ts"],
+      "encoding/claude_v5": ["./dist/encoding/claude_v5.d.ts"]
     }
   },
   "exports": {
@@ -107,6 +111,46 @@
         "types": "./dist/encoding/claude.d.cts",
         "default": "./dist/encoding/claude.cjs"
       }
+    },
+    "./encoding/claude_v3": {
+      "import": {
+        "types": "./dist/encoding/claude_v3.d.ts",
+        "default": "./dist/encoding/claude_v3.js"
+      },
+      "require": {
+        "types": "./dist/encoding/claude_v3.d.cts",
+        "default": "./dist/encoding/claude_v3.cjs"
+      }
+    },
+    "./encoding/claude_v4_7": {
+      "import": {
+        "types": "./dist/encoding/claude_v4_7.d.ts",
+        "default": "./dist/encoding/claude_v4_7.js"
+      },
+      "require": {
+        "types": "./dist/encoding/claude_v4_7.d.cts",
+        "default": "./dist/encoding/claude_v4_7.cjs"
+      }
+    },
+    "./encoding/claude_v4_8": {
+      "import": {
+        "types": "./dist/encoding/claude_v4_8.d.ts",
+        "default": "./dist/encoding/claude_v4_8.js"
+      },
+      "require": {
+        "types": "./dist/encoding/claude_v4_8.d.cts",
+        "default": "./dist/encoding/claude_v4_8.cjs"
+      }
+    },
+    "./encoding/claude_v5": {
+      "import": {
+        "types": "./dist/encoding/claude_v5.d.ts",
+        "default": "./dist/encoding/claude_v5.js"
+      },
+      "require": {
+        "types": "./dist/encoding/claude_v5.d.cts",
+        "default": "./dist/encoding/claude_v5.cjs"
+      }
     }
   },
   "description": "A faster than tiktoken tokenizer with first-class support for Vercel's AI SDK.",