diff --git a/src/tokenization/claude.ts b/src/tokenization/claude.ts index a72452f..0f93111 100644 --- a/src/tokenization/claude.ts +++ b/src/tokenization/claude.ts @@ -22,6 +22,6 @@ export function getTokenCount(prompt: string, _model: string) { return { tokenizer: "@anthropic-ai/tokenizer", - token_count: encoder.encode(prompt).length, + token_count: encoder.encode(prompt.normalize("NFKC"), "all").length, }; }