fixes issue with Claude <EOT> token disallowed

2023-08-08 17:43:12 -05:00 · 2023-08-08 17:43:12 -05:00 · 358339d48b
parent c8d8e2e58f
commit 358339d48b
1 changed files with 1 additions and 1 deletions
--- a/src/tokenization/claude.ts
+++ b/src/tokenization/claude.ts
@ -22,6 +22,6 @@ export function getTokenCount(prompt: string, _model: string) {

  return {
    tokenizer: "@anthropic-ai/tokenizer",
-    token_count: encoder.encode(prompt).length,
+    token_count: encoder.encode(prompt.normalize("NFKC"), "all").length,
  };
 }