From 358339d48bff5d2356ba6dff188448c17a25eb41 Mon Sep 17 00:00:00 2001
From: nai-degen <khoners@protonmail.com>
Date: Tue, 8 Aug 2023 17:43:12 -0500
Subject: [PATCH] fixes issue with Claude <EOT> token disallowed

---
 src/tokenization/claude.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/tokenization/claude.ts b/src/tokenization/claude.ts
index a72452f..0f93111 100644
--- a/src/tokenization/claude.ts
+++ b/src/tokenization/claude.ts
@@ -22,6 +22,6 @@ export function getTokenCount(prompt: string, _model: string) {
 
   return {
     tokenizer: "@anthropic-ai/tokenizer",
-    token_count: encoder.encode(prompt).length,
+    token_count: encoder.encode(prompt.normalize("NFKC"), "all").length,
   };
 }