Mirrors
/
oai-reverse-proxy
mirror of https://gitgud.io/khanon/oai-reverse-proxy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fixes issue with Claude <EOT> token disallowed

This commit is contained in:
nai-degen 2023-08-08 17:43:12 -05:00
parent c8d8e2e58f
commit 358339d48b
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/tokenization/claude.ts
View File

@ -22,6 +22,6 @@ export function getTokenCount(prompt: string, _model: string) {
return { return {
tokenizer: "@anthropic-ai/tokenizer", tokenizer: "@anthropic-ai/tokenizer",
token_count: encoder.encode(prompt).length, token_count: encoder.encode(prompt.normalize("NFKC"), "all").length,
}; };
} }