This repository has been archived on 2024-10-27. You can view files and clone it, but cannot push or open issues or pull requests.
|
import tiktoken
|
|
|
|
tokenizer = tiktoken.get_encoding("cl100k_base")
|
|
|
|
|
|
def tokenize(prompt: str) -> int:
|
|
return len(tokenizer.encode(prompt)) + 10
|