import traceback

import requests
import tiktoken

from llm_server import opts

tokenizer = tiktoken.get_encoding("cl100k_base")


def tokenize(prompt: str) -> int:
    try:
        r = requests.post(f'{opts.backend_url}/tokenize', json={'input': prompt}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout)
        j = r.json()
        return j['length']
    except:
        print(traceback.format_exc())
        return len(tokenizer.encode(prompt)) + 10