local-llm-server/llm_server/llm/vllm/tokenize.py

import traceback

import requests
import tiktoken

from llm_server import opts

tokenizer = tiktoken.get_encoding("cl100k_base")


def tokenize(prompt: str) -> int:
    if not prompt:
        # The tokenizers have issues when the prompt is None.
        return 0
    try:
        r = requests.post(f'{opts.backend_url}/tokenize', json={'input': prompt}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout)
        j = r.json()
        return j['length']
    except:
        traceback.print_exc()
        return len(tokenizer.encode(prompt)) + 10