From 5a61bdccd4843fa7e166b9067c9e4a5df4cfaa5b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:07:59 -0600 Subject: [PATCH] f --- llm_server/llm/__init__.py | 10 +++++----- llm_server/llm/vllm/tokenize.py | 9 +++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 7736684..5f87c61 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -1,15 +1,15 @@ from llm_server.cluster.cluster_config import cluster_config from llm_server.llm import oobabooga, vllm -from llm_server.custom_redis import redis def get_token_count(prompt: str, backend_url: str): - if not isinstance(prompt, str): - print(prompt) - - assert isinstance(prompt, str) assert isinstance(backend_url, str) + if not prompt: + # The tokenizers have issues when the prompt is None. + return 0 + assert isinstance(prompt, str) + backend_mode = cluster_config.get_backend(backend_url)['mode'] if backend_mode == 'vllm': return vllm.tokenize(prompt, backend_url) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index db2c49d..8b18073 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -9,16 +9,17 @@ from llm_server.cluster.cluster_config import cluster_config def tokenize(prompt: str, backend_url: str) -> int: assert backend_url - assert isinstance(prompt, str) assert isinstance(backend_url, str) + if not prompt: + # The tokenizers have issues when the prompt is None. + return 0 + assert isinstance(prompt, str) + # The backend could have died between when the request was # submitted and now, so let's double check it's still online. backend_url = cluster_config.validate_backend(backend_url) - if not prompt: - # The tokenizers have issues when the prompt is None. - return 0 tokenizer = tiktoken.get_encoding("cl100k_base") # Split the prompt into 300 character chunks