cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge cluster to master #3

Merged
cyberes merged 163 commits from cluster into master 2023-10-27 19:19:22 -06:00
Conversation 0 Commits 163 Files Changed 91 -1
1 changed files with 0 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 7acaa3c885 - Show all commits

1
llm_server/llm/vllm/tokenize.py
View File

@ -28,7 +28,6 @@ def tokenize(prompt: str, backend_url: str) -> int:
return j['length'] return j['length']
except Exception as e: except Exception as e:
print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}')
raise Exception
return len(tokenizer.encode(chunk)) + 10 return len(tokenizer.encode(chunk)) + 10
# Use a ThreadPoolExecutor to send all chunks to the server at once # Use a ThreadPoolExecutor to send all chunks to the server at once