cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge cluster to master #3

Merged
cyberes merged 163 commits from cluster into master 2023-10-27 19:19:22 -06:00
Conversation 0 Commits 163 Files Changed 91 +4
1 changed files with 4 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 07d6f6d8e9 - Show all commits

4
llm_server/routes/openai_request_handler.py
View File

@ -39,6 +39,7 @@ class OpenAIRequestHandler(RequestHandler):
return invalid_response return invalid_response
if opts.openai_api_key and is_api_key_moderated(self.token): if opts.openai_api_key and is_api_key_moderated(self.token):
print('moderating')
try: try:
# Gather the last message from the user and all preceeding system messages # Gather the last message from the user and all preceeding system messages
msg_l = self.request.json['messages'].copy() msg_l = self.request.json['messages'].copy()
@ -59,8 +60,10 @@ class OpenAIRequestHandler(RequestHandler):
print(traceback.format_exc()) print(traceback.format_exc())
# TODO: support Ooba # TODO: support Ooba
print('converting to vllm')
self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode']) self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode'])
print('generating')
llm_request = {**self.parameters, 'prompt': self.prompt} llm_request = {**self.parameters, 'prompt': self.prompt}
(success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request)
@ -70,6 +73,7 @@ class OpenAIRequestHandler(RequestHandler):
print('sent success response') print('sent success response')
return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code
else: else:
print(backend_response)
return backend_response, backend_response_status_code return backend_response, backend_response_status_code
def handle_ratelimited(self, do_log: bool = True): def handle_ratelimited(self, do_log: bool = True):