cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

misc adjustments

This commit is contained in:
Cyberes 2024-05-07 22:56:36 -06:00
parent fe23a2282f
commit 20366fbd08
4 changed files with 4 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llm_server/database/create.py
View File

@ -12,7 +12,7 @@ def create_db():
backend_mode text COLLATE pg_catalog."default" NOT NULL, backend_mode text COLLATE pg_catalog."default" NOT NULL,
backend_url text COLLATE pg_catalog."default" NOT NULL, backend_url text COLLATE pg_catalog."default" NOT NULL,
request_url text COLLATE pg_catalog."default" NOT NULL, request_url text COLLATE pg_catalog."default" NOT NULL,
generation_time double precision NOT NULL, generation_time double precision,
prompt text COLLATE pg_catalog."default" NOT NULL, prompt text COLLATE pg_catalog."default" NOT NULL,
prompt_tokens integer NOT NULL, prompt_tokens integer NOT NULL,
response text COLLATE pg_catalog."default" NOT NULL, response text COLLATE pg_catalog."default" NOT NULL,

2
llm_server/database/database.py
View File

@ -34,7 +34,7 @@ def do_db_log(ip: str, token: str, prompt: str, response: Union[str, None], gen_
# Sometimes we may want to insert null into the DB, but # Sometimes we may want to insert null into the DB, but
# usually we want to insert a float. # usually we want to insert a float.
if gen_time: if gen_time is not None:
gen_time = round(gen_time, 3) gen_time = round(gen_time, 3)
if is_error: if is_error:
gen_time = None gen_time = None

4
llm_server/llm/openai/oai_to_vllm.py
View File

@ -97,7 +97,7 @@ def return_oai_internal_server_error():
return jsonify({ return jsonify({
"error": { "error": {
"message": "Internal server error", "message": "Internal server error",
"type": "auth_subrequest_error", "type": None,
"param": None, "param": None,
"code": "internal_error" "code": "internal_error"
} }
@ -110,6 +110,6 @@ def return_oai_invalid_request_error(msg: str = None):
"message": msg, "message": msg,
"type": "invalid_request_error", "type": "invalid_request_error",
"param": None, "param": None,
"code": "model_not_found" "code": None
} }
}), 404 }), 404

2
llm_server/routes/openai_request_handler.py
View File

@ -46,8 +46,6 @@ class OpenAIRequestHandler(RequestHandler):
self.prompt = transform_messages_to_prompt(oai_messages, disable_openai_handling) self.prompt = transform_messages_to_prompt(oai_messages, disable_openai_handling)
self.request_json_body = oai_to_vllm(self.request_json_body, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) self.request_json_body = oai_to_vllm(self.request_json_body, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode'])
print(self.prompt)
request_valid, invalid_response = self.validate_request() request_valid, invalid_response = self.validate_request()
if not request_valid: if not request_valid:
return invalid_response return invalid_response