cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fix copied code

This commit is contained in:
Cyberes 2023-09-25 12:38:02 -06:00
parent 44e692c9cf
commit 3eaabc8c35
2 changed files with 21 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
llm_server/routes/openai/chat_completions.py
View File

@ -65,6 +65,10 @@ def openai_chat_completions():
json_obj = json.loads(json_str.decode()) json_obj = json.loads(json_str.decode())
new = json_obj['text'][0].split(handler.prompt + generated_text)[1] new = json_obj['text'][0].split(handler.prompt + generated_text)[1]
generated_text = generated_text + new generated_text = generated_text + new
except IndexError:
# ????
continue
data = { data = {
"id": f"chatcmpl-{generate_oai_string(30)}", "id": f"chatcmpl-{generate_oai_string(30)}",
"object": "chat.completion.chunk", "object": "chat.completion.chunk",
@ -81,8 +85,6 @@ def openai_chat_completions():
] ]
} }
yield f'data: {json.dumps(data)}\n\n' yield f'data: {json.dumps(data)}\n\n'
except IndexError:
continue
yield 'data: [DONE]\n\n' yield 'data: [DONE]\n\n'
end_time = time.time() end_time = time.time()

6
llm_server/routes/v1/generate_stream.py
View File

@ -9,7 +9,7 @@ from ..helpers.client import format_sillytavern_err
from ..helpers.http import require_api_key, validate_json from ..helpers.http import require_api_key, validate_json
from ..ooba_request_handler import OobaRequestHandler from ..ooba_request_handler import OobaRequestHandler
from ... import opts from ... import opts
from ...database.database import increment_token_uses, log_prompt from ...database.database import log_prompt
from ...llm.generator import generator from ...llm.generator import generator
from ...llm.vllm import tokenize from ...llm.vllm import tokenize
from ...stream import sock from ...stream import sock
@ -81,7 +81,9 @@ def stream(ws):
for json_str in json_strs: for json_str in json_strs:
if json_str: if json_str:
try: try:
json_obj = json.loads(json_str.decode())
new = json_obj['text'][0].split(input_prompt + generated_text)[1] new = json_obj['text'][0].split(input_prompt + generated_text)[1]
generated_text = generated_text + new
except IndexError: except IndexError:
# ???? # ????
continue continue
@ -92,8 +94,6 @@ def stream(ws):
'text': new 'text': new
})) }))
message_num += 1 message_num += 1
generated_text = generated_text + new
partial_response = b'' # Reset the partial response partial_response = b'' # Reset the partial response
# If there is no more data, break the loop # If there is no more data, break the loop