cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fix copied code

This commit is contained in:
Cyberes 2023-09-25 12:38:02 -06:00
parent 44e692c9cf
commit 3eaabc8c35
2 changed files with 21 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
llm_server/routes/openai/chat_completions.py
View File

@ -65,25 +65,27 @@ def openai_chat_completions():
json_obj = json.loads(json_str.decode())
new = json_obj['text'][0].split(handler.prompt + generated_text)[1]
generated_text = generated_text + new
data = {
"id": f"chatcmpl-{generate_oai_string(30)}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"delta": {
"content": new
},
"finish_reason": None
}
]
}
yield f'data: {json.dumps(data)}\n\n'
except IndexError:
# ????
continue
data = {
"id": f"chatcmpl-{generate_oai_string(30)}",
"object": "chat.completion.chunk",
"created": int(time.time()),
"model": model,
"choices": [
{
"index": 0,
"delta": {
"content": new
},
"finish_reason": None
}
]
}
yield f'data: {json.dumps(data)}\n\n'
yield 'data: [DONE]\n\n'
end_time = time.time()
elapsed_time = end_time - start_time

6
llm_server/routes/v1/generate_stream.py
View File

@ -9,7 +9,7 @@ from ..helpers.client import format_sillytavern_err
from ..helpers.http import require_api_key, validate_json
from ..ooba_request_handler import OobaRequestHandler
from ... import opts
from ...database.database import increment_token_uses, log_prompt
from ...database.database import log_prompt
from ...llm.generator import generator
from ...llm.vllm import tokenize
from ...stream import sock
@ -81,7 +81,9 @@ def stream(ws):
for json_str in json_strs:
if json_str:
try:
json_obj = json.loads(json_str.decode())
new = json_obj['text'][0].split(input_prompt + generated_text)[1]
generated_text = generated_text + new
except IndexError:
# ????
continue
@ -92,8 +94,6 @@ def stream(ws):
'text': new
}))
message_num += 1
generated_text = generated_text + new
partial_response = b'' # Reset the partial response
# If there is no more data, break the loop