cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

test

This commit is contained in:
Cyberes 2023-10-17 12:32:41 -06:00
parent 4c2c164ce1
commit 90adffaec8
3 changed files with 3 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llm_server/routes/openai/chat_completions.py
View File

@ -120,7 +120,7 @@ def openai_chat_completions(model_name=None):
timestamp = int(stream_index.decode('utf-8').split('-')[0]) timestamp = int(stream_index.decode('utf-8').split('-')[0])
data = ujson.loads(item[b'data']) data = ujson.loads(item[b'data'])
if data['error']: if data['error']:
print(data['error']) print('OAI streaming error:', data['error'])
yield 'data: [DONE]\n\n' yield 'data: [DONE]\n\n'
return return
elif data['new']: elif data['new']:

2
llm_server/routes/openai/completions.py
View File

@ -164,7 +164,7 @@ def openai_completions(model_name=None):
timestamp = int(stream_index.decode('utf-8').split('-')[0]) timestamp = int(stream_index.decode('utf-8').split('-')[0])
data = ujson.loads(item[b'data']) data = ujson.loads(item[b'data'])
if data['error']: if data['error']:
print(data['error']) print('OAI streaming error:', data['error'])
yield 'data: [DONE]\n\n' yield 'data: [DONE]\n\n'
return return
elif data['new']: elif data['new']:

1
llm_server/workers/inferencer.py
View File

@ -66,6 +66,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str
json_obj = json.loads(json_str.decode()) json_obj = json.loads(json_str.decode())
new = json_obj['text'][0].split(prompt + generated_text)[1] new = json_obj['text'][0].split(prompt + generated_text)[1]
generated_text = generated_text + new generated_text = generated_text + new
print(new)
except IndexError: except IndexError:
# ???? # ????
continue continue