From 90adffaec8951e87d4a35b6ee7c8cb4b9d2cba30 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 17 Oct 2023 12:32:41 -0600 Subject: [PATCH] test --- llm_server/routes/openai/chat_completions.py | 2 +- llm_server/routes/openai/completions.py | 2 +- llm_server/workers/inferencer.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 520652e..0ef28ff 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -120,7 +120,7 @@ def openai_chat_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: - print(data['error']) + print('OAI streaming error:', data['error']) yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index c78ac64..d77ff0d 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -164,7 +164,7 @@ def openai_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: - print(data['error']) + print('OAI streaming error:', data['error']) yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 0738c1b..a1dd749 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -66,6 +66,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str json_obj = json.loads(json_str.decode()) new = json_obj['text'][0].split(prompt + generated_text)[1] generated_text = generated_text + new + print(new) except IndexError: # ???? continue