llm_server/routes/openai/chat_completions.py

@@ -120,7 +120,7 @@ def openai_chat_completions(model_name=None):
                                     timestamp = int(stream_index.decode('utf-8').split('-')[0])
                                     data = ujson.loads(item[b'data'])
                                     if data['error']:
-                                        print(data['error'])
+                                        print('OAI streaming error:', data['error'])
                                         yield 'data: [DONE]\n\n'
                                         return
                                     elif data['new']:

llm_server/routes/openai/completions.py

@@ -164,7 +164,7 @@ def openai_completions(model_name=None):
                                     timestamp = int(stream_index.decode('utf-8').split('-')[0])
                                     data = ujson.loads(item[b'data'])
                                     if data['error']:
-                                        print(data['error'])
+                                        print('OAI streaming error:', data['error'])
                                         yield 'data: [DONE]\n\n'
                                         return
                                     elif data['new']:

llm_server/workers/inferencer.py

@@ -66,6 +66,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str
                             json_obj = json.loads(json_str.decode())
                             new = json_obj['text'][0].split(prompt + generated_text)[1]
                             generated_text = generated_text + new
+                            print(new)
                         except IndexError:
                             # ????
                             continue