cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

test

This commit is contained in:
Cyberes 2023-10-16 23:29:17 -06:00
parent 70cf6843e5
commit c3c053e071
3 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
llm_server/routes/openai/chat_completions.py
View File

@ -157,8 +157,8 @@ def openai_chat_completions(model_name=None):
traceback.print_exc() traceback.print_exc()
yield 'data: [DONE]\n\n' yield 'data: [DONE]\n\n'
finally: finally:
if event: # if event:
redis.lpush(f'notifications:{event.event_id}', 'canceled') # redis.lpush(f'notifications:{event.event_id}', 'canceled')
stream_redis.delete(stream_name) stream_redis.delete(stream_name)
return Response(generate(), mimetype='text/event-stream') return Response(generate(), mimetype='text/event-stream')

4
llm_server/routes/v1/generate_stream.py
View File

@ -185,8 +185,8 @@ def do_stream(ws, model_name):
backend_url=handler.backend_url backend_url=handler.backend_url
) )
finally: finally:
if event_id: # if event_id:
redis.lpush(f'notifications:{event_id}', 'canceled') # redis.lpush(f'notifications:{event_id}', 'canceled')
try: try:
# Must close the connection or greenlets will complain. # Must close the connection or greenlets will complain.
ws.close() ws.close()

10
llm_server/workers/inferencer.py
View File

@ -33,11 +33,11 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str
# If there is no more data, break the loop # If there is no more data, break the loop
if not chunk: if not chunk:
break break
message = redis.lpop(f'notifications:{event_id}') # message = redis.lpop(f'notifications:{event_id}')
if message and message.decode('utf-8') == 'canceled': # if message and message.decode('utf-8') == 'canceled':
print('Client canceled generation') # print('Client canceled generation')
response.close() # response.close()
return # return
partial_response += chunk partial_response += chunk
if partial_response.endswith(b'\x00'): if partial_response.endswith(b'\x00'):