cyberes
/
local-llm-server
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

cleanup

This commit is contained in:
Cyberes 2023-10-16 23:47:34 -06:00
parent 2ed0e01db6
commit 7998cfca87
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
llm_server/workers/inferencer.py
View File

@ -20,7 +20,7 @@ STREAM_NAME_PREFIX = 'stream'
def check_cancellation(event, event_id):
"""
This thread checks the pub/sub channel in the background so the main process
isn't bogged down with Redis calls.
isn't bogged down with Redis calls. Otherwise, the main process slows down to 1 token/sec.
:param event:
:param event_id:
:return:
@ -43,8 +43,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str
stream_name = get_stream_name(stream_name)
stream_redis.delete(get_stream_name(stream_name)) # be extra sure
event = threading.Event()
t = threading.Thread(target=check_cancellation, args=(event, event_id))
t.start()
threading.Thread(target=check_cancellation, args=(event, event_id)).start()
try:
response = generator(msg_to_backend, backend_url)
generated_text = ''