cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

t

This commit is contained in:
Cyberes 2023-10-05 18:30:54 -06:00
parent 6be1e9acd3
commit fb8bc05b4c
2 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
llm_server/routes/openai/chat_completions.py
View File

@ -151,6 +151,7 @@ def openai_chat_completions():
# The worker incremented it, we'll decrement it. # The worker incremented it, we'll decrement it.
decrement_ip_count(handler.client_ip, 'processing_ips') decrement_ip_count(handler.client_ip, 'processing_ips')
decr_active_workers(handler.selected_model, handler.backend_url) decr_active_workers(handler.selected_model, handler.backend_url)
print('cleaned up')
return Response(generate(), mimetype='text/event-stream') return Response(generate(), mimetype='text/event-stream')
except Exception: except Exception:

2
llm_server/workers/inferencer.py
View File

@ -69,5 +69,5 @@ def need_to_wait(backend_url: str):
while active_workers >= concurrent_gens: while active_workers >= concurrent_gens:
time.sleep(0.01) time.sleep(0.01)
e = time.time() e = time.time()
if e - s > 0.5: if e - s > 0.1:
print(f'Worker was delayed {e - s} seconds.') print(f'Worker was delayed {e - s} seconds.')