cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fix

This commit is contained in:
Cyberes 2023-10-05 20:19:57 -06:00
parent da20d1807b
commit ea61766838
1 changed files with 2 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
llm_server/workers/inferencer.py
View File

@ -19,11 +19,11 @@ def worker():
if not selected_model: if not selected_model:
selected_model = backend_info['model'] selected_model = backend_info['model']
need_to_wait(backend_url)
increment_ip_count(client_ip, 'processing_ips') increment_ip_count(client_ip, 'processing_ips')
incr_active_workers(selected_model, backend_url) incr_active_workers(selected_model, backend_url)
need_to_wait(backend_url)
try: try:
if not request_json_body: if not request_json_body:
# This was a dummy request from the streaming handlers. # This was a dummy request from the streaming handlers.
@ -65,7 +65,6 @@ def need_to_wait(backend_url: str):
active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int)
concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1)
s = time.time() s = time.time()
print(active_workers)
while active_workers >= concurrent_gens: while active_workers >= concurrent_gens:
time.sleep(0.01) time.sleep(0.01)
e = time.time() e = time.time()