cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

t

This commit is contained in:
Cyberes 2023-10-05 18:16:17 -06:00
parent 10eb6269b7
commit 6be1e9acd3
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llm_server/workers/inferencer.py
View File

@ -65,8 +65,8 @@ def need_to_wait(backend_url: str):
active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int)
concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1)
s = time.time() s = time.time()
print(active_workers >= concurrent_gens, active_workers, concurrent_gens)
while active_workers >= concurrent_gens: while active_workers >= concurrent_gens:
print('worker waiting')
time.sleep(0.01) time.sleep(0.01)
e = time.time() e = time.time()
if e - s > 0.5: if e - s > 0.5: