From ea61766838582d19547b50cad3a63d000da6e527 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 20:19:57 -0600 Subject: [PATCH] fix --- llm_server/workers/inferencer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 05a2981..4a1e61f 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -19,11 +19,11 @@ def worker(): if not selected_model: selected_model = backend_info['model'] - need_to_wait(backend_url) - increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) + need_to_wait(backend_url) + try: if not request_json_body: # This was a dummy request from the streaming handlers. @@ -65,7 +65,6 @@ def need_to_wait(backend_url: str): active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) s = time.time() - print(active_workers) while active_workers >= concurrent_gens: time.sleep(0.01) e = time.time()