llm_server/workers/inferencer.py

@@ -19,11 +19,11 @@ def worker():
         if not selected_model:
             selected_model = backend_info['model']
 
-        need_to_wait(backend_url)
-
         increment_ip_count(client_ip, 'processing_ips')
         incr_active_workers(selected_model, backend_url)
 
+        need_to_wait(backend_url)
+
         try:
             if not request_json_body:
                 # This was a dummy request from the streaming handlers.
@@ -65,7 +65,6 @@ def need_to_wait(backend_url: str):
     active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int)
     concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1)
     s = time.time()
-    print(active_workers)
     while active_workers >= concurrent_gens:
         time.sleep(0.01)
     e = time.time()