t

llm_server/routes/openai/chat_completions.py

@@ -151,6 +151,7 @@ def openai_chat_completions():
                             # The worker incremented it, we'll decrement it.
                             decrement_ip_count(handler.client_ip, 'processing_ips')
                             decr_active_workers(handler.selected_model, handler.backend_url)
+                            print('cleaned up')
 
                     return Response(generate(), mimetype='text/event-stream')
                 except Exception:

llm_server/workers/inferencer.py

@@ -69,5 +69,5 @@ def need_to_wait(backend_url: str):
     while active_workers >= concurrent_gens:
         time.sleep(0.01)
     e = time.time()
-    if e - s > 0.5:
+    if e - s > 0.1:
         print(f'Worker was delayed {e - s} seconds.')