adjust

llm_server/workers/inferencer.py

@@ -34,7 +34,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str
             # If there is no more data, break the loop
             if not chunk:
                 break
-            message = pubsub.get_message()
+            message = pubsub.get_message(timeout=0.001)
             if message and message['data'] == b'canceled':
                 print('Client canceled generation')
                 response.close()