From 6f657917955d0afdd8f0712a27142afadeae0197 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:40:07 -0600 Subject: [PATCH] adjust --- llm_server/workers/inferencer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 765ec8b..4bc1472 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -34,7 +34,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str # If there is no more data, break the loop if not chunk: break - message = pubsub.get_message() + message = pubsub.get_message(timeout=0.001) if message and message['data'] == b'canceled': print('Client canceled generation') response.close()