llm_server/workers/inferencer.py

@@ -20,7 +20,7 @@ STREAM_NAME_PREFIX = 'stream'
 def check_cancellation(event, event_id):
     """
     This thread checks the pub/sub channel in the background so the main process
-    isn't bogged down with Redis calls.
+    isn't bogged down with Redis calls. Otherwise, the main process slows down to 1 token/sec.
     :param event:
     :param event_id:
     :return:
@@ -43,8 +43,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str
     stream_name = get_stream_name(stream_name)
     stream_redis.delete(get_stream_name(stream_name))  # be extra sure
     event = threading.Event()
-    t = threading.Thread(target=check_cancellation, args=(event, event_id))
-    t.start()
+    threading.Thread(target=check_cancellation, args=(event, event_id)).start()
     try:
         response = generator(msg_to_backend, backend_url)
         generated_text = ''