llm_server/routes/openai/chat_completions.py

@@ -40,7 +40,7 @@ def openai_chat_completions(model_name=None):
                 return 'Internal server error', 500
         else:
             if not opts.enable_streaming:
-                return
+                return 'Streaming disabled', 403
 
             handler.parameters, _ = handler.get_parameters()
             handler.request_json_body = {

llm_server/routes/openai/completions.py

@@ -87,7 +87,7 @@ def openai_completions(model_name=None):
             return response, 200
         else:
             if not opts.enable_streaming:
-                return 'DISABLED', 401
+                return 'Streaming disabled', 403
 
             event_id = None
             start_time = time.time()

llm_server/routes/v1/generate_stream.py

@@ -61,7 +61,7 @@ def do_stream(ws, model_name):
                       )
 
         if not opts.enable_streaming:
-            return 'Streaming is disabled', 500
+            return 'Streaming disabled', 403
 
         r_headers = dict(request.headers)
         r_url = request.url

llm_server/workers/mainer.py

@@ -36,6 +36,7 @@ def main_background_thread():
             except Exception as e:
                 print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}')
 
+        # TODO: test
         backends = priority_queue.get_backends()
         for backend_url in backends:
             queue = RedisPriorityQueue(backend_url)