From 24aab3cd938c3a054ee22d1984b647072167c856 Mon Sep 17 00:00:00 2001
From: Cyberes <cyberes@evulid.cc>
Date: Sun, 15 Oct 2023 20:59:11 -0600
Subject: [PATCH] fix streaming disabled

---
 llm_server/routes/openai/chat_completions.py | 2 +-
 llm_server/routes/openai/completions.py      | 2 +-
 llm_server/routes/v1/generate_stream.py      | 2 +-
 llm_server/workers/mainer.py                 | 1 +
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py
index 87a7330..b1e74be 100644
--- a/llm_server/routes/openai/chat_completions.py
+++ b/llm_server/routes/openai/chat_completions.py
@@ -40,7 +40,7 @@ def openai_chat_completions(model_name=None):
                 return 'Internal server error', 500
         else:
             if not opts.enable_streaming:
-                return
+                return 'Streaming disabled', 403
 
             handler.parameters, _ = handler.get_parameters()
             handler.request_json_body = {
diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py
index 2cd8578..dc7f9e6 100644
--- a/llm_server/routes/openai/completions.py
+++ b/llm_server/routes/openai/completions.py
@@ -87,7 +87,7 @@ def openai_completions(model_name=None):
             return response, 200
         else:
             if not opts.enable_streaming:
-                return 'DISABLED', 401
+                return 'Streaming disabled', 403
 
             event_id = None
             start_time = time.time()
diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py
index 79be511..332fe4c 100644
--- a/llm_server/routes/v1/generate_stream.py
+++ b/llm_server/routes/v1/generate_stream.py
@@ -61,7 +61,7 @@ def do_stream(ws, model_name):
                       )
 
         if not opts.enable_streaming:
-            return 'Streaming is disabled', 500
+            return 'Streaming disabled', 403
 
         r_headers = dict(request.headers)
         r_url = request.url
diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py
index d342f4b..e06e803 100644
--- a/llm_server/workers/mainer.py
+++ b/llm_server/workers/mainer.py
@@ -36,6 +36,7 @@ def main_background_thread():
             except Exception as e:
                 print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}')
 
+        # TODO: test
         backends = priority_queue.get_backends()
         for backend_url in backends:
             queue = RedisPriorityQueue(backend_url)