test

llm_server/llm/openai/moderation.py

@@ -10,7 +10,7 @@ def check_moderation_endpoint(prompt: str):
     }
     response = requests.post('https://api.openai.com/v1/moderations', headers=headers, json={"input": prompt}, timeout=10)
     if response.status_code != 200:
-        print(response)
+        print('moderation failed:', response)
         response.raise_for_status()
     response = response.json()
 

llm_server/routes/openai_request_handler.py

@@ -73,7 +73,6 @@ class OpenAIRequestHandler(RequestHandler):
             print('sent success response')
             return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code
         else:
-            print(backend_response_status_code, backend_response.data)
             return backend_response, backend_response_status_code
 
     def handle_ratelimited(self, do_log: bool = True):

server.py

@@ -1,3 +1,5 @@
+from llm_server.routes.queue import priority_queue
+
 try:
     import gevent.monkey
 
@@ -95,6 +97,8 @@ create_db()
 @app.route('/api/openai')
 @flask_cache.cached(timeout=10)
 def home():
+    print(len(priority_queue))
+
     base_client_api = redis.get('base_client_api', dtype=str)
     stats = generate_stats()
     model_choices, default_model = get_model_choices()