test

llm_server/routes/openai_request_handler.py

@@ -39,6 +39,7 @@ class OpenAIRequestHandler(RequestHandler):
             return invalid_response
 
         if opts.openai_api_key and is_api_key_moderated(self.token):
+            print('moderating')
             try:
                 # Gather the last message from the user and all preceeding system messages
                 msg_l = self.request.json['messages'].copy()
@@ -59,8 +60,10 @@ class OpenAIRequestHandler(RequestHandler):
                 print(traceback.format_exc())
 
         # TODO: support Ooba
+        print('converting to vllm')
         self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode'])
 
+        print('generating')
         llm_request = {**self.parameters, 'prompt': self.prompt}
         (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request)
 
@@ -70,6 +73,7 @@ class OpenAIRequestHandler(RequestHandler):
             print('sent success response')
             return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code
         else:
+            print(backend_response)
             return backend_response, backend_response_status_code
 
     def handle_ratelimited(self, do_log: bool = True):