llm_server/database/database.py

@@ -52,6 +52,7 @@ def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen
 
         backend_info = cluster_config.get_backend(backend_url)
         running_model = backend_info.get('model')
+        backend_mode = backend_info['mode']
         timestamp = int(time.time())
         cursor = database.cursor()
         try:
@@ -60,7 +61,7 @@ def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen
             (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp)
             VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
             """,
-                           (ip, token, running_model, cluster_config['mode'], backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp))
+                           (ip, token, running_model, backend_mode, backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp))
         finally:
             cursor.close()
 

llm_server/routes/request_handler.py

@@ -45,7 +45,7 @@ class RequestHandler:
         self.parameters = None
         self.used = False
 
-        if not self.token.startswith('SYSTEM__'):
+        if self.token and not self.token.startswith('SYSTEM__'):
             # "recent_prompters" is only used for stats.
             redis.zadd('recent_prompters', {self.client_ip: time.time()})