llm_server/routes/request_handler.py

@@ -36,6 +36,8 @@ class RequestHandler:
         self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token)
         self.backend_url = get_a_cluster_backend(selected_model)
         self.cluster_backend_info = cluster_config.get_backend(self.backend_url)
+        self.parameters = None
+        self.used = False
 
         if not self.cluster_backend_info.get('mode'):
             print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info)
@@ -48,8 +50,6 @@ class RequestHandler:
             self.offline = False
             self.selected_model = self.cluster_backend_info['model']
             self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url)
-            self.parameters = None
-            self.used = False
             if self.token and not self.token.startswith('SYSTEM__'):
                 # "recent_prompters" is only used for stats.
                 redis.zadd('recent_prompters', {self.client_ip: time.time()})