diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index b8c2b81..d5d7175 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -36,6 +36,8 @@ class RequestHandler: self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + self.parameters = None + self.used = False if not self.cluster_backend_info.get('mode'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) @@ -48,8 +50,6 @@ class RequestHandler: self.offline = False self.selected_model = self.cluster_backend_info['model'] self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) - self.parameters = None - self.used = False if self.token and not self.token.startswith('SYSTEM__'): # "recent_prompters" is only used for stats. redis.zadd('recent_prompters', {self.client_ip: time.time()})