From 09fa69e031a857671d15638c5bd36e164aad6675 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:37:39 -0600 Subject: [PATCH] fix --- llm_server/routes/request_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index b8c2b81..d5d7175 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -36,6 +36,8 @@ class RequestHandler: self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + self.parameters = None + self.used = False if not self.cluster_backend_info.get('mode'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) @@ -48,8 +50,6 @@ class RequestHandler: self.offline = False self.selected_model = self.cluster_backend_info['model'] self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) - self.parameters = None - self.used = False if self.token and not self.token.startswith('SYSTEM__'): # "recent_prompters" is only used for stats. redis.zadd('recent_prompters', {self.client_ip: time.time()})