cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge cluster to master #3

Merged
cyberes merged 163 commits from cluster into master 2023-10-27 19:19:22 -06:00
Conversation 0 Commits 163 Files Changed 91 +2 -2
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 09fa69e031 - Show all commits

4
llm_server/routes/request_handler.py
View File

@ -36,6 +36,8 @@ class RequestHandler:
self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token)
self.backend_url = get_a_cluster_backend(selected_model) self.backend_url = get_a_cluster_backend(selected_model)
self.cluster_backend_info = cluster_config.get_backend(self.backend_url) self.cluster_backend_info = cluster_config.get_backend(self.backend_url)
self.parameters = None
self.used = False
if not self.cluster_backend_info.get('mode'): if not self.cluster_backend_info.get('mode'):
print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info)
@ -48,8 +50,6 @@ class RequestHandler:
self.offline = False self.offline = False
self.selected_model = self.cluster_backend_info['model'] self.selected_model = self.cluster_backend_info['model']
self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url)
self.parameters = None
self.used = False
if self.token and not self.token.startswith('SYSTEM__'): if self.token and not self.token.startswith('SYSTEM__'):
# "recent_prompters" is only used for stats. # "recent_prompters" is only used for stats.
redis.zadd('recent_prompters', {self.client_ip: time.time()}) redis.zadd('recent_prompters', {self.client_ip: time.time()})