cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

adjust stats

This commit is contained in:
Cyberes 2023-09-30 20:42:48 -06:00
parent 91ba2fad1b
commit 1151bb5475
2 changed files with 26 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
llm_server/cluster/model_choices.py
View File

@ -54,7 +54,7 @@ def get_model_choices(regen: bool = False):
'estimated_wait': estimated_wait_sec,
'queued': proompters_in_queue,
'processing': active_gen_workers,
'avg_generation_time': average_generation_elapsed_sec
'avg_generation_time': average_generation_elapsed_sec,
}
if len(context_size):
@ -63,6 +63,8 @@ def get_model_choices(regen: bool = False):
model_choices = dict(sorted(model_choices.items()))
default_backend = get_a_cluster_backend()
default_backend_dict = {}
if default_backend:
default_backend_info = cluster_config.get_backend(default_backend)
default_context_size = default_backend_info['model_config']['max_position_embeddings']
default_average_generation_elapsed_sec = default_backend_info.get('average_generation_elapsed_sec')
@ -80,7 +82,8 @@ def get_model_choices(regen: bool = False):
'context_size': default_context_size,
'hash': default_backend_info['hash'],
'model': default_backend_info['model'],
'avg_generation_time': default_average_generation_elapsed_sec
'avg_generation_time': default_average_generation_elapsed_sec,
'online': True
}
redis.setp('model_choices', (model_choices, default_backend_dict))

4
llm_server/routes/request_handler.py
View File

@ -39,6 +39,10 @@ class RequestHandler:
self.token_priority, self.token_simultaneous_ip = self.get_token_ratelimit()
self.backend_url = get_a_cluster_backend(selected_model)
self.cluster_backend_info = cluster_config.get_backend(self.backend_url)
if not self.cluster_backend_info.get('mode'):
print(self.backend_url, self.cluster_backend_info)
self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url)
self.parameters = None
self.used = False