cyberes
/
local-llm-server
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fix stats page

This commit is contained in:
Cyberes 2023-10-03 20:42:53 -06:00
parent f88e2362c5
commit 67f5df9bb9
3 changed files with 11 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
llm_server/cluster/model_choices.py
View File

@ -61,7 +61,10 @@ def get_model_choices(regen: bool = False):
model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper())) model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper()))
default_backend_url = get_a_cluster_backend() default_backend_url = get_a_cluster_backend()
default_model = cluster_config.get_backend(default_backend_url)['model'] default_backend_info = cluster_config.get_backend(default_backend_url)
if not default_backend_info.get('model'):
return None, None
default_model = default_backend_info['model']
redis.setp('model_choices', (model_choices, default_model)) redis.setp('model_choices', (model_choices, default_model))
return model_choices, default_model return model_choices, default_model

17
llm_server/routes/v1/generate_stats.py
View File

@ -2,7 +2,6 @@ import time
from datetime import datetime from datetime import datetime
from llm_server import opts from llm_server import opts
from llm_server.cluster.backend import get_a_cluster_backend
from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.cluster_config import cluster_config
from llm_server.cluster.model_choices import get_model_choices from llm_server.cluster.model_choices import get_model_choices
from llm_server.custom_redis import redis from llm_server.custom_redis import redis
@ -17,17 +16,17 @@ def generate_stats(regen: bool = False):
if c: if c:
return c return c
default_backend_url = get_a_cluster_backend() model_choices, default_model = get_model_choices(regen=True)
default_backend_info = cluster_config.get_backend(default_backend_url) if not model_choices or not default_model:
if not default_backend_info.get('mode'): return 'Please wait for Redis to be populated...'
return
base_client_api = redis.get('base_client_api', dtype=str) base_client_api = redis.get('base_client_api', dtype=str)
proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf')) proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf'))
output = { output = {
'default': { 'models': {
'model': default_backend_info['model'], 'choices': model_choices,
'backend': default_backend_url, 'default': default_model,
}, },
'stats': { 'stats': {
'proompters': { 'proompters': {
@ -76,8 +75,6 @@ def generate_stats(regen: bool = False):
else: else:
output['backend_info'] = {} output['backend_info'] = {}
output['default_model'] = get_model_choices(regen=True)[1]
result = deep_sort(output) result = deep_sort(output)
# It may take a bit to get the base client API, so don't cache until then. # It may take a bit to get the base client API, so don't cache until then.

1
server.py
View File

@ -24,7 +24,6 @@ from llm_server.routes.server_error import handle_server_error
from llm_server.routes.v1 import bp from llm_server.routes.v1 import bp
from llm_server.sock import init_socketio from llm_server.sock import init_socketio
# TODO: redis SCAN vs KEYS?? # TODO: redis SCAN vs KEYS??
# TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s)
# TODO: is frequency penalty the same as ooba repetition penalty??? # TODO: is frequency penalty the same as ooba repetition penalty???