cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

Merge cluster to master #3

Merged
cyberes merged 163 commits from cluster into master 2023-10-27 19:19:22 -06:00
Conversation 0 Commits 163 Files Changed 91 +2 -4
1 changed files with 2 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 1a7f22ec55 - Show all commits

6
llm_server/routes/v1/generate_stats.py
View File

@ -53,7 +53,7 @@ def generate_stats(regen: bool = False):
'openaiKeys': '∞', 'openaiKeys': '∞',
'anthropicKeys': '∞', 'anthropicKeys': '∞',
}, },
'backend_info': redis.get_dict('backend_info') if opts.show_backend_info else None, 'backends': {},
} }
# TODO: have get_model_choices() return all the info so we don't have to loop over the backends ourself # TODO: have get_model_choices() return all the info so we don't have to loop over the backends ourself
@ -64,7 +64,7 @@ def generate_stats(regen: bool = False):
if not backend_info['online']: if not backend_info['online']:
continue continue
backend_uptime = int((datetime.now() - datetime.fromtimestamp(backend_info['startup_time'])).total_seconds()) if opts.show_uptime else None backend_uptime = int((datetime.now() - datetime.fromtimestamp(backend_info['startup_time'])).total_seconds()) if opts.show_uptime else None
output['backend_info'][backend_info['hash']] = { output['backends'][backend_info['hash']] = {
'uptime': backend_uptime, 'uptime': backend_uptime,
'max_tokens': backend_info['model_config']['max_position_embeddings'], 'max_tokens': backend_info['model_config']['max_position_embeddings'],
'model': backend_info['model'], 'model': backend_info['model'],
@ -72,8 +72,6 @@ def generate_stats(regen: bool = False):
'nvidia': backend_info['nvidia'], 'nvidia': backend_info['nvidia'],
'priority': backend_info['priority'], 'priority': backend_info['priority'],
} }
else:
output['backend_info'] = {}
result = deep_sort(output) result = deep_sort(output)