cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

track down keyerror

This commit is contained in:
Cyberes 2023-10-20 13:57:20 -06:00
parent 4f5b2dbecb
commit 0abd4b94fb
1 changed files with 20 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
llm_server/cluster/cluster_config.py
View File

@ -1,5 +1,6 @@
import hashlib import hashlib
import pickle import pickle
import traceback
from llm_server import opts from llm_server import opts
from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle
@ -75,24 +76,28 @@ def get_backends():
priority = b['priority'] priority = b['priority']
result[k] = {'status': status, 'priority': priority} result[k] = {'status': status, 'priority': priority}
if not opts.prioritize_by_size: try:
online_backends = sorted( if not opts.prioritize_by_size:
((url, info) for url, info in backends.items() if info['online']), online_backends = sorted(
((url, info) for url, info in backends.items() if info['online']),
key=lambda kv: -kv[1]['priority'],
reverse=True
)
else:
online_backends = sorted(
((url, info) for url, info in backends.items() if info['online']),
key=lambda kv: estimate_model_size(kv[1]['model_config']),
reverse=True
)
offline_backends = sorted(
((url, info) for url, info in backends.items() if not info['online']),
key=lambda kv: -kv[1]['priority'], key=lambda kv: -kv[1]['priority'],
reverse=True reverse=True
) )
else: return [url for url, info in online_backends], [url for url, info in offline_backends]
online_backends = sorted( except KeyError:
((url, info) for url, info in backends.items() if info['online']), traceback.print_exc()
key=lambda kv: estimate_model_size(kv[1]['model_config']), print(backends)
reverse=True
)
offline_backends = sorted(
((url, info) for url, info in backends.items() if not info['online']),
key=lambda kv: -kv[1]['priority'],
reverse=True
)
return [url for url, info in online_backends], [url for url, info in offline_backends]
def get_a_cluster_backend(model=None): def get_a_cluster_backend(model=None):