27 lines
834 B
Python
27 lines
834 B
Python
from llm_server.cluster.redis_config_cache import RedisClusterStore
|
|
from llm_server.llm.info import get_running_model
|
|
|
|
|
|
def test_backend(backend_url: str):
|
|
running_model, err = get_running_model(backend_url)
|
|
if not running_model:
|
|
return False
|
|
return True
|
|
|
|
|
|
def get_best_backends():
|
|
cluster_config = RedisClusterStore('cluster_config')
|
|
backends = cluster_config.all()
|
|
result = {}
|
|
for k, v in backends.items():
|
|
b = cluster_config.get_backend(k)
|
|
status = b['online']
|
|
priority = b['priority']
|
|
result[k] = {'status': status, 'priority': priority}
|
|
online_backends = sorted(
|
|
((url, info) for url, info in backends.items() if info['online']),
|
|
key=lambda kv: kv[1]['priority'],
|
|
reverse=True
|
|
)
|
|
return [url for url, info in online_backends]
|