from llm_server import opts from llm_server.cluster.redis_config_cache import RedisClusterStore from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle from llm_server.cluster.stores import redis_running_models from llm_server.routes.helpers.model import estimate_model_size cluster_config = RedisClusterStore('cluster_config') def get_backends(): backends = cluster_config.all() result = {} for k, v in backends.items(): b = cluster_config.get_backend(k) status = b.get('online', False) priority = b['priority'] result[k] = {'status': status, 'priority': priority} if not opts.prioritize_by_size: online_backends = sorted( ((url, info) for url, info in backends.items() if info['online']), key=lambda kv: -kv[1]['priority'], reverse=True ) else: online_backends = sorted( ((url, info) for url, info in backends.items() if info['online']), key=lambda kv: estimate_model_size(kv[1]['model_config']), reverse=True ) offline_backends = sorted( ((url, info) for url, info in backends.items() if not info['online']), key=lambda kv: -kv[1]['priority'], reverse=True ) return [url for url, info in online_backends], [url for url, info in offline_backends] def get_a_cluster_backend(model=None): """ Get a backend from Redis. If there are no online backends, return None. If `model` is not supplied, we will pick one ourself. """ if model: # First, determine if there are multiple backends hosting the same model. backends_hosting_model = [i.decode('utf-8') for i in redis_running_models.smembers(model)] # If so, create an iterator for those backends if len(backends_hosting_model): add_backend_cycler(model, backends_hosting_model) cycled = redis_cycle(model) if len(cycled): return cycled[0] else: # No backend hosting that model return None else: online, _ = get_backends() if len(online): return online[0]