local-llm-server/test-cluster.py

try:
    import gevent.monkey

    gevent.monkey.patch_all()
except ImportError:
    pass

import time
from threading import Thread
from llm_server.cluster.redis_cycle import load_backend_cycle

from llm_server.cluster.backend import get_backends, get_a_cluster_backend
from llm_server.cluster.worker import cluster_worker
from llm_server.config.load import parse_backends, load_config
from llm_server.cluster.redis_config_cache import RedisClusterStore

import argparse

parser = argparse.ArgumentParser()
parser.add_argument('config')
args = parser.parse_args()

success, config, msg = load_config(args.config)

cluster_config = RedisClusterStore('cluster_config')
cluster_config.clear()
cluster_config.load(parse_backends(config))
on, off = get_backends()
load_backend_cycle('backend_cycler', on + off)

t = Thread(target=cluster_worker)
t.daemon = True
t.start()

while True:
    # online, offline = get_backends()
    # print(online, offline)
    # print(get_a_cluster_backend())
    time.sleep(3)