30 lines
714 B
Python
30 lines
714 B
Python
|
try:
|
||
|
|
import gevent.monkey
|
||
|
|
|
||
|
|
gevent.monkey.patch_all()
|
||
|
|
except ImportError:
|
||
|
|
pass
|
||
|
|
|
||
|
|
import time
|
||
|
|
from threading import Thread
|
||
|
|
|
||
|
|
from llm_server.cluster.funcs.backend import get_best_backends
|
||
|
|
from llm_server.cluster.redis_config_cache import RedisClusterStore
|
||
|
|
from llm_server.cluster.worker import cluster_worker
|
||
|
|
from llm_server.config.load import parse_backends, load_config
|
||
|
|
|
||
|
|
success, config, msg = load_config('./config/config.yml').resolve().absolute()
|
||
|
|
|
||
|
|
cluster_config = RedisClusterStore('cluster_config')
|
||
|
|
cluster_config.clear()
|
||
|
|
cluster_config.load(parse_backends(config))
|
||
|
|
|
||
|
|
t = Thread(target=cluster_worker)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
|
||
|
|
while True:
|
||
|
|
x = get_best_backends()
|
||
|
|
print(x)
|
||
|
|
time.sleep(3)
|