59 lines
1.6 KiB
Python
59 lines
1.6 KiB
Python
|
import time
|
||
|
|
from threading import Thread
|
||
|
|
|
||
|
|
from llm_server import opts
|
||
|
|
from llm_server.cluster.worker import cluster_worker
|
||
|
|
from llm_server.logging import create_logger
|
||
|
|
from llm_server.routes.v1.generate_stats import generate_stats
|
||
|
|
from llm_server.workers.inferencer import start_workers
|
||
|
|
from llm_server.workers.logger import db_logger
|
||
|
|
from llm_server.workers.mainer import main_background_thread
|
||
|
|
from llm_server.workers.moderator import start_moderation_workers
|
||
|
|
from llm_server.workers.printer import console_printer
|
||
|
|
from llm_server.workers.recenter import recent_prompters_thread
|
||
|
|
|
||
|
|
|
||
|
|
def cache_stats():
|
||
|
|
while True:
|
||
|
|
generate_stats(regen=True)
|
||
|
|
time.sleep(5)
|
||
|
|
|
||
|
|
|
||
|
|
def start_background():
|
||
|
|
logger = create_logger('threader')
|
||
|
|
start_workers(opts.cluster)
|
||
|
|
|
||
|
|
t = Thread(target=main_background_thread)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
logger.info('Started the main background thread.')
|
||
|
|
|
||
|
|
num_moderators = opts.cluster_workers * 3
|
||
|
|
start_moderation_workers(num_moderators)
|
||
|
|
logger.info(f'Started {num_moderators} moderation workers.')
|
||
|
|
|
||
|
|
t = Thread(target=cache_stats)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
logger.info('Started the stats cacher.')
|
||
|
|
|
||
|
|
t = Thread(target=recent_prompters_thread)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
logger.info('Started the recent proompters thread.')
|
||
|
|
|
||
|
|
t = Thread(target=console_printer)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
logger.info('Started the console logger.infoer.')
|
||
|
|
|
||
|
|
t = Thread(target=cluster_worker)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
logger.info('Started the cluster worker.')
|
||
|
|
|
||
|
|
t = Thread(target=db_logger)
|
||
|
|
t.daemon = True
|
||
|
|
t.start()
|
||
|
|
logger.info('Started background logger.')
|