import time from threading import Thread from llm_server import opts from llm_server.cluster.stores import redis_running_models from llm_server.cluster.worker import cluster_worker from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.inferencer import start_workers from llm_server.workers.logger import db_logger from llm_server.workers.mainer import main_background_thread from llm_server.workers.moderator import start_moderation_workers from llm_server.workers.printer import console_printer from llm_server.workers.recenter import recent_prompters_thread def cache_stats(): while True: generate_stats(regen=True) time.sleep(5) def start_background(): start_workers(opts.cluster_workers) t = Thread(target=main_background_thread) t.daemon = True t.start() print('Started the main background thread.') start_moderation_workers(opts.cluster_workers * 3) t = Thread(target=cache_stats) t.daemon = True t.start() print('Started the stats cacher.') t = Thread(target=recent_prompters_thread) t.daemon = True t.start() print('Started the recent proompters thread.') t = Thread(target=console_printer) t.daemon = True t.start() print('Started the console printer.') redis_running_models.flush() t = Thread(target=cluster_worker) t.daemon = True t.start() print('Started the cluster worker.') t = Thread(target=db_logger) t.daemon = True t.start() print('Started background logger.')