local-llm-server/llm_server/workers/threader.py

import time
from threading import Thread

from llm_server.cluster.worker import cluster_worker
from llm_server.config.config import cluster_worker_count
from llm_server.config.global_config import GlobalConfig
from llm_server.logging import create_logger
from llm_server.routes.v1.generate_stats import generate_stats
from llm_server.workers.inferencer import start_workers
from llm_server.workers.logger import db_logger
from llm_server.workers.mainer import main_background_thread
from llm_server.workers.moderator import start_moderation_workers
from llm_server.workers.printer import console_printer
from llm_server.workers.recenter import recent_prompters_thread


def cache_stats():
    while True:
        generate_stats(regen=True)
        time.sleep(5)


def start_background():
    logger = create_logger('threader')
    start_workers(GlobalConfig.get().cluster)

    t = Thread(target=main_background_thread)
    t.daemon = True
    t.start()
    logger.info('Started the main background thread.')

    num_moderators = cluster_worker_count() * 3
    start_moderation_workers(num_moderators)
    logger.info(f'Started {num_moderators} moderation workers.')

    t = Thread(target=cache_stats)
    t.daemon = True
    t.start()
    logger.info('Started the stats cacher.')

    t = Thread(target=recent_prompters_thread)
    t.daemon = True
    t.start()
    logger.info('Started the recent proompters thread.')

    t = Thread(target=console_printer)
    t.daemon = True
    t.start()
    logger.info('Started the console logger.infoer.')

    t = Thread(target=cluster_worker)
    t.daemon = True
    t.start()
    logger.info('Started the cluster worker.')

    t = Thread(target=db_logger)
    t.daemon = True
    t.start()
    logger.info('Started background logger.')