local-llm-server/llm_server/workers/printer.py

import logging
import time

from llm_server.cluster.cluster_config import cluster_config
from llm_server.custom_redis import redis
from llm_server.routes.queue import priority_queue

logger = logging.getLogger('console_printer')
if not logger.handlers:
    handler = logging.StreamHandler()
    handler.setLevel(logging.INFO)
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter("%(asctime)s: %(levelname)s:%(name)s - %(message)s")
    handler.setFormatter(formatter)
    logger.addHandler(handler)


def console_printer():
    time.sleep(3)
    while True:
        processing = redis.keys('active_gen_workers:http*')  # backends always start with http
        processing_count = 0
        if len(processing):
            for k in processing:
                processing_count += redis.get(k, default=0, dtype=int)
        backends = [k for k, v in cluster_config.all().items() if v['online']]
        logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}')
        time.sleep(1)