local-llm-server/llm_server/routes/stats.py

import time
from datetime import datetime
from threading import Semaphore, Thread

from llm_server.opts import concurrent_gens
from llm_server.routes.cache import redis

# proompters_1_min = 0
concurrent_semaphore = Semaphore(concurrent_gens)
start_time = datetime.now()


def get_count():
    count = redis.get('proompts')
    if count is None:
        count = 0
    else:
        count = int(count)
    return count


class SemaphoreCheckerThread(Thread):
    proompters_1_min = 0
    recent_prompters = {}

    def __init__(self):
        Thread.__init__(self)
        self.daemon = True

    def run(self):
        while True:
            current_time = time.time()
            SemaphoreCheckerThread.recent_prompters = {ip: timestamp for ip, timestamp in SemaphoreCheckerThread.recent_prompters.items() if current_time - timestamp <= 60}
            SemaphoreCheckerThread.proompters_1_min = len(SemaphoreCheckerThread.recent_prompters)
            time.sleep(1)