local-llm-server/llm_server/routes/v1/generate_stats.py

import time
from datetime import datetime

from llm_server import opts
from llm_server.database import get_distinct_ips_24h, sum_column
from llm_server.helpers import deep_sort
from llm_server.llm.info import get_running_model
from llm_server.netdata import get_power_states
from llm_server.routes.cache import redis
from llm_server.routes.queue import priority_queue
from llm_server.routes.stats import SemaphoreCheckerThread, calculate_avg_gen_time, get_active_gen_workers, get_total_proompts, server_start_time


# TODO: have routes/__init__.py point to the latest API version generate_stats()

def generate_stats():
    model_name, error = get_running_model()  # will return False when the fetch fails
    if isinstance(model_name, bool):
        online = False
    else:
        online = True
        opts.running_model = model_name

    # t = elapsed_times.copy()  # copy since we do multiple operations and don't want it to change
    # if len(t) == 0:
    #     estimated_wait = 0
    # else:
    #     waits = [elapsed for end, elapsed in t]
    #     estimated_wait = int(sum(waits) / len(waits))

    active_gen_workers = get_active_gen_workers()
    proompters_in_queue = len(priority_queue)
    average_tps = float(redis.get('average_tps'))

    if opts.average_generation_time_mode == 'database':
        average_generation_time = float(redis.get('average_generation_elapsed_sec'))
        average_output_tokens = float(redis.get('average_output_tokens'))
        # average_generation_time_from_tps = (average_output_tokens / average_tps)

        # What to use in our math that calculates the wait time.
        # We could use the average TPS but we don't know the exact TPS value, only
        # the backend knows that. So, let's just stick with the elapsed time.
        gen_time_calc = average_generation_time

        estimated_wait_sec = (
                                     (gen_time_calc * proompters_in_queue) / opts.concurrent_gens  # Calculate wait time for items in queue
                             ) + (
                                     active_gen_workers * gen_time_calc  # Calculate wait time for in-process items
                             ) if average_tps > 0 else 0
    elif opts.average_generation_time_mode == 'minute':
        average_generation_time = calculate_avg_gen_time()
        gen_time_calc = average_generation_time
        estimated_wait_sec = ((gen_time_calc * proompters_in_queue) / opts.concurrent_gens) + (active_gen_workers * gen_time_calc)
    else:
        raise Exception

    if opts.netdata_root:
        netdata_stats = {}
        power_states = get_power_states()
        for gpu, power_state in power_states.items():
            netdata_stats[gpu] = {
                'power_state': power_state,
                # 'wh_wasted_1_hr': get_gpu_wh(int(gpu.strip('gpu')))
            }

    else:
        netdata_stats = {}

    output = {
        'stats': {
            'proompters': {
                '1_min': SemaphoreCheckerThread.proompters_1_min,
                '24_hrs': get_distinct_ips_24h(),
            },
            'proompts_total': get_total_proompts() if opts.show_num_prompts else None,
            'uptime': int((datetime.now() - server_start_time).total_seconds()) if opts.show_uptime else None,
            'average_generation_elapsed_sec': int(gen_time_calc),
            'average_tps': average_tps,
            'tokens_generated': sum_column('prompts', 'response_tokens') if opts.show_total_output_tokens else None,
            'nvidia': netdata_stats
        },
        'online': online,
        'endpoints': {
            'blocking': opts.full_client_api,
        },
        'queue': {
            'processing': active_gen_workers,
            'queued': proompters_in_queue,
            'estimated_wait_sec': int(estimated_wait_sec),
        },
        'timestamp': int(time.time()),
        'config': {
            'gatekeeper': 'none' if opts.auth_required is False else 'token',
            'context_size': opts.context_size,
            'queue_size': opts.concurrent_gens,
            'model': model_name,
            'mode': opts.mode,
            'simultaneous_requests': opts.ip_in_queue_max,
        },
        'keys': {
            'openaiKeys': '∞',
            'anthropicKeys': '∞',
        },
    }
    return deep_sort(output)