import time import requests from llm_server import opts from llm_server.cluster.cluster_config import cluster_config, get_backends from llm_server.custom_redis import redis from llm_server.database.database import weighted_average_column_for_model from llm_server.llm.info import get_info from llm_server.routes.queue import RedisPriorityQueue, priority_queue def main_background_thread(): while True: online, offline = get_backends() for backend_url in online: backend_info = cluster_config.get_backend(backend_url) backend_mode = backend_info['mode'] backend_info = get_info(backend_url, backend_mode) running_model = backend_info.get('model') if not running_model: continue average_generation_elapsed_sec, average_output_tokens, estimated_avg_tps = calc_stats_for_backend(backend_url, running_model, backend_mode) if average_generation_elapsed_sec: # returns None on exception cluster_config.set_backend_value(backend_url, 'average_generation_elapsed_sec', average_generation_elapsed_sec) if average_output_tokens: cluster_config.set_backend_value(backend_url, 'average_output_tokens', average_output_tokens) if average_generation_elapsed_sec and average_output_tokens: cluster_config.set_backend_value(backend_url, 'estimated_avg_tps', estimated_avg_tps) if opts.background_homepage_cacher: try: base_client_api = redis.get('base_client_api', dtype=str) r = requests.get('https://' + base_client_api, timeout=5) except Exception as e: print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}') backends = priority_queue.get_backends() for backend_url in backends: queue = RedisPriorityQueue(backend_url) queue.cleanup() time.sleep(30) def calc_stats_for_backend(backend_url, running_model, backend_mode): # exclude_zeros=True filters out rows where an error message was returned. Previously, if there was an error, 0 # was entered into the column. The new code enters null instead but we need to be backwards compatible for now. average_generation_elapsed_sec = weighted_average_column_for_model('prompts', 'generation_time', running_model, backend_mode, backend_url, exclude_zeros=True, include_system_tokens=opts.include_system_tokens_in_stats) or 0 average_output_tokens = weighted_average_column_for_model('prompts', 'response_tokens', running_model, backend_mode, backend_url, exclude_zeros=True, include_system_tokens=opts.include_system_tokens_in_stats) or 0 estimated_avg_tps = round(average_output_tokens / average_generation_elapsed_sec, 2) if average_generation_elapsed_sec > 0 else 0 # Avoid division by zero return average_generation_elapsed_sec, average_output_tokens, estimated_avg_tps