reorganize stats page again

llm_server/routes/v1/generate_stats.py

@@ -72,11 +72,7 @@ def generate_stats():
                 '1_min': SemaphoreCheckerThread.proompters_1_min,
                 '24_hrs': get_distinct_ips_24h(),
             },
-            'proompts': {
-                'processing': active_gen_workers,
-                'queued': proompters_in_queue,
-                'total': get_total_proompts() if opts.show_num_prompts else None,
-            },
+            'proompts_total': get_total_proompts() if opts.show_num_prompts else None,
             'uptime': int((datetime.now() - server_start_time).total_seconds()) if opts.show_uptime else None,
             'average_generation_elapsed_sec': int(gen_time_calc),
             'average_tps': average_tps,
@@ -87,7 +83,11 @@ def generate_stats():
         'endpoints': {
             'blocking': opts.full_client_api,
         },
-        'estimated_wait_sec': int(estimated_wait_sec),
+        'queue': {
+            'processing': active_gen_workers,
+            'queued': proompters_in_queue,
+            'estimated_wait_sec': int(estimated_wait_sec),
+        },
         'timestamp': int(time.time()),
         'config': {
             'gatekeeper': 'none' if opts.auth_required is False else 'token',

requirements.txt

@@ -6,4 +6,5 @@ requests
 tiktoken
 gunicorn
 redis
-gevent
+gevent
+async-timeout

server.py

@@ -106,12 +106,12 @@ def home():
     else:
         running_model = opts.running_model
 
-        if stats['estimated_wait_sec'] == 0 and stats['stats']['proompts']['processing'] > 0:
+        if stats['queue']['queued'] == 0 and stats['queue']['processing'] > 0:
             # There will be a wait if the queue is empty but prompts are processing, but we don't
             # know how long.
             estimated_wait_sec = f"less than {stats['stats']['average_generation_elapsed_sec']} seconds"
         else:
-            estimated_wait_sec = f"{stats['estimated_wait_sec']} seconds"
+            estimated_wait_sec = f"{stats['queue']['estimated_wait_sec']} seconds"
 
     if len(config['analytics_tracking_code']):
         analytics_tracking_code = f"<script>\n{config['analytics_tracking_code']}\n</script>"