llm_server/cluster/cluster_config.py

@@ -1,5 +1,6 @@
 import hashlib
 import pickle
+import traceback
 
 from llm_server import opts
 from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle
@@ -75,6 +76,7 @@ def get_backends():
         priority = b['priority']
         result[k] = {'status': status, 'priority': priority}
 
+    try:
         if not opts.prioritize_by_size:
             online_backends = sorted(
                 ((url, info) for url, info in backends.items() if info['online']),
@@ -93,6 +95,9 @@ def get_backends():
             reverse=True
         )
         return [url for url, info in online_backends], [url for url, info in offline_backends]
+    except KeyError:
+        traceback.print_exc()
+        print(backends)
 
 
 def get_a_cluster_backend(model=None):