llm_server/routes/queue.py

@@ -50,6 +50,11 @@ class RedisPriorityQueue:
                 return item
             time.sleep(0.1)  # wait for something to be added to the queue
 
+    def print_all_items(self):
+        items = self.redis.zrange('queue', 0, -1)
+        for item in items:
+            print(item.decode('utf-8'))
+
     def increment_ip_count(self, client_ip: str, redis_key):
         self.redis.hincrby(redis_key, client_ip, 1)
 

llm_server/workers/printer.py

@@ -25,4 +25,6 @@ def console_printer():
                 processing_count += redis.get(k, default=0, dtype=int)
         backends = [k for k, v in cluster_config.all().items() if v['online']]
         logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}')
+        priority_queue.print_all_items()
+        print('============================')
         time.sleep(10)

llm_server/workers/threader.py

@@ -53,4 +53,4 @@ def start_background():
     t = Thread(target=db_logger)
     t.daemon = True
     t.start()
-    print('Started background logger')
+    print('Started background logger.')