llm_server/routes/queue.py

@@ -58,12 +58,25 @@ class RedisPriorityQueue:
         return self.redis.zcard('queue')
 
     def get_ip_request_count(self, client_ip: str):
+        """
+        Get the number of requests in the queue from a specific IP.
+        This is a bit inefficient since we iterate over the entire queue, but
+        keeps the queue as a single point of truth instead of tracking a separate hashed
+        set which can get confusing.
+        If we run into slowdowns in the future, we should go back to the hashed set approach.
+        :param client_ip:
+        :return:
+        """
+        start_time = time.time()
         items = self.redis.zrange('queue', 0, -1)
         count = 0
         for item in items:
             item_data = json.loads(item)
             if item_data[0][1] == client_ip:
                 count += 1
+        elapsed_time = time.time() - start_time
+        if elapsed_time > 0.5:
+            raise Exception(f"!!! get_ip_request_count took {elapsed_time} seconds to execute !!!")
         return count
 
     def flush(self):