From 50377eca224a92d1f920f45feb69617d5a0b2479 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 18 Oct 2023 09:09:22 -0600 Subject: [PATCH] track lag on get_ip_request_count() --- llm_server/routes/queue.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 66659d4..62c9c42 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -58,12 +58,25 @@ class RedisPriorityQueue: return self.redis.zcard('queue') def get_ip_request_count(self, client_ip: str): + """ + Get the number of requests in the queue from a specific IP. + This is a bit inefficient since we iterate over the entire queue, but + keeps the queue as a single point of truth instead of tracking a separate hashed + set which can get confusing. + If we run into slowdowns in the future, we should go back to the hashed set approach. + :param client_ip: + :return: + """ + start_time = time.time() items = self.redis.zrange('queue', 0, -1) count = 0 for item in items: item_data = json.loads(item) if item_data[0][1] == client_ip: count += 1 + elapsed_time = time.time() - start_time + if elapsed_time > 0.5: + raise Exception(f"!!! get_ip_request_count took {elapsed_time} seconds to execute !!!") return count def flush(self):