cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

track lag on get_ip_request_count()

This commit is contained in:
Cyberes 2023-10-18 09:09:22 -06:00
parent 92e4ecd8a1
commit 50377eca22
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
llm_server/routes/queue.py
View File

@ -58,12 +58,25 @@ class RedisPriorityQueue:
return self.redis.zcard('queue') return self.redis.zcard('queue')
def get_ip_request_count(self, client_ip: str): def get_ip_request_count(self, client_ip: str):
"""
Get the number of requests in the queue from a specific IP.
This is a bit inefficient since we iterate over the entire queue, but
keeps the queue as a single point of truth instead of tracking a separate hashed
set which can get confusing.
If we run into slowdowns in the future, we should go back to the hashed set approach.
:param client_ip:
:return:
"""
start_time = time.time()
items = self.redis.zrange('queue', 0, -1) items = self.redis.zrange('queue', 0, -1)
count = 0 count = 0
for item in items: for item in items:
item_data = json.loads(item) item_data = json.loads(item)
if item_data[0][1] == client_ip: if item_data[0][1] == client_ip:
count += 1 count += 1
elapsed_time = time.time() - start_time
if elapsed_time > 0.5:
raise Exception(f"!!! get_ip_request_count took {elapsed_time} seconds to execute !!!")
return count return count
def flush(self): def flush(self):