cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fix stat

This commit is contained in:
Cyberes 2023-10-05 21:43:49 -06:00
parent e8964fcfd2
commit 3e5feb9c97
1 changed files with 1 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
llm_server/routes/queue.py
View File

@ -66,14 +66,6 @@ class RedisPriorityQueue:
def __len__(self): def __len__(self):
return self.redis.zcard('queue') return self.redis.zcard('queue')
def len(self, model_name):
count = 0
for key in self.redis.zrange('queue', 0, -1):
item = json.loads(key)
if item[2] == model_name:
count += 1
return count
def get_queued_ip_count(self, client_ip: str): def get_queued_ip_count(self, client_ip: str):
q = self.redis.hget('queued_ip_count', client_ip) q = self.redis.hget('queued_ip_count', client_ip)
if not q: if not q:
@ -152,8 +144,7 @@ class PriorityQueue:
if info.get('model') == model_name: if info.get('model') == model_name:
backends_with_models.append(k) backends_with_models.append(k)
for backend_url in backends_with_models: for backend_url in backends_with_models:
queue = RedisPriorityQueue(backend_url) count += len(RedisPriorityQueue(backend_url))
count += queue.len(model_name)
return count return count
def __len__(self): def __len__(self):