33 lines
892 B
Python
33 lines
892 B
Python
import collections
|
|
import time
|
|
from datetime import datetime
|
|
from threading import Semaphore, Thread
|
|
|
|
from llm_server import opts
|
|
from llm_server.integer import ThreadSafeInteger
|
|
from llm_server.opts import concurrent_gens
|
|
|
|
proompters_1_min = 0
|
|
concurrent_semaphore = Semaphore(concurrent_gens)
|
|
proompts = ThreadSafeInteger(0)
|
|
start_time = datetime.now()
|
|
|
|
|
|
class SemaphoreCheckerThread(Thread):
|
|
def __init__(self, semaphore):
|
|
Thread.__init__(self)
|
|
self.semaphore = semaphore
|
|
self.values = collections.deque(maxlen=60)
|
|
self.daemon = True
|
|
|
|
def run(self):
|
|
global proompters_1_min
|
|
while True:
|
|
self.values.append(opts.concurrent_gens - concurrent_semaphore._value)
|
|
proompters_1_min = sum(self.values) / len(self.values)
|
|
time.sleep(1)
|
|
|
|
|
|
thread = SemaphoreCheckerThread(concurrent_semaphore)
|
|
thread.start()
|