adjust a few final things
This commit is contained in:
parent
fab7b7ccdd
commit
0015e653b2
|
@ -120,7 +120,7 @@ def weighted_average_column_for_model(table_name, column_name, model_name, backe
|
|||
results = cursor.fetchall()
|
||||
except Exception:
|
||||
traceback.print_exc()
|
||||
return -1
|
||||
return None
|
||||
|
||||
total_weight = 0
|
||||
weighted_sum = 0
|
||||
|
|
|
@ -14,7 +14,7 @@ from ...stream import sock
|
|||
|
||||
# TODO: have workers process streaming requests
|
||||
|
||||
@sock.route('/api/v1/stream') # TODO: use blueprint route???
|
||||
@sock.route('/api/v1/stream')
|
||||
def stream(ws):
|
||||
if not opts.enable_streaming:
|
||||
# TODO: return a formatted ST error message
|
||||
|
@ -70,13 +70,12 @@ def stream(ws):
|
|||
if partial_response.endswith(b'\x00'):
|
||||
json_str = partial_response[:-1].decode() # Remove the null character and decode the byte string to a string
|
||||
json_obj = json.loads(json_str)
|
||||
if len(json_obj['text'][0].split(input_prompt + generated_text)) > 2:
|
||||
try:
|
||||
new = json_obj['text'][0].split(input_prompt + generated_text)[1]
|
||||
except IndexError:
|
||||
# ????
|
||||
print(json_obj)
|
||||
continue
|
||||
|
||||
new = json_obj['text'][0].split(input_prompt + generated_text)[1]
|
||||
|
||||
ws.send(json.dumps({
|
||||
'event': 'text_stream',
|
||||
'message_num': message_num,
|
||||
|
|
|
@ -47,14 +47,14 @@ class MainBackgroundThread(Thread):
|
|||
# exclude_zeros=True filters out rows where an error message was returned. Previously, if there was an error, 0
|
||||
# was entered into the column. The new code enters null instead but we need to be backwards compatible for now.
|
||||
average_generation_elapsed_sec = weighted_average_column_for_model('prompts', 'generation_time', opts.running_model, opts.mode, opts.backend_url, exclude_zeros=True) or 0
|
||||
if average_generation_elapsed_sec > -1:
|
||||
if average_generation_elapsed_sec: # returns None on exception
|
||||
redis.set('average_generation_elapsed_sec', average_generation_elapsed_sec)
|
||||
|
||||
# overall = average_column_for_model('prompts', 'generation_time', opts.running_model)
|
||||
# print(f'Weighted: {average_generation_elapsed_sec}, overall: {overall}')
|
||||
|
||||
average_output_tokens = weighted_average_column_for_model('prompts', 'response_tokens', opts.running_model, opts.mode, opts.backend_url, exclude_zeros=True) or 0
|
||||
if average_generation_elapsed_sec > -1:
|
||||
if average_generation_elapsed_sec:
|
||||
redis.set('average_output_tokens', average_output_tokens)
|
||||
|
||||
# overall = average_column_for_model('prompts', 'response_tokens', opts.running_model)
|
||||
|
|
Reference in New Issue