cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

adjust netdata json, don't log error messages during generationg

This commit is contained in:
Cyberes 2023-08-24 22:53:06 -06:00
parent 06173f900e
commit e5aca7b09d
2 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llm_server/netdata.py
View File

@ -26,4 +26,4 @@ def get_power_states():
print('Failed to fetch Netdata metrics:', e) print('Failed to fetch Netdata metrics:', e)
return output return output
gpu_num += 1 gpu_num += 1
return output return {'power_states': output}

6
llm_server/routes/v1/generate.py
View File

@ -82,19 +82,21 @@ def generate():
**response_json_body **response_json_body
}), 200 }), 200
response_valid_json, response_json_body = validate_json(response) response_valid_json, response_json_body = validate_json(response)
backend_err = False
if response_valid_json: if response_valid_json:
redis.incr('proompts') redis.incr('proompts')
backend_response = safe_list_get(response_json_body.get('results', []), 0, {}).get('text') backend_response = safe_list_get(response_json_body.get('results', []), 0, {}).get('text')
if not backend_response: if not backend_response:
if opts.mode == 'oobabooga': if opts.mode == 'oobabooga':
backend_err = True
backend_response = format_sillytavern_err( backend_response = format_sillytavern_err(
f'Backend (oobabooga) returned an empty string. This can happen when your parameters are incorrect. Make sure your context size is no greater than {opts.context_size}. Furthermore, oobabooga does not support concurrent requests so all users have to wait in line and the backend server may have glitched for a moment. Please try again.', f'Backend (oobabooga) returned an empty string. This is usually due to an error on the backend during inference. Make sure your context size is no greater than {opts.context_size}. Please try again.',
'error') 'error')
response_json_body['results'][0]['text'] = backend_response response_json_body['results'][0]['text'] = backend_response
else: else:
raise Exception raise Exception
log_prompt(client_ip, token, request_json_body['prompt'], backend_response, elapsed_time, parameters, dict(request.headers), response.status_code) log_prompt(client_ip, token, request_json_body['prompt'], backend_response if not backend_err else None, elapsed_time, parameters, dict(request.headers), response.status_code)
return jsonify({ return jsonify({
**response_json_body **response_json_body
}), 200 }), 200