cyberes
/
local-llm-server
Archived
1
1
Fork 0
Code Issues Pull Requests Packages Projects Releases 1 Wiki Activity

fix streaming disabled

This commit is contained in:
Cyberes 2023-10-15 20:59:11 -06:00
parent 381bdb950f
commit 24aab3cd93
4 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llm_server/routes/openai/chat_completions.py
View File

@ -40,7 +40,7 @@ def openai_chat_completions(model_name=None):
return 'Internal server error', 500 return 'Internal server error', 500
else: else:
if not opts.enable_streaming: if not opts.enable_streaming:
return return 'Streaming disabled', 403
handler.parameters, _ = handler.get_parameters() handler.parameters, _ = handler.get_parameters()
handler.request_json_body = { handler.request_json_body = {

2
llm_server/routes/openai/completions.py
View File

@ -87,7 +87,7 @@ def openai_completions(model_name=None):
return response, 200 return response, 200
else: else:
if not opts.enable_streaming: if not opts.enable_streaming:
return 'DISABLED', 401 return 'Streaming disabled', 403
event_id = None event_id = None
start_time = time.time() start_time = time.time()

2
llm_server/routes/v1/generate_stream.py
View File

@ -61,7 +61,7 @@ def do_stream(ws, model_name):
) )
if not opts.enable_streaming: if not opts.enable_streaming:
return 'Streaming is disabled', 500 return 'Streaming disabled', 403
r_headers = dict(request.headers) r_headers = dict(request.headers)
r_url = request.url r_url = request.url

1
llm_server/workers/mainer.py
View File

@ -36,6 +36,7 @@ def main_background_thread():
except Exception as e: except Exception as e:
print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}') print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}')
# TODO: test
backends = priority_queue.get_backends() backends = priority_queue.get_backends()
for backend_url in backends: for backend_url in backends:
queue = RedisPriorityQueue(backend_url) queue = RedisPriorityQueue(backend_url)