From 747d838138a090d7aec18e7e60ce7e257dd8eba8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 11 Sep 2023 21:05:22 -0600 Subject: [PATCH] move where the vllm model is set --- llm_server/llm/vllm/generate.py | 4 ---- llm_server/routes/request_handler.py | 6 ++++++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/llm_server/llm/vllm/generate.py b/llm_server/llm/vllm/generate.py index 618ca31..1a0bf92 100644 --- a/llm_server/llm/vllm/generate.py +++ b/llm_server/llm/vllm/generate.py @@ -106,10 +106,6 @@ def handle_blocking_request(json_data: dict): def generate(json_data: dict): - full_model_path = redis.get('full_model_path') - if not full_model_path: - raise Exception - json_data['model'] = full_model_path.decode() if json_data.get('stream'): raise Exception('streaming not implemented') else: diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index ef64968..8ea7356 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -99,6 +99,12 @@ class OobaRequestHandler: if opts.mode == 'hf-textgen' and self.parameters.get('typical_p', 0) > 0.998: self.request_json_body['typical_p'] = 0.998 + if opts.mode == 'vllm': + full_model_path = redis.get('full_model_path') + if not full_model_path: + raise Exception + self.request_json_body['model'] = full_model_path.decode() + request_valid, invalid_request_err_msg = self.validate_request() params_valid, invalid_params_err_msg = self.backend.validate_params(self.parameters) if not request_valid or not params_valid: