This repository has been archived on 2024-10-27. You can view files and clone it, but cannot push or open issues or pull requests.
2023-08-23 20:12:38 -06:00
|
|
|
from llm_server import opts
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def generator(request_json_body):
|
|
|
|
|
if opts.mode == 'oobabooga':
|
2023-09-27 21:15:54 -06:00
|
|
|
# from .oobabooga.generate import generate
|
|
|
|
|
# return generate(request_json_body)
|
|
|
|
|
raise NotImplementedError
|
2023-09-11 20:47:19 -06:00
|
|
|
elif opts.mode == 'vllm':
|
|
|
|
|
from .vllm.generate import generate
|
|
|
|
|
r = generate(request_json_body)
|
|
|
|
|
return r
|
2023-08-23 20:12:38 -06:00
|
|
|
else:
|
|
|
|
|
raise Exception
|
