local-llm-server/llm_server/llm/oobabooga/generate.py

"""
This file is used by the worker that processes requests.
"""
import traceback

import requests

from llm_server import opts


def generate(json_data: dict):
    try:
        r = requests.post(f'{opts.backend_url}/api/v1/generate', json=json_data, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout)
    except requests.exceptions.ReadTimeout:
        return False, None, 'Request to backend timed out'
    except Exception as e:
        traceback.print_exc()
        return False, None, 'Request to backend encountered error'
    if r.status_code != 200:
        return False, r, f'Backend returned {r.status_code}'
    return True, r, None