local-llm-server/llm_server/routes/v1/info.py

import time

from flask import jsonify, request

from . import bp
from ..auth import requires_auth
from llm_server.custom_redis import flask_cache
from ... import opts
from ...llm.info import get_running_model


# @bp.route('/info', methods=['GET'])
# # @cache.cached(timeout=3600, query_string=True)
# def get_info():
#     # requests.get()
#     return 'yes'


@bp.route('/model', methods=['GET'])
def get_model():
    # We will manage caching ourself since we don't want to cache
    # when the backend is down. Also, Cloudflare won't cache 500 errors.
    cache_key = 'model_cache::' + request.url
    cached_response = flask_cache.get(cache_key)

    if cached_response:
        return cached_response

    model_name, error = get_running_model()
    if not model_name:
        response = jsonify({
            'code': 502,
            'msg': 'failed to reach backend',
            'type': error.__class__.__name__
        }), 500  # return 500 so Cloudflare doesn't intercept us
    else:
        response = jsonify({
            'result': opts.manual_model_name if opts.manual_model_name else model_name,
            'timestamp': int(time.time())
        }), 200
        flask_cache.set(cache_key, response, timeout=60)

    return response


@bp.route('/backend', methods=['GET'])
@requires_auth
def get_backend():
    return jsonify({'backend': opts.backend_url, 'mode': opts.mode}), 200