99 lines
3.0 KiB
Python
99 lines
3.0 KiB
Python
import os
|
|
import sys
|
|
from pathlib import Path
|
|
from threading import Thread
|
|
|
|
import config
|
|
from flask import Flask, jsonify
|
|
|
|
from llm_server import opts
|
|
from llm_server.config import ConfigLoader, config_default_vars, config_required_vars
|
|
from llm_server.database import get_number_of_rows, init_db
|
|
from llm_server.helpers import resolve_path
|
|
from llm_server.routes.cache import cache, redis
|
|
from llm_server.routes.helpers.http import cache_control
|
|
from llm_server.routes.queue import start_workers
|
|
from llm_server.routes.stats import SemaphoreCheckerThread, process_avg_gen_time
|
|
from llm_server.routes.v1 import bp
|
|
|
|
script_path = os.path.dirname(os.path.realpath(__file__))
|
|
|
|
config_path_environ = os.getenv("CONFIG_PATH")
|
|
if config_path_environ:
|
|
config_path = config_path_environ
|
|
else:
|
|
config_path = Path(script_path, 'config', 'config.yml')
|
|
|
|
config_loader = ConfigLoader(config_path, config_default_vars, config_required_vars)
|
|
success, config, msg = config_loader.load_config()
|
|
if not success:
|
|
print('Failed to load config:', msg)
|
|
sys.exit(1)
|
|
|
|
# Resolve relative directory to the directory of the script
|
|
if config['database_path'].startswith('./'):
|
|
config['database_path'] = resolve_path(script_path, config['database_path'].strip('./'))
|
|
|
|
opts.database_path = resolve_path(config['database_path'])
|
|
init_db()
|
|
|
|
if config['mode'] not in ['oobabooga', 'hf-textgen']:
|
|
print('Unknown mode:', config['mode'])
|
|
opts.mode = config['mode']
|
|
opts.auth_required = config['auth_required']
|
|
opts.log_prompts = config['log_prompts']
|
|
opts.concurrent_gens = config['concurrent_gens']
|
|
opts.frontend_api_client = config['frontend_api_client']
|
|
opts.context_size = config['token_limit']
|
|
opts.show_num_prompts = config['show_num_prompts']
|
|
opts.show_uptime = config['show_uptime']
|
|
opts.backend_url = config['backend_url'].strip('/')
|
|
|
|
opts.verify_ssl = config['verify_ssl']
|
|
if not opts.verify_ssl:
|
|
import urllib3
|
|
|
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
flushed_keys = redis.flush()
|
|
print('Flushed', len(flushed_keys), 'keys from Redis.')
|
|
|
|
if config['load_num_prompts']:
|
|
redis.set('proompts', get_number_of_rows('prompts'))
|
|
|
|
start_workers(opts.concurrent_gens)
|
|
|
|
# cleanup_thread = Thread(target=elapsed_times_cleanup)
|
|
# cleanup_thread.daemon = True
|
|
# cleanup_thread.start()
|
|
# Start the background thread
|
|
process_avg_gen_time_background_thread = Thread(target=process_avg_gen_time)
|
|
process_avg_gen_time_background_thread.daemon = True
|
|
process_avg_gen_time_background_thread.start()
|
|
SemaphoreCheckerThread().start()
|
|
|
|
app = Flask(__name__)
|
|
cache.init_app(app)
|
|
cache.clear() # clear redis cache
|
|
# with app.app_context():
|
|
# current_app.tokenizer = tiktoken.get_encoding("cl100k_base")
|
|
app.register_blueprint(bp, url_prefix='/api/v1/')
|
|
|
|
|
|
# print(app.url_map)
|
|
|
|
|
|
@app.route('/')
|
|
@app.route('/<first>')
|
|
@app.route('/<first>/<path:rest>')
|
|
@cache_control(-1)
|
|
def fallback(first=None, rest=None):
|
|
return jsonify({
|
|
'error': 404,
|
|
'msg': 'not found'
|
|
}), 404
|
|
|
|
|
|
if __name__ == "__main__":
|
|
app.run(host='0.0.0.0')
|