94 lines
3.9 KiB
Python
94 lines
3.9 KiB
Python
import re
|
|
import sys
|
|
|
|
import openai
|
|
|
|
import llm_server
|
|
from llm_server import opts
|
|
from llm_server.config.config import ConfigLoader, config_default_vars, config_required_vars
|
|
from llm_server.custom_redis import redis
|
|
from llm_server.database.conn import database
|
|
from llm_server.database.database import get_number_of_rows
|
|
from llm_server.routes.queue import PriorityQueue
|
|
|
|
|
|
def load_config(config_path):
|
|
config_loader = ConfigLoader(config_path, config_default_vars, config_required_vars)
|
|
success, config, msg = config_loader.load_config()
|
|
if not success:
|
|
return success, config, msg
|
|
|
|
# TODO: this is atrocious
|
|
opts.auth_required = config['auth_required']
|
|
opts.log_prompts = config['log_prompts']
|
|
opts.frontend_api_client = config['frontend_api_client']
|
|
opts.show_num_prompts = config['show_num_prompts']
|
|
opts.show_uptime = config['show_uptime']
|
|
opts.cluster = config['cluster']
|
|
opts.show_total_output_tokens = config['show_total_output_tokens']
|
|
opts.netdata_root = config['netdata_root']
|
|
opts.simultaneous_requests_per_ip = config['simultaneous_requests_per_ip']
|
|
opts.show_backend_info = config['show_backend_info']
|
|
opts.max_new_tokens = config['max_new_tokens']
|
|
opts.manual_model_name = config['manual_model_name']
|
|
opts.llm_middleware_name = config['llm_middleware_name']
|
|
opts.enable_openi_compatible_backend = config['enable_openi_compatible_backend']
|
|
opts.openai_system_prompt = config['openai_system_prompt']
|
|
opts.expose_openai_system_prompt = config['expose_openai_system_prompt']
|
|
opts.enable_streaming = config['enable_streaming']
|
|
opts.openai_api_key = config['openai_api_key']
|
|
openai.api_key = opts.openai_api_key
|
|
opts.admin_token = config['admin_token']
|
|
opts.openai_expose_our_model = config['openai_epose_our_model']
|
|
opts.openai_force_no_hashes = config['openai_force_no_hashes']
|
|
opts.include_system_tokens_in_stats = config['include_system_tokens_in_stats']
|
|
opts.openai_moderation_scan_last_n = config['openai_moderation_scan_last_n']
|
|
opts.openai_org_name = config['openai_org_name']
|
|
opts.openai_silent_trim = config['openai_silent_trim']
|
|
opts.openai_moderation_enabled = config['openai_moderation_enabled']
|
|
opts.show_backends = config['show_backends']
|
|
opts.background_homepage_cacher = config['background_homepage_cacher']
|
|
opts.openai_moderation_timeout = config['openai_moderation_timeout']
|
|
opts.frontend_api_mode = config['frontend_api_mode']
|
|
opts.prioritize_by_size = config['prioritize_by_size']
|
|
|
|
# Scale the number of workers.
|
|
for item in config['cluster']:
|
|
opts.cluster_workers += item['concurrent_gens']
|
|
|
|
llm_server.routes.queue.priority_queue = PriorityQueue([x['backend_url'] for x in config['cluster']])
|
|
|
|
if opts.openai_expose_our_model and not opts.openai_api_key:
|
|
print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.')
|
|
sys.exit(1)
|
|
|
|
opts.verify_ssl = config['verify_ssl']
|
|
if not opts.verify_ssl:
|
|
import urllib3
|
|
|
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
if config['http_host']:
|
|
http_host = re.sub(r'http(?:s)?://', '', config["http_host"])
|
|
redis.set('http_host', http_host)
|
|
redis.set('base_client_api', f'{http_host}/{opts.frontend_api_client.strip("/")}')
|
|
|
|
database.init_db(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database'])
|
|
|
|
if config['load_num_prompts']:
|
|
redis.set('proompts', get_number_of_rows('prompts'))
|
|
|
|
return success, config, msg
|
|
|
|
|
|
def parse_backends(config):
|
|
if not config.get('cluster'):
|
|
return False
|
|
cluster = config.get('cluster')
|
|
config = {}
|
|
for item in cluster:
|
|
backend_url = item['backend_url'].strip('/')
|
|
item['backend_url'] = backend_url
|
|
config[backend_url] = item
|
|
return config
|