60 lines
3.5 KiB
Python
60 lines
3.5 KiB
Python
from typing import Union
|
|
|
|
import bison
|
|
|
|
from llm_server.opts import default_openai_system_prompt
|
|
|
|
config_scheme = bison.Scheme(
|
|
bison.Option('frontend_api_mode', choices=['ooba'], field_type=str),
|
|
bison.ListOption('cluster', member_scheme=bison.Scheme(
|
|
bison.Option('backend_url', field_type=str),
|
|
bison.Option('concurrent_gens', field_type=int),
|
|
bison.Option('mode', choices=['vllm'], field_type=str),
|
|
bison.Option('priority', field_type=int),
|
|
)),
|
|
bison.Option('prioritize_by_size', default=True, field_type=bool),
|
|
bison.Option('admin_token', default=None, field_type=Union[str, None]),
|
|
bison.ListOption('mysql', member_scheme=bison.Scheme(
|
|
bison.Option('host', field_type=str),
|
|
bison.Option('username', field_type=str),
|
|
bison.Option('password', field_type=str),
|
|
bison.Option('database', field_type=str),
|
|
bison.Option('maxconn', field_type=int)
|
|
)),
|
|
bison.Option('http_host', default='', field_type=str),
|
|
bison.Option('webserver_log_directory', default='/var/log/localllm', field_type=str),
|
|
bison.Option('include_system_tokens_in_stats', default=True, field_type=bool),
|
|
bison.Option('background_homepage_cacher', default=True, field_type=bool),
|
|
bison.Option('max_new_tokens', default=500, field_type=int),
|
|
bison.Option('enable_streaming', default=True, field_type=bool),
|
|
bison.Option('show_backends', default=True, field_type=bool),
|
|
bison.Option('log_prompts', default=True, field_type=bool),
|
|
bison.Option('verify_ssl', default=False, field_type=bool),
|
|
bison.Option('auth_required', default=False, field_type=bool),
|
|
bison.Option('simultaneous_requests_per_ip', default=1, field_type=int),
|
|
bison.Option('max_queued_prompts_per_ip', default=1, field_type=int),
|
|
bison.Option('llm_middleware_name', default='LocalLLM', field_type=str),
|
|
bison.Option('analytics_tracking_code', default=None, field_type=Union[str, None]),
|
|
bison.Option('info_html', default=None, field_type=Union[str, None]),
|
|
bison.Option('enable_openi_compatible_backend', default=True, field_type=bool),
|
|
bison.Option('openai_api_key', default=None, field_type=Union[str, None]),
|
|
bison.Option('expose_openai_system_prompt', default=True, field_type=bool),
|
|
bison.Option('openai_expose_our_model', default='', field_type=bool),
|
|
bison.Option('openai_force_no_hashes', default=True, field_type=bool),
|
|
bison.Option('openai_system_prompt', default=default_openai_system_prompt, field_type=str),
|
|
bison.Option('openai_moderation_enabled', default=False, field_type=bool),
|
|
bison.Option('openai_moderation_timeout', default=5, field_type=int),
|
|
bison.Option('openai_moderation_scan_last_n', default=5, field_type=int),
|
|
bison.Option('openai_org_name', default='OpenAI', field_type=str),
|
|
bison.Option('openai_silent_trim', default=True, field_type=bool),
|
|
bison.Option('frontend_api_client', default='/api', field_type=str),
|
|
bison.Option('average_generation_time_mode', default='database', choices=['database', 'minute'], field_type=str),
|
|
bison.Option('show_num_prompts', default=True, field_type=bool),
|
|
bison.Option('show_uptime', default=True, field_type=bool),
|
|
bison.Option('show_total_output_tokens', default=True, field_type=bool),
|
|
bison.Option('show_backend_info', default=True, field_type=bool),
|
|
bison.Option('load_num_prompts', default=True, field_type=bool),
|
|
bison.Option('manual_model_name', default=None, field_type=Union[str, None]),
|
|
bison.Option('backend_request_timeout', default=30, field_type=int)
|
|
)
|