From e7b57cad7b6a5c4a93b2066de2058553da349bb6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 28 Sep 2023 18:40:24 -0600 Subject: [PATCH 001/163] set up cluster config and basic background workers --- daemon.py | 2 +- llm_server/cluster/__init__.py | 0 llm_server/cluster/datastore.py | 0 llm_server/cluster/funcs/__init__.py | 0 llm_server/cluster/funcs/backend.py | 26 +++++++++++ llm_server/cluster/redis_config_cache.py | 42 +++++++++++++++++ llm_server/cluster/worker.py | 25 +++++++++++ llm_server/config/load.py | 23 ++++++---- llm_server/config/redis_config.py | 3 ++ .../{routes/cache.py => custom_redis.py} | 45 ++++++++++++++++--- llm_server/database/database.py | 2 +- llm_server/helpers.py | 2 +- llm_server/llm/__init__.py | 2 +- llm_server/llm/info.py | 11 ++--- llm_server/llm/llm_backend.py | 2 +- llm_server/llm/oobabooga/ooba_backend.py | 2 +- llm_server/llm/openai/transform.py | 2 +- llm_server/llm/vllm/generate.py | 2 +- llm_server/opts.py | 1 + llm_server/pre_fork.py | 2 +- llm_server/routes/helpers/client.py | 3 +- llm_server/routes/openai/chat_completions.py | 2 +- llm_server/routes/openai/completions.py | 5 +-- llm_server/routes/openai/info.py | 2 +- llm_server/routes/openai/models.py | 2 +- llm_server/routes/openai/simulated.py | 2 +- llm_server/routes/queue.py | 2 +- llm_server/routes/request_handler.py | 2 +- llm_server/routes/stats.py | 2 +- llm_server/routes/v1/generate_stats.py | 2 +- llm_server/routes/v1/generate_stream.py | 1 - llm_server/routes/v1/info.py | 2 +- llm_server/routes/v1/proxy.py | 4 +- llm_server/workers/blocking.py | 2 +- llm_server/workers/main.py | 3 +- llm_server/workers/printer.py | 2 +- llm_server/workers/recent.py | 2 +- requirements.txt | 1 - server.py | 9 ++-- test-cluster.py | 29 ++++++++++++ 40 files changed, 219 insertions(+), 54 deletions(-) create mode 100644 llm_server/cluster/__init__.py create mode 100644 llm_server/cluster/datastore.py create mode 100644 llm_server/cluster/funcs/__init__.py create mode 100644 llm_server/cluster/funcs/backend.py create mode 100644 llm_server/cluster/redis_config_cache.py create mode 100644 llm_server/cluster/worker.py create mode 100644 llm_server/config/redis_config.py rename llm_server/{routes/cache.py => custom_redis.py} (79%) create mode 100644 test-cluster.py diff --git a/daemon.py b/daemon.py index 20ec300..93e8d34 100644 --- a/daemon.py +++ b/daemon.py @@ -1,6 +1,6 @@ import time -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis try: import gevent.monkey diff --git a/llm_server/cluster/__init__.py b/llm_server/cluster/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/llm_server/cluster/datastore.py b/llm_server/cluster/datastore.py new file mode 100644 index 0000000..e69de29 diff --git a/llm_server/cluster/funcs/__init__.py b/llm_server/cluster/funcs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/llm_server/cluster/funcs/backend.py b/llm_server/cluster/funcs/backend.py new file mode 100644 index 0000000..5b7b535 --- /dev/null +++ b/llm_server/cluster/funcs/backend.py @@ -0,0 +1,26 @@ +from llm_server.cluster.redis_config_cache import RedisClusterStore +from llm_server.llm.info import get_running_model + + +def test_backend(backend_url: str): + running_model, err = get_running_model(backend_url) + if not running_model: + return False + return True + + +def get_best_backends(): + cluster_config = RedisClusterStore('cluster_config') + backends = cluster_config.all() + result = {} + for k, v in backends.items(): + b = cluster_config.get_backend(k) + status = b['online'] + priority = b['priority'] + result[k] = {'status': status, 'priority': priority} + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: kv[1]['priority'], + reverse=True + ) + return [url for url, info in online_backends] diff --git a/llm_server/cluster/redis_config_cache.py b/llm_server/cluster/redis_config_cache.py new file mode 100644 index 0000000..00a6a02 --- /dev/null +++ b/llm_server/cluster/redis_config_cache.py @@ -0,0 +1,42 @@ +import pickle + +from llm_server.custom_redis import RedisCustom + + +class RedisClusterStore: + def __init__(self, name: str, **kwargs): + self.name = name + self.config_redis = RedisCustom(name, **kwargs) + + def clear(self): + self.config_redis.flush() + + def load(self, config: dict): + for k, v in config.items(): + self.set_backend(k, v) + + def set_backend(self, name: str, values: dict): + self.config_redis.hset(name, mapping={k: pickle.dumps(v) for k, v in values.items()}) + self.set_backend_value(name, 'online', False) + + def set_backend_value(self, key: str, name: str, value): + self.config_redis.hset(key, name, pickle.dumps(value)) + + def get_backend(self, name: str): + r = self.config_redis.hgetall(name) + output = {} + for k, v in r.items(): + output[k.decode('utf8')] = pickle.loads(v) + return output + + def all(self): + keys = self.config_redis.keys('*') + if keys: + result = {} + for key in keys: + if key != f'{self.name}:____': + v = self.get_backend(key) + result[key] = v + return result + else: + return {} diff --git a/llm_server/cluster/worker.py b/llm_server/cluster/worker.py new file mode 100644 index 0000000..4aaaf6a --- /dev/null +++ b/llm_server/cluster/worker.py @@ -0,0 +1,25 @@ +import time +from threading import Thread + +from llm_server.cluster.funcs.backend import test_backend +from llm_server.cluster.redis_config_cache import RedisClusterStore + +cluster_config = RedisClusterStore('cluster_config') + + +def cluster_worker(): + while True: + threads = [] + for n, v in cluster_config.all().items(): + thread = Thread(target=check_backend, args=(n, v)) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + time.sleep(10) + + +def check_backend(n, v): + # Check if backends are online + online = test_backend(v['backend_url']) + cluster_config.set_backend_value(n, 'online', online) diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 64469b2..82afe81 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -5,22 +5,17 @@ import openai from llm_server import opts from llm_server.config.config import ConfigLoader, config_default_vars, config_required_vars +from llm_server.custom_redis import redis from llm_server.database.conn import database from llm_server.database.database import get_number_of_rows -from llm_server.helpers import resolve_path -from llm_server.routes.cache import redis -def load_config(config_path, script_path): +def load_config(config_path): config_loader = ConfigLoader(config_path, config_default_vars, config_required_vars) success, config, msg = config_loader.load_config() if not success: return success, config, msg - # Resolve relative directory to the directory of the script - if config['database_path'].startswith('./'): - config['database_path'] = resolve_path(script_path, config['database_path'].strip('./')) - if config['mode'] not in ['oobabooga', 'vllm']: print('Unknown mode:', config['mode']) sys.exit(1) @@ -34,7 +29,7 @@ def load_config(config_path, script_path): opts.context_size = config['token_limit'] opts.show_num_prompts = config['show_num_prompts'] opts.show_uptime = config['show_uptime'] - opts.backend_url = config['backend_url'].strip('/') + opts.cluster = config['cluster'] opts.show_total_output_tokens = config['show_total_output_tokens'] opts.netdata_root = config['netdata_root'] opts.simultaneous_requests_per_ip = config['simultaneous_requests_per_ip'] @@ -81,3 +76,15 @@ def load_config(config_path, script_path): redis.set('backend_mode', opts.mode) return success, config, msg + + +def parse_backends(config): + if not config.get('cluster'): + return False + cluster = config.get('cluster') + config = {} + for item in cluster: + backend_url = item['backend_url'].strip('/') + item['backend_url'] = backend_url + config[backend_url] = item + return config diff --git a/llm_server/config/redis_config.py b/llm_server/config/redis_config.py new file mode 100644 index 0000000..06ab1d3 --- /dev/null +++ b/llm_server/config/redis_config.py @@ -0,0 +1,3 @@ +from llm_server.custom_redis import RedisCustom + +redis_config = RedisCustom('redis_config') diff --git a/llm_server/routes/cache.py b/llm_server/custom_redis.py similarity index 79% rename from llm_server/routes/cache.py rename to llm_server/custom_redis.py index d7046db..1b0b5f3 100644 --- a/llm_server/routes/cache.py +++ b/llm_server/custom_redis.py @@ -1,19 +1,20 @@ +import pickle import sys import traceback -from typing import Callable, List, Mapping, Union +from typing import Callable, List, Mapping, Union, Optional import redis as redis_pkg import simplejson as json from flask_caching import Cache from redis import Redis -from redis.typing import AnyKeyT, EncodableT, ExpiryT, FieldT, KeyT, ZScoreBoundT +from redis.typing import AnyKeyT, EncodableT, ExpiryT, FieldT, KeyT, ZScoreBoundT, PatternT flask_cache = Cache(config={'CACHE_TYPE': 'RedisCache', 'CACHE_REDIS_URL': 'redis://localhost:6379/0', 'CACHE_KEY_PREFIX': 'local_llm_flask'}) ONE_MONTH_SECONDS = 2678000 -class RedisWrapper: +class RedisCustom: """ A wrapper class to set prefixes to keys. """ @@ -40,7 +41,6 @@ class RedisWrapper: :param dtype: convert to this type :return: """ - d = self.redis.get(self._key(key)) if dtype and d: try: @@ -129,9 +129,35 @@ class RedisWrapper: ): return self.redis.zadd(self._key(name), mapping, nx, xx, ch, incr, gt, lt) + def hset( + self, + name: str, + key: Optional = None, + value=None, + mapping: Optional[dict] = None, + items: Optional[list] = None, + ): + return self.redis.hset(self._key(name), key, value, mapping, items) + def hkeys(self, name: str): return self.redis.hkeys(self._key(name)) + def hmget(self, name: str, keys: List, *args: List): + return self.redis.hmget(self._key(name), keys, *args) + + def hgetall(self, name: str): + return self.redis.hgetall(self._key(name)) + + def keys(self, pattern: PatternT = "*", **kwargs): + raw_keys = self.redis.keys(self._key(pattern), **kwargs) + keys = [] + for key in raw_keys: + p = key.decode('utf-8').split(':') + if len(p) > 2: + del p[0] + keys.append(':'.join(p)) + return keys + def set_dict(self, key: Union[list, dict], dict_value, ex: Union[ExpiryT, None] = None): return self.set(key, json.dumps(dict_value), ex=ex) @@ -142,6 +168,15 @@ class RedisWrapper: else: return json.loads(r.decode("utf-8")) + def setp(self, name, value): + self.redis.set(name, pickle.dumps(value)) + + def getp(self, name: str): + r = self.redis.get(name) + if r: + return pickle.load(r) + return r + def flush(self): flushed = [] for key in self.redis.scan_iter(f'{self.prefix}:*'): @@ -150,4 +185,4 @@ class RedisWrapper: return flushed -redis = RedisWrapper('local_llm') +redis = RedisCustom('local_llm') diff --git a/llm_server/database/database.py b/llm_server/database/database.py index 9bfe578..3779c83 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -6,7 +6,7 @@ import llm_server from llm_server import opts from llm_server.database.conn import database from llm_server.llm.vllm import tokenize -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, response_tokens: int = None, is_error: bool = False): diff --git a/llm_server/helpers.py b/llm_server/helpers.py index 44b436b..d6eb7d9 100644 --- a/llm_server/helpers.py +++ b/llm_server/helpers.py @@ -8,7 +8,7 @@ import simplejson as json from flask import make_response from llm_server import opts -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def resolve_path(*p: str): diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 742b1a5..a08b25e 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -1,5 +1,5 @@ from llm_server.llm import oobabooga, vllm -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def get_token_count(prompt: str): diff --git a/llm_server/llm/info.py b/llm_server/llm/info.py index 5a529ba..bedf3eb 100644 --- a/llm_server/llm/info.py +++ b/llm_server/llm/info.py @@ -3,20 +3,21 @@ import requests from llm_server import opts -def get_running_model(): - # TODO: cache the results for 1 min so we don't have to keep calling the backend - # TODO: only use one try/catch +def get_running_model(backend_url: str): + # TODO: remove this once we go to Redis + if not backend_url: + backend_url = opts.backend_url if opts.mode == 'oobabooga': try: - backend_response = requests.get(f'{opts.backend_url}/api/v1/model', timeout=opts.backend_request_timeout, verify=opts.verify_ssl) + backend_response = requests.get(f'{backend_url}/api/v1/model', timeout=opts.backend_request_timeout, verify=opts.verify_ssl) r_json = backend_response.json() return r_json['result'], None except Exception as e: return False, e elif opts.mode == 'vllm': try: - backend_response = requests.get(f'{opts.backend_url}/model', timeout=opts.backend_request_timeout, verify=opts.verify_ssl) + backend_response = requests.get(f'{backend_url}/model', timeout=opts.backend_request_timeout, verify=opts.verify_ssl) r_json = backend_response.json() return r_json['model'], None except Exception as e: diff --git a/llm_server/llm/llm_backend.py b/llm_server/llm/llm_backend.py index 1c11c17..153f66d 100644 --- a/llm_server/llm/llm_backend.py +++ b/llm_server/llm/llm_backend.py @@ -4,7 +4,7 @@ import flask from llm_server import opts from llm_server.llm import get_token_count -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis class LLMBackend: diff --git a/llm_server/llm/oobabooga/ooba_backend.py b/llm_server/llm/oobabooga/ooba_backend.py index 4336756..78f2190 100644 --- a/llm_server/llm/oobabooga/ooba_backend.py +++ b/llm_server/llm/oobabooga/ooba_backend.py @@ -3,7 +3,7 @@ from flask import jsonify from ..llm_backend import LLMBackend from ...database.database import log_prompt from ...helpers import safe_list_get -from ...routes.cache import redis +from llm_server.custom_redis import redis from ...routes.helpers.client import format_sillytavern_err from ...routes.helpers.http import validate_json diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index d5b64e3..8f1898e 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -12,7 +12,7 @@ from flask import jsonify, make_response import llm_server from llm_server import opts from llm_server.llm import get_token_count -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis ANTI_RESPONSE_RE = re.compile(r'^### (.*?)(?:\:)?\s') # Match a "### XXX" line. ANTI_CONTINUATION_RE = re.compile(r'(.*?### .*?(?:\:)?(.|\n)*)') # Match everything after a "### XXX" line. diff --git a/llm_server/llm/vllm/generate.py b/llm_server/llm/vllm/generate.py index 1549f2e..308b1de 100644 --- a/llm_server/llm/vllm/generate.py +++ b/llm_server/llm/vllm/generate.py @@ -9,7 +9,7 @@ import requests import llm_server from llm_server import opts -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis # TODO: make the VLMM backend return TPS and time elapsed diff --git a/llm_server/opts.py b/llm_server/opts.py index de23c7a..5eec1fa 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -37,3 +37,4 @@ openai_moderation_workers = 10 openai_org_name = 'OpenAI' openai_silent_trim = False openai_moderation_enabled = True +cluster = {} diff --git a/llm_server/pre_fork.py b/llm_server/pre_fork.py index f3ea0f4..21da08e 100644 --- a/llm_server/pre_fork.py +++ b/llm_server/pre_fork.py @@ -2,7 +2,7 @@ import sys from redis import Redis -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis from llm_server.routes.v1.generate_stats import generate_stats diff --git a/llm_server/routes/helpers/client.py b/llm_server/routes/helpers/client.py index 48e721e..d97e9c5 100644 --- a/llm_server/routes/helpers/client.py +++ b/llm_server/routes/helpers/client.py @@ -1,5 +1,4 @@ -from llm_server import opts -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def format_sillytavern_err(msg: str, level: str = 'info'): diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index cc27dce..a289c78 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -6,7 +6,7 @@ import traceback from flask import Response, jsonify, request from . import openai_bp -from ..cache import redis +from llm_server.custom_redis import redis from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler from ... import opts diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 503f628..05ac7a5 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -4,13 +4,12 @@ import traceback from flask import jsonify, make_response, request from . import openai_bp -from ..cache import redis -from ..helpers.client import format_sillytavern_err +from llm_server.custom_redis import redis from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler from ... import opts from ...llm import get_token_count -from ...llm.openai.transform import build_openai_response, generate_oai_string +from ...llm.openai.transform import generate_oai_string # TODO: add rate-limit headers? diff --git a/llm_server/routes/openai/info.py b/llm_server/routes/openai/info.py index 54959ae..4fc578a 100644 --- a/llm_server/routes/openai/info.py +++ b/llm_server/routes/openai/info.py @@ -1,7 +1,7 @@ from flask import Response from . import openai_bp -from ..cache import flask_cache +from llm_server.custom_redis import flask_cache from ... import opts diff --git a/llm_server/routes/openai/models.py b/llm_server/routes/openai/models.py index 47223e7..4f732e6 100644 --- a/llm_server/routes/openai/models.py +++ b/llm_server/routes/openai/models.py @@ -4,7 +4,7 @@ import requests from flask import jsonify from . import openai_bp -from ..cache import ONE_MONTH_SECONDS, flask_cache, redis +from llm_server.custom_redis import ONE_MONTH_SECONDS, flask_cache, redis from ..stats import server_start_time from ... import opts from ...helpers import jsonify_pretty diff --git a/llm_server/routes/openai/simulated.py b/llm_server/routes/openai/simulated.py index f626490..301e8de 100644 --- a/llm_server/routes/openai/simulated.py +++ b/llm_server/routes/openai/simulated.py @@ -1,7 +1,7 @@ from flask import jsonify from . import openai_bp -from ..cache import ONE_MONTH_SECONDS, flask_cache +from llm_server.custom_redis import ONE_MONTH_SECONDS, flask_cache from ...llm.openai.transform import generate_oai_string from ..stats import server_start_time diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 84cc614..8d85319 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -6,7 +6,7 @@ from uuid import uuid4 from redis import Redis from llm_server import opts -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def increment_ip_count(client_ip: str, redis_key): diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 4b1f640..bb64859 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -11,7 +11,7 @@ from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.auth import parse_token -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis from llm_server.routes.helpers.http import require_api_key, validate_json from llm_server.routes.queue import priority_queue diff --git a/llm_server/routes/stats.py b/llm_server/routes/stats.py index a6e9e17..a0846d9 100644 --- a/llm_server/routes/stats.py +++ b/llm_server/routes/stats.py @@ -1,6 +1,6 @@ from datetime import datetime -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis # proompters_5_min = 0 # concurrent_semaphore = Semaphore(concurrent_gens) diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index e144099..b2dd527 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -6,7 +6,7 @@ from llm_server.database.database import get_distinct_ips_24h, sum_column from llm_server.helpers import deep_sort, round_up_base from llm_server.llm.info import get_running_model from llm_server.netdata import get_power_states -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue from llm_server.routes.stats import get_active_gen_workers, get_total_proompts, server_start_time diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 0fc8f40..45fbf12 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -6,7 +6,6 @@ from typing import Union from flask import request -from ..cache import redis from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v1/info.py index 2091118..7cdbf0f 100644 --- a/llm_server/routes/v1/info.py +++ b/llm_server/routes/v1/info.py @@ -4,7 +4,7 @@ from flask import jsonify, request from . import bp from ..auth import requires_auth -from ..cache import flask_cache +from llm_server.custom_redis import flask_cache from ... import opts from ...llm.info import get_running_model diff --git a/llm_server/routes/v1/proxy.py b/llm_server/routes/v1/proxy.py index 4349526..5ffd194 100644 --- a/llm_server/routes/v1/proxy.py +++ b/llm_server/routes/v1/proxy.py @@ -1,8 +1,6 @@ -from flask import jsonify - from . import bp from .generate_stats import generate_stats -from ..cache import flask_cache +from llm_server.custom_redis import flask_cache from ...helpers import jsonify_pretty diff --git a/llm_server/workers/blocking.py b/llm_server/workers/blocking.py index 27b0815..dcf0047 100644 --- a/llm_server/workers/blocking.py +++ b/llm_server/workers/blocking.py @@ -3,7 +3,7 @@ import time from llm_server import opts from llm_server.llm.generator import generator -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, priority_queue diff --git a/llm_server/workers/main.py b/llm_server/workers/main.py index 747f699..f592c5e 100644 --- a/llm_server/workers/main.py +++ b/llm_server/workers/main.py @@ -1,10 +1,9 @@ import time -from threading import Thread from llm_server import opts from llm_server.database.database import weighted_average_column_for_model from llm_server.llm.info import get_running_model -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def main_background_thread(): diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index cb0f032..6a33835 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -1,7 +1,7 @@ import logging import time -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue logger = logging.getLogger('console_printer') diff --git a/llm_server/workers/recent.py b/llm_server/workers/recent.py index d378a87..c6158d6 100644 --- a/llm_server/workers/recent.py +++ b/llm_server/workers/recent.py @@ -1,6 +1,6 @@ import time -from llm_server.routes.cache import redis +from llm_server.custom_redis import redis def recent_prompters_thread(): diff --git a/requirements.txt b/requirements.txt index 9b0c8eb..7b49eed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -18,4 +18,3 @@ websockets~=11.0.3 basicauth~=1.0.0 openai~=0.28.0 urllib3~=2.0.4 -celery[redis] diff --git a/server.py b/server.py index 06482d4..3c334bc 100644 --- a/server.py +++ b/server.py @@ -8,7 +8,7 @@ except ImportError: pass from llm_server.pre_fork import server_startup -from llm_server.config.load import load_config +from llm_server.config.load import load_config, parse_backends import os import sys from pathlib import Path @@ -36,6 +36,7 @@ from llm_server.stream import init_socketio # TODO: add a way to cancel VLLM gens. Maybe use websockets? # TODO: use coloredlogs # TODO: need to update opts. for workers +# TODO: add a healthcheck to VLLM # Lower priority # TODO: estiamted wait time needs to account for full concurrent_gens but the queue is less than concurrent_gens @@ -63,7 +64,7 @@ import config from llm_server import opts from llm_server.helpers import auto_set_base_client_api from llm_server.llm.vllm.info import vllm_info -from llm_server.routes.cache import RedisWrapper, flask_cache +from llm_server.custom_redis import RedisCustom, flask_cache from llm_server.llm import redis from llm_server.routes.stats import get_active_gen_workers from llm_server.routes.v1.generate_stats import generate_stats @@ -89,9 +90,11 @@ if not success: database.init_db(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database']) create_db() -llm_server.llm.redis = RedisWrapper('local_llm') +llm_server.llm.redis = RedisCustom('local_llm') create_db() +x = parse_backends(config) +print(x) # print(app.url_map) diff --git a/test-cluster.py b/test-cluster.py new file mode 100644 index 0000000..531892b --- /dev/null +++ b/test-cluster.py @@ -0,0 +1,29 @@ +try: + import gevent.monkey + + gevent.monkey.patch_all() +except ImportError: + pass + +import time +from threading import Thread + +from llm_server.cluster.funcs.backend import get_best_backends +from llm_server.cluster.redis_config_cache import RedisClusterStore +from llm_server.cluster.worker import cluster_worker +from llm_server.config.load import parse_backends, load_config + +success, config, msg = load_config('./config/config.yml').resolve().absolute() + +cluster_config = RedisClusterStore('cluster_config') +cluster_config.clear() +cluster_config.load(parse_backends(config)) + +t = Thread(target=cluster_worker) +t.daemon = True +t.start() + +while True: + x = get_best_backends() + print(x) + time.sleep(3) -- 2.34.1 From 624ca74ce5f31bc81292c6507eb7830081d3e625 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 29 Sep 2023 00:09:44 -0600 Subject: [PATCH 002/163] mvp --- daemon.py | 18 +--- llm_server/cluster/backend.py | 71 ++++++++++++ llm_server/cluster/cluster_config.py | 3 + llm_server/cluster/datastore.py | 0 llm_server/cluster/funcs/__init__.py | 0 llm_server/cluster/funcs/backend.py | 26 ----- llm_server/cluster/redis_config_cache.py | 12 ++- llm_server/cluster/redis_cycle.py | 21 ++++ llm_server/cluster/stores.py | 3 + llm_server/cluster/worker.py | 20 ++-- llm_server/config/config.py | 3 +- llm_server/config/load.py | 1 + llm_server/custom_redis.py | 34 ++++-- llm_server/database/database.py | 91 +++++++++------- llm_server/helpers.py | 2 +- llm_server/integer.py | 12 --- llm_server/llm/__init__.py | 2 +- llm_server/llm/generator.py | 4 +- llm_server/llm/info.py | 10 +- llm_server/llm/llm_backend.py | 2 +- llm_server/llm/openai/transform.py | 4 +- llm_server/llm/vllm/generate.py | 12 +-- llm_server/llm/vllm/vllm_backend.py | 12 +-- llm_server/opts.py | 2 +- llm_server/pre_fork.py | 2 +- llm_server/routes/helpers/client.py | 8 +- llm_server/routes/ooba_request_handler.py | 4 +- llm_server/routes/openai/chat_completions.py | 10 +- llm_server/routes/openai/completions.py | 4 +- llm_server/routes/openai/models.py | 3 +- llm_server/routes/queue.py | 2 +- llm_server/routes/request_handler.py | 22 ++-- llm_server/routes/stats.py | 23 ---- llm_server/routes/v1/generate_stats.py | 101 +++++++++++------- llm_server/routes/v1/generate_stream.py | 25 ++--- llm_server/routes/v1/info.py | 40 +++---- llm_server/{stream.py => sock.py} | 0 llm_server/workers/app.py | 35 ------ .../workers/{blocking.py => inferencer.py} | 8 +- llm_server/workers/main.py | 55 ---------- llm_server/workers/mainer.py | 56 ++++++++++ llm_server/workers/{recent.py => recenter.py} | 0 llm_server/workers/threader.py | 50 +++++++++ llm_server/workers/threads.py | 9 -- gunicorn.py => other/gunicorn.py | 5 + server.py | 49 +++++---- test-cluster.py | 20 +++- 47 files changed, 506 insertions(+), 390 deletions(-) create mode 100644 llm_server/cluster/backend.py create mode 100644 llm_server/cluster/cluster_config.py delete mode 100644 llm_server/cluster/datastore.py delete mode 100644 llm_server/cluster/funcs/__init__.py delete mode 100644 llm_server/cluster/funcs/backend.py create mode 100644 llm_server/cluster/redis_cycle.py create mode 100644 llm_server/cluster/stores.py delete mode 100644 llm_server/integer.py rename llm_server/{stream.py => sock.py} (100%) delete mode 100644 llm_server/workers/app.py rename llm_server/workers/{blocking.py => inferencer.py} (88%) delete mode 100644 llm_server/workers/main.py create mode 100644 llm_server/workers/mainer.py rename llm_server/workers/{recent.py => recenter.py} (100%) create mode 100644 llm_server/workers/threader.py delete mode 100644 llm_server/workers/threads.py rename gunicorn.py => other/gunicorn.py (60%) diff --git a/daemon.py b/daemon.py index 93e8d34..82635f0 100644 --- a/daemon.py +++ b/daemon.py @@ -1,22 +1,12 @@ -import time - -from llm_server.custom_redis import redis - -try: - import gevent.monkey - - gevent.monkey.patch_all() -except ImportError: - pass - import os import sys +import time from pathlib import Path from llm_server.config.load import load_config +from llm_server.custom_redis import redis from llm_server.database.create import create_db - -from llm_server.workers.app import start_background +from llm_server.workers.threader import start_background script_path = os.path.dirname(os.path.realpath(__file__)) config_path_environ = os.getenv("CONFIG_PATH") @@ -29,7 +19,7 @@ if __name__ == "__main__": flushed_keys = redis.flush() print('Flushed', len(flushed_keys), 'keys from Redis.') - success, config, msg = load_config(config_path, script_path) + success, config, msg = load_config(config_path) if not success: print('Failed to load config:', msg) sys.exit(1) diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py new file mode 100644 index 0000000..7b28e86 --- /dev/null +++ b/llm_server/cluster/backend.py @@ -0,0 +1,71 @@ +from llm_server.cluster.redis_config_cache import RedisClusterStore +from llm_server.cluster.redis_cycle import redis_cycle +from llm_server.cluster.stores import redis_running_models +from llm_server.llm.info import get_running_model + + +def test_backend(backend_url: str, mode: str): + running_model, err = get_running_model(backend_url, mode) + if not running_model: + return False + return True + + +def get_backends(): + cluster_config = RedisClusterStore('cluster_config') + backends = cluster_config.all() + result = {} + for k, v in backends.items(): + b = cluster_config.get_backend(k) + status = b['online'] + priority = b['priority'] + result[k] = {'status': status, 'priority': priority} + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: -kv[1]['priority'], + reverse=True + ) + offline_backends = sorted( + ((url, info) for url, info in backends.items() if not info['online']), + key=lambda kv: -kv[1]['priority'], + reverse=True + ) + return [url for url, info in online_backends], [url for url, info in offline_backends] + + +def get_a_cluster_backend(): + """ + Get a backend from Redis. If there are no online backends, return None. + """ + online, offline = get_backends() + cycled = redis_cycle('backend_cycler') + c = cycled.copy() + for i in range(len(cycled)): + if cycled[i] in offline: + del c[c.index(cycled[i])] + if len(c): + return c[0] + else: + return None + + +def get_backends_from_model(model_name: str): + cluster_config = RedisClusterStore('cluster_config') + a = cluster_config.all() + matches = [] + for k, v in a.items(): + if v['online'] and v['running_model'] == model_name: + matches.append(k) + return matches + + +def purge_backend_from_running_models(backend_url: str): + keys = redis_running_models.keys() + pipeline = redis_running_models.pipeline() + for model in keys: + pipeline.srem(model, backend_url) + pipeline.execute() + + +def is_valid_model(model_name: str): + return redis_running_models.exists(model_name) diff --git a/llm_server/cluster/cluster_config.py b/llm_server/cluster/cluster_config.py new file mode 100644 index 0000000..14a6cb0 --- /dev/null +++ b/llm_server/cluster/cluster_config.py @@ -0,0 +1,3 @@ +from llm_server.cluster.redis_config_cache import RedisClusterStore + +cluster_config = RedisClusterStore('cluster_config') diff --git a/llm_server/cluster/datastore.py b/llm_server/cluster/datastore.py deleted file mode 100644 index e69de29..0000000 diff --git a/llm_server/cluster/funcs/__init__.py b/llm_server/cluster/funcs/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/llm_server/cluster/funcs/backend.py b/llm_server/cluster/funcs/backend.py deleted file mode 100644 index 5b7b535..0000000 --- a/llm_server/cluster/funcs/backend.py +++ /dev/null @@ -1,26 +0,0 @@ -from llm_server.cluster.redis_config_cache import RedisClusterStore -from llm_server.llm.info import get_running_model - - -def test_backend(backend_url: str): - running_model, err = get_running_model(backend_url) - if not running_model: - return False - return True - - -def get_best_backends(): - cluster_config = RedisClusterStore('cluster_config') - backends = cluster_config.all() - result = {} - for k, v in backends.items(): - b = cluster_config.get_backend(k) - status = b['online'] - priority = b['priority'] - result[k] = {'status': status, 'priority': priority} - online_backends = sorted( - ((url, info) for url, info in backends.items() if info['online']), - key=lambda kv: kv[1]['priority'], - reverse=True - ) - return [url for url, info in online_backends] diff --git a/llm_server/cluster/redis_config_cache.py b/llm_server/cluster/redis_config_cache.py index 00a6a02..ebb6099 100644 --- a/llm_server/cluster/redis_config_cache.py +++ b/llm_server/cluster/redis_config_cache.py @@ -1,3 +1,4 @@ +import hashlib import pickle from llm_server.custom_redis import RedisCustom @@ -13,14 +14,17 @@ class RedisClusterStore: def load(self, config: dict): for k, v in config.items(): - self.set_backend(k, v) + self.add_backend(k, v) - def set_backend(self, name: str, values: dict): + def add_backend(self, name: str, values: dict): self.config_redis.hset(name, mapping={k: pickle.dumps(v) for k, v in values.items()}) self.set_backend_value(name, 'online', False) + h = hashlib.sha256(name.encode('utf-8')).hexdigest() + self.set_backend_value(name, 'hash', f'{h[:8]}-{h[-8:]}') - def set_backend_value(self, key: str, name: str, value): - self.config_redis.hset(key, name, pickle.dumps(value)) + def set_backend_value(self, backend: str, key: str, value): + # By storing the value as a pickle we don't have to cast anything when getting the value from Redis. + self.config_redis.hset(backend, key, pickle.dumps(value)) def get_backend(self, name: str): r = self.config_redis.hgetall(name) diff --git a/llm_server/cluster/redis_cycle.py b/llm_server/cluster/redis_cycle.py new file mode 100644 index 0000000..87893ba --- /dev/null +++ b/llm_server/cluster/redis_cycle.py @@ -0,0 +1,21 @@ +import redis + +r = redis.Redis(host='localhost', port=6379, db=9) + + +def redis_cycle(list_name): + while True: + pipe = r.pipeline() + pipe.lpop(list_name) + popped_element = pipe.execute()[0] + if popped_element is None: + return None + r.rpush(list_name, popped_element) + new_list = r.lrange(list_name, 0, -1) + return [x.decode('utf-8') for x in new_list] + + +def load_backend_cycle(list_name: str, elements: list): + r.delete(list_name) + for element in elements: + r.rpush(list_name, element) diff --git a/llm_server/cluster/stores.py b/llm_server/cluster/stores.py new file mode 100644 index 0000000..c0cbdcc --- /dev/null +++ b/llm_server/cluster/stores.py @@ -0,0 +1,3 @@ +from llm_server.custom_redis import RedisCustom + +redis_running_models = RedisCustom('running_models') diff --git a/llm_server/cluster/worker.py b/llm_server/cluster/worker.py index 4aaaf6a..bee280a 100644 --- a/llm_server/cluster/worker.py +++ b/llm_server/cluster/worker.py @@ -1,10 +1,10 @@ -import time +from datetime import datetime from threading import Thread -from llm_server.cluster.funcs.backend import test_backend -from llm_server.cluster.redis_config_cache import RedisClusterStore - -cluster_config = RedisClusterStore('cluster_config') +from llm_server.cluster.backend import purge_backend_from_running_models, test_backend +from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.stores import redis_running_models +from llm_server.llm.info import get_running_model def cluster_worker(): @@ -16,10 +16,16 @@ def cluster_worker(): threads.append(thread) for thread in threads: thread.join() - time.sleep(10) def check_backend(n, v): # Check if backends are online - online = test_backend(v['backend_url']) + # TODO: also have test_backend() get the uptime + online = test_backend(v['backend_url'], v['mode']) + if online: + running_model, err = get_running_model(v['backend_url'], v['mode']) + if not err: + cluster_config.set_backend_value(n, 'running_model', running_model) + purge_backend_from_running_models(n) + redis_running_models.sadd(running_model, n) cluster_config.set_backend_value(n, 'online', online) diff --git a/llm_server/config/config.py b/llm_server/config/config.py index 59568d7..b98ea49 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -32,7 +32,8 @@ config_default_vars = { 'openai_org_name': 'OpenAI', 'openai_silent_trim': False, 'openai_moderation_enabled': True, - 'netdata_root': None + 'netdata_root': None, + 'show_backends': True, } config_required_vars = ['token_limit', 'concurrent_gens', 'mode', 'llm_middleware_name'] diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 82afe81..09fb127 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -52,6 +52,7 @@ def load_config(config_path): opts.openai_org_name = config['openai_org_name'] opts.openai_silent_trim = config['openai_silent_trim'] opts.openai_moderation_enabled = config['openai_moderation_enabled'] + opts.show_backends = config['show_backends'] if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index 1b0b5f3..b0db49e 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -1,13 +1,13 @@ import pickle import sys import traceback -from typing import Callable, List, Mapping, Union, Optional +from typing import Callable, List, Mapping, Optional, Union import redis as redis_pkg import simplejson as json from flask_caching import Cache from redis import Redis -from redis.typing import AnyKeyT, EncodableT, ExpiryT, FieldT, KeyT, ZScoreBoundT, PatternT +from redis.typing import AnyKeyT, EncodableT, ExpiryT, FieldT, KeyT, PatternT, ZScoreBoundT flask_cache = Cache(config={'CACHE_TYPE': 'RedisCache', 'CACHE_REDIS_URL': 'redis://localhost:6379/0', 'CACHE_KEY_PREFIX': 'local_llm_flask'}) @@ -35,12 +35,12 @@ class RedisCustom: def set(self, key, value, ex: Union[ExpiryT, None] = None): return self.redis.set(self._key(key), value, ex=ex) - def get(self, key, dtype=None, default=None): - """ - :param key: - :param dtype: convert to this type - :return: - """ + def get(self, key, default=None, dtype=None): + # TODO: use pickle + import inspect + if inspect.isclass(default): + raise Exception + d = self.redis.get(self._key(key)) if dtype and d: try: @@ -153,11 +153,23 @@ class RedisCustom: keys = [] for key in raw_keys: p = key.decode('utf-8').split(':') - if len(p) > 2: + if len(p) >= 2: + # Delete prefix del p[0] - keys.append(':'.join(p)) + k = ':'.join(p) + if k != '____': + keys.append(k) return keys + def pipeline(self, transaction=True, shard_hint=None): + return self.redis.pipeline(transaction, shard_hint) + + def exists(self, *names: KeyT): + n = [] + for name in names: + n.append(self._key(name)) + return self.redis.exists(*n) + def set_dict(self, key: Union[list, dict], dict_value, ex: Union[ExpiryT, None] = None): return self.set(key, json.dumps(dict_value), ex=ex) @@ -174,7 +186,7 @@ class RedisCustom: def getp(self, name: str): r = self.redis.get(name) if r: - return pickle.load(r) + return pickle.loads(r) return r def flush(self): diff --git a/llm_server/database/database.py b/llm_server/database/database.py index 3779c83..bf5f537 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -1,60 +1,69 @@ import json import time import traceback +from threading import Thread import llm_server from llm_server import opts +from llm_server.custom_redis import redis from llm_server.database.conn import database from llm_server.llm.vllm import tokenize -from llm_server.custom_redis import redis -def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, response_tokens: int = None, is_error: bool = False): - if isinstance(response, dict) and response.get('results'): - response = response['results'][0]['text'] - try: - j = json.loads(response) - if j.get('results'): - response = j['results'][0]['text'] - except: - pass +def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, cluster_backend, response_tokens: int = None, is_error: bool = False): + def background_task(): + nonlocal ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, cluster_backend, response_tokens, is_error + # Try not to shove JSON into the database. + if isinstance(response, dict) and response.get('results'): + response = response['results'][0]['text'] + try: + j = json.loads(response) + if j.get('results'): + response = j['results'][0]['text'] + except: + pass - prompt_tokens = llm_server.llm.get_token_count(prompt) - if not is_error: - if not response_tokens: - response_tokens = llm_server.llm.get_token_count(response) - else: - response_tokens = None + prompt_tokens = llm_server.llm.get_token_count(prompt) + if not is_error: + if not response_tokens: + response_tokens = llm_server.llm.get_token_count(response) + else: + response_tokens = None - # Sometimes we may want to insert null into the DB, but - # usually we want to insert a float. - if gen_time: - gen_time = round(gen_time, 3) - if is_error: - gen_time = None + # Sometimes we may want to insert null into the DB, but + # usually we want to insert a float. + if gen_time: + gen_time = round(gen_time, 3) + if is_error: + gen_time = None - if not opts.log_prompts: - prompt = None + if not opts.log_prompts: + prompt = None - if not opts.log_prompts and not is_error: - # TODO: test and verify this works as expected - response = None + if not opts.log_prompts and not is_error: + # TODO: test and verify this works as expected + response = None - if token: - increment_token_uses(token) + if token: + increment_token_uses(token) - running_model = redis.get('running_model', str, 'ERROR') - timestamp = int(time.time()) - cursor = database.cursor() - try: - cursor.execute(""" - INSERT INTO prompts - (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) - """, - (ip, token, running_model, opts.mode, opts.backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) - finally: - cursor.close() + running_model = redis.get('running_model', str, 'ERROR') + timestamp = int(time.time()) + cursor = database.cursor() + try: + cursor.execute(""" + INSERT INTO prompts + (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + (ip, token, running_model, opts.mode, cluster_backend, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) + finally: + cursor.close() + + # TODO: use async/await instead of threads + thread = Thread(target=background_task) + thread.start() + thread.join() def is_valid_api_key(api_key): diff --git a/llm_server/helpers.py b/llm_server/helpers.py index d6eb7d9..9fc7274 100644 --- a/llm_server/helpers.py +++ b/llm_server/helpers.py @@ -60,7 +60,7 @@ def round_up_base(n, base): def auto_set_base_client_api(request): - http_host = redis.get('http_host', str) + http_host = redis.get('http_host', dtype=str) host = request.headers.get("Host") if http_host and not re.match(r'((25[0-5]|(2[0-4]|1\d|[1-9]|)\d)\.?\b){4}', http_host): # If the current http_host is not an IP, don't do anything. diff --git a/llm_server/integer.py b/llm_server/integer.py deleted file mode 100644 index 1410dd1..0000000 --- a/llm_server/integer.py +++ /dev/null @@ -1,12 +0,0 @@ -import threading - - -class ThreadSafeInteger: - def __init__(self, value=0): - self.value = value - self._value_lock = threading.Lock() - - def increment(self): - with self._value_lock: - self.value += 1 - return self.value diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index a08b25e..6e39b42 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -3,7 +3,7 @@ from llm_server.custom_redis import redis def get_token_count(prompt: str): - backend_mode = redis.get('backend_mode', str) + backend_mode = redis.get('backend_mode', dtype=str) if backend_mode == 'vllm': return vllm.tokenize(prompt) elif backend_mode == 'ooba': diff --git a/llm_server/llm/generator.py b/llm_server/llm/generator.py index 5dd2093..42c3bb7 100644 --- a/llm_server/llm/generator.py +++ b/llm_server/llm/generator.py @@ -1,14 +1,14 @@ from llm_server import opts -def generator(request_json_body): +def generator(request_json_body, cluster_backend): if opts.mode == 'oobabooga': # from .oobabooga.generate import generate # return generate(request_json_body) raise NotImplementedError elif opts.mode == 'vllm': from .vllm.generate import generate - r = generate(request_json_body) + r = generate(request_json_body, cluster_backend) return r else: raise Exception diff --git a/llm_server/llm/info.py b/llm_server/llm/info.py index bedf3eb..117da3f 100644 --- a/llm_server/llm/info.py +++ b/llm_server/llm/info.py @@ -3,19 +3,15 @@ import requests from llm_server import opts -def get_running_model(backend_url: str): - # TODO: remove this once we go to Redis - if not backend_url: - backend_url = opts.backend_url - - if opts.mode == 'oobabooga': +def get_running_model(backend_url: str, mode: str): + if mode == 'ooba': try: backend_response = requests.get(f'{backend_url}/api/v1/model', timeout=opts.backend_request_timeout, verify=opts.verify_ssl) r_json = backend_response.json() return r_json['result'], None except Exception as e: return False, e - elif opts.mode == 'vllm': + elif mode == 'vllm': try: backend_response = requests.get(f'{backend_url}/model', timeout=opts.backend_request_timeout, verify=opts.verify_ssl) r_json = backend_response.json() diff --git a/llm_server/llm/llm_backend.py b/llm_server/llm/llm_backend.py index 153f66d..e8268b1 100644 --- a/llm_server/llm/llm_backend.py +++ b/llm_server/llm/llm_backend.py @@ -40,6 +40,6 @@ class LLMBackend: def validate_prompt(self, prompt: str) -> Tuple[bool, Union[str, None]]: prompt_len = get_token_count(prompt) if prompt_len > opts.context_size - 10: - model_name = redis.get('running_model', str, 'NO MODEL ERROR') + model_name = redis.get('running_model', 'NO MODEL ERROR', dtype=str) return False, f'Token indices sequence length is longer than the specified maximum sequence length for this model ({prompt_len} > {opts.context_size}, model: {model_name}). Please lower your context size' return True, None diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index 8f1898e..62e0ed8 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -34,7 +34,7 @@ def build_openai_response(prompt, response, model=None): # TODO: async/await prompt_tokens = llm_server.llm.get_token_count(prompt) response_tokens = llm_server.llm.get_token_count(response) - running_model = redis.get('running_model', str, 'ERROR') + running_model = redis.get('running_model', 'ERROR', dtype=str) response = make_response(jsonify({ "id": f"chatcmpl-{generate_oai_string(30)}", @@ -57,7 +57,7 @@ def build_openai_response(prompt, response, model=None): } }), 200) - stats = redis.get('proxy_stats', dict) + stats = redis.get('proxy_stats', dtype=dict) if stats: response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] return response diff --git a/llm_server/llm/vllm/generate.py b/llm_server/llm/vllm/generate.py index 308b1de..caac445 100644 --- a/llm_server/llm/vllm/generate.py +++ b/llm_server/llm/vllm/generate.py @@ -49,7 +49,7 @@ def transform_to_text(json_request, api_response): prompt_tokens = len(llm_server.llm.get_token_count(prompt)) completion_tokens = len(llm_server.llm.get_token_count(text)) - running_model = redis.get('running_model', str, 'ERROR') + running_model = redis.get('running_model', 'ERROR', dtype=str) # https://platform.openai.com/docs/api-reference/making-requests?lang=python return { @@ -82,9 +82,9 @@ def transform_prompt_to_text(prompt: list): return text.strip('\n') -def handle_blocking_request(json_data: dict): +def handle_blocking_request(json_data: dict, cluster_backend): try: - r = requests.post(f'{opts.backend_url}/generate', json=prepare_json(json_data), verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) + r = requests.post(f'{cluster_backend}/generate', json=prepare_json(json_data), verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) except requests.exceptions.ReadTimeout: print(f'Failed to reach VLLM inference endpoint - request to backend timed out') return False, None, 'Request to backend timed out' @@ -97,11 +97,11 @@ def handle_blocking_request(json_data: dict): return True, r, None -def generate(json_data: dict): +def generate(json_data: dict, cluster_backend): if json_data.get('stream'): try: - return requests.post(f'{opts.backend_url}/generate', json=prepare_json(json_data), stream=True, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) + return requests.post(f'{cluster_backend}/generate', json=prepare_json(json_data), stream=True, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) except Exception as e: print(f'Failed to reach VLLM inference endpoint -', f'{e.__class__.__name__}: {e}') else: - return handle_blocking_request(json_data) + return handle_blocking_request(json_data, cluster_backend) diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index e5b0fad..3db99d9 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -19,16 +19,8 @@ class VLLMBackend(LLMBackend): # Failsafe backend_response = '' - r_url = request.url - - def background_task(): - log_prompt(ip=client_ip, token=token, prompt=prompt, response=backend_response, gen_time=elapsed_time, parameters=parameters, headers=headers, backend_response_code=response_status_code, request_url=r_url, - response_tokens=response_json_body.get('details', {}).get('generated_tokens')) - - # TODO: use async/await instead of threads - thread = threading.Thread(target=background_task) - thread.start() - thread.join() + log_prompt(ip=client_ip, token=token, prompt=prompt, response=backend_response, gen_time=elapsed_time, parameters=parameters, headers=headers, backend_response_code=response_status_code, request_url=request.url, + response_tokens=response_json_body.get('details', {}).get('generated_tokens')) return jsonify({'results': [{'text': backend_response}]}), 200 diff --git a/llm_server/opts.py b/llm_server/opts.py index 5eec1fa..0d13979 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -2,7 +2,6 @@ # TODO: rewrite the config system so I don't have to add every single config default here -running_model = 'ERROR' concurrent_gens = 3 mode = 'oobabooga' backend_url = None @@ -38,3 +37,4 @@ openai_org_name = 'OpenAI' openai_silent_trim = False openai_moderation_enabled = True cluster = {} +show_backends = True diff --git a/llm_server/pre_fork.py b/llm_server/pre_fork.py index 21da08e..900210c 100644 --- a/llm_server/pre_fork.py +++ b/llm_server/pre_fork.py @@ -7,7 +7,7 @@ from llm_server.routes.v1.generate_stats import generate_stats def server_startup(s): - if not redis.get('daemon_started', bool): + if not redis.get('daemon_started', dtype=bool): print('Could not find the key daemon_started in Redis. Did you forget to start the daemon process?') sys.exit(1) diff --git a/llm_server/routes/helpers/client.py b/llm_server/routes/helpers/client.py index d97e9c5..a914362 100644 --- a/llm_server/routes/helpers/client.py +++ b/llm_server/routes/helpers/client.py @@ -1,10 +1,14 @@ +from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis -def format_sillytavern_err(msg: str, level: str = 'info'): - http_host = redis.get('http_host', str) +def format_sillytavern_err(msg: str, backend_url: str, level: str = 'info'): + cluster_backend_hash = cluster_config.get_backend_handler(backend_url)['hash'] + http_host = redis.get('http_host', dtype=str) return f"""``` === MESSAGE FROM LLM MIDDLEWARE AT {http_host} === -> {level.upper()} <- {msg} + +BACKEND HASH: {cluster_backend_hash} ```""" diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index d6b02e2..8e0036c 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -31,7 +31,7 @@ class OobaRequestHandler(RequestHandler): msg = f'Ratelimited: you are only allowed to have {opts.simultaneous_requests_per_ip} simultaneous requests at a time. Please complete your other requests before sending another.' backend_response = self.handle_error(msg) if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, is_error=True) + log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.cluster_backend, is_error=True) return backend_response[0], 200 # We only return the response from handle_error(), not the error code def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: @@ -40,7 +40,7 @@ class OobaRequestHandler(RequestHandler): # TODO: how to format this response_msg = error_msg else: - response_msg = format_sillytavern_err(error_msg, error_type) + response_msg = format_sillytavern_err(error_msg, error_type, self.cluster_backend) return jsonify({ 'results': [{'text': response_msg}] diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index a289c78..b3159a5 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -5,11 +5,12 @@ import traceback from flask import Response, jsonify, request -from . import openai_bp from llm_server.custom_redis import redis +from . import openai_bp from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler from ... import opts +from ...cluster.backend import get_a_cluster_backend from ...database.database import log_prompt from ...llm.generator import generator from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt @@ -48,10 +49,11 @@ def openai_chat_completions(): 'stream': True, } try: - response = generator(msg_to_backend) + cluster_backend = get_a_cluster_backend() + response = generator(msg_to_backend, cluster_backend) r_headers = dict(request.headers) r_url = request.url - model = redis.get('running_model', str, 'ERROR') if opts.openai_expose_our_model else request_json_body.get('model') + model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') oai_string = generate_oai_string(30) def generate(): @@ -94,7 +96,7 @@ def openai_chat_completions(): def background_task(): generated_tokens = tokenize(generated_text) - log_prompt(handler.client_ip, handler.token, handler.prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, response_tokens=generated_tokens) + log_prompt(handler.client_ip, handler.token, handler.prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, cluster_backend, response_tokens=generated_tokens) # TODO: use async/await instead of threads thread = threading.Thread(target=background_task) diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 05ac7a5..8950927 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -29,7 +29,7 @@ def openai_completions(): # TODO: async/await prompt_tokens = get_token_count(request_json_body['prompt']) response_tokens = get_token_count(output) - running_model = redis.get('running_model', str, 'ERROR') + running_model = redis.get('running_model', 'ERROR', dtype=str) response = make_response(jsonify({ "id": f"cmpl-{generate_oai_string(30)}", @@ -51,7 +51,7 @@ def openai_completions(): } }), 200) - stats = redis.get('proxy_stats', dict) + stats = redis.get('proxy_stats', dtype=dict) if stats: response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] return response diff --git a/llm_server/routes/openai/models.py b/llm_server/routes/openai/models.py index 4f732e6..657f084 100644 --- a/llm_server/routes/openai/models.py +++ b/llm_server/routes/openai/models.py @@ -7,6 +7,7 @@ from . import openai_bp from llm_server.custom_redis import ONE_MONTH_SECONDS, flask_cache, redis from ..stats import server_start_time from ... import opts +from ...cluster.backend import get_a_cluster_backend from ...helpers import jsonify_pretty from ...llm.info import get_running_model @@ -22,7 +23,7 @@ def openai_list_models(): 'type': error.__class__.__name__ }), 500 # return 500 so Cloudflare doesn't intercept us else: - running_model = redis.get('running_model', str, 'ERROR') + running_model = redis.get('running_model', 'ERROR', dtype=str) oai = fetch_openai_models() r = [] if opts.openai_expose_our_model: diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 8d85319..09ed06c 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -93,6 +93,6 @@ def incr_active_workers(): def decr_active_workers(): redis.decr('active_gen_workers') - new_count = redis.get('active_gen_workers', int, 0) + new_count = redis.get('active_gen_workers', 0, dtype=int) if new_count < 0: redis.set('active_gen_workers', 0) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index bb64859..ecae085 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -5,13 +5,15 @@ import flask from flask import Response, request from llm_server import opts +from llm_server.cluster.backend import get_a_cluster_backend +from llm_server.cluster.cluster_config import cluster_config +from llm_server.custom_redis import redis from llm_server.database.conn import database from llm_server.database.database import log_prompt from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.auth import parse_token -from llm_server.custom_redis import redis from llm_server.routes.helpers.http import require_api_key, validate_json from llm_server.routes.queue import priority_queue @@ -35,7 +37,9 @@ class RequestHandler: self.client_ip = self.get_client_ip() self.token = self.get_auth_token() self.token_priority, self.token_simultaneous_ip = self.get_token_ratelimit() - self.backend = get_backend() + self.cluster_backend = get_a_cluster_backend() + self.cluster_backend_info = cluster_config.get_backend(self.cluster_backend) + self.backend = get_backend_handler(self.cluster_backend) self.parameters = None self.used = False redis.zadd('recent_prompters', {self.client_ip: time.time()}) @@ -119,7 +123,7 @@ class RequestHandler: backend_response = self.handle_error(combined_error_message, 'Validation Error') if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), 0, self.parameters, dict(self.request.headers), 0, self.request.url, is_error=True) + log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), 0, self.parameters, dict(self.request.headers), 0, self.request.url, self.cluster_backend, is_error=True) return False, backend_response return True, (None, 0) @@ -131,7 +135,7 @@ class RequestHandler: request_valid, invalid_response = self.validate_request(prompt, do_log=True) if not request_valid: return (False, None, None, 0), invalid_response - event = priority_queue.put((llm_request, self.client_ip, self.token, self.parameters), self.token_priority) + event = priority_queue.put((llm_request, self.client_ip, self.token, self.parameters, self.cluster_backend), self.token_priority) else: event = None @@ -160,7 +164,7 @@ class RequestHandler: else: error_msg = error_msg.strip('.') + '.' backend_response = self.handle_error(error_msg) - log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), response_status_code, self.request.url, is_error=True) + log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.cluster_backend, is_error=True) return (False, None, None, 0), backend_response # =============================================== @@ -180,7 +184,7 @@ class RequestHandler: if return_json_err: error_msg = 'The backend did not return valid JSON.' backend_response = self.handle_error(error_msg) - log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), elapsed_time, self.parameters, dict(self.request.headers), response_status_code, self.request.url, is_error=True) + log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), elapsed_time, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.cluster_backend, is_error=True) return (False, None, None, 0), backend_response # =============================================== @@ -214,10 +218,10 @@ class RequestHandler: raise NotImplementedError -def get_backend(): - if opts.mode == 'oobabooga': +def get_backend_handler(mode): + if mode == 'oobabooga': return OobaboogaBackend() - elif opts.mode == 'vllm': + elif mode == 'vllm': return VLLMBackend() else: raise Exception diff --git a/llm_server/routes/stats.py b/llm_server/routes/stats.py index a0846d9..b4dea54 100644 --- a/llm_server/routes/stats.py +++ b/llm_server/routes/stats.py @@ -2,32 +2,9 @@ from datetime import datetime from llm_server.custom_redis import redis -# proompters_5_min = 0 -# concurrent_semaphore = Semaphore(concurrent_gens) - server_start_time = datetime.now() -# TODO: do I need this? -# def elapsed_times_cleanup(): -# global wait_in_queue_elapsed -# while True: -# current_time = time.time() -# with wait_in_queue_elapsed_lock: -# global wait_in_queue_elapsed -# wait_in_queue_elapsed = [(end_time, elapsed_time) for end_time, elapsed_time in wait_in_queue_elapsed if current_time - end_time <= 60] -# time.sleep(1) - - -def calculate_avg_gen_time(): - # Get the average generation time from Redis - average_generation_time = redis.get('average_generation_time') - if average_generation_time is None: - return 0 - else: - return float(average_generation_time) - - def get_total_proompts(): count = redis.get('proompts') if count is None: diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index b2dd527..66dd316 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -2,11 +2,12 @@ import time from datetime import datetime from llm_server import opts +from llm_server.cluster.backend import get_a_cluster_backend, test_backend +from llm_server.cluster.cluster_config import cluster_config +from llm_server.custom_redis import redis from llm_server.database.database import get_distinct_ips_24h, sum_column from llm_server.helpers import deep_sort, round_up_base from llm_server.llm.info import get_running_model -from llm_server.netdata import get_power_states -from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue from llm_server.routes.stats import get_active_gen_workers, get_total_proompts, server_start_time @@ -33,52 +34,43 @@ def calculate_wait_time(gen_time_calc, proompters_in_queue, concurrent_gens, act return gen_time_calc -# TODO: have routes/__init__.py point to the latest API version generate_stats() - def generate_stats(regen: bool = False): if not regen: - c = redis.get('proxy_stats', dict) + c = redis.get('proxy_stats', dtype=dict) if c: return c - model_name, error = get_running_model() # will return False when the fetch fails - if isinstance(model_name, bool): - online = False - else: - online = True - redis.set('running_model', model_name) + default_backend_url = get_a_cluster_backend() + default_backend_info = cluster_config.get_backend(default_backend_url) + if not default_backend_info.get('mode'): + # TODO: remove + print('DAEMON NOT FINISHED STARTING') + return + base_client_api = redis.get('base_client_api', dtype=str) + proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf')) + average_generation_elapsed_sec = redis.get('average_generation_elapsed_sec', 0) - # t = elapsed_times.copy() # copy since we do multiple operations and don't want it to change - # if len(t) == 0: - # estimated_wait = 0 - # else: - # waits = [elapsed for end, elapsed in t] - # estimated_wait = int(sum(waits) / len(waits)) + online = test_backend(default_backend_url, default_backend_info['mode']) + if online: + running_model, err = get_running_model(default_backend_url, default_backend_info['mode']) + cluster_config.set_backend_value(default_backend_url, 'running_model', running_model) + else: + running_model = None active_gen_workers = get_active_gen_workers() proompters_in_queue = len(priority_queue) - # This is so wildly inaccurate it's disabled until I implement stats reporting into VLLM. + # This is so wildly inaccurate it's disabled. # estimated_avg_tps = redis.get('estimated_avg_tps', float, default=0) - average_generation_time = redis.get('average_generation_elapsed_sec', float, default=0) - estimated_wait_sec = calculate_wait_time(average_generation_time, proompters_in_queue, opts.concurrent_gens, active_gen_workers) - - if opts.netdata_root: - netdata_stats = {} - power_states = get_power_states() - for gpu, power_state in power_states.items(): - netdata_stats[gpu] = { - 'power_state': power_state, - # 'wh_wasted_1_hr': get_gpu_wh(int(gpu.strip('gpu'))) - } - else: - netdata_stats = {} - - base_client_api = redis.get('base_client_api', str) - proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf')) + # TODO: make this for the currently selected backend + estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, opts.concurrent_gens, active_gen_workers) output = { + 'default': { + 'model': running_model, + 'backend': default_backend_info['hash'], + }, 'stats': { 'proompters': { '5_min': proompters_5_min, @@ -86,9 +78,10 @@ def generate_stats(regen: bool = False): }, 'proompts_total': get_total_proompts() if opts.show_num_prompts else None, 'uptime': int((datetime.now() - server_start_time).total_seconds()) if opts.show_uptime else None, - 'average_generation_elapsed_sec': int(average_generation_time), + 'average_generation_elapsed_sec': int(average_generation_elapsed_sec), # 'estimated_avg_tps': estimated_avg_tps, 'tokens_generated': sum_column('prompts', 'response_tokens') if opts.show_total_output_tokens else None, + 'num_backends': len(cluster_config.all()) if opts.show_backends else None, }, 'online': online, 'endpoints': { @@ -103,10 +96,7 @@ def generate_stats(regen: bool = False): 'timestamp': int(time.time()), 'config': { 'gatekeeper': 'none' if opts.auth_required is False else 'token', - 'context_size': opts.context_size, 'concurrent': opts.concurrent_gens, - 'model': opts.manual_model_name if opts.manual_model_name else model_name, - 'mode': opts.mode, 'simultaneous_requests_per_ip': opts.simultaneous_requests_per_ip, }, 'keys': { @@ -114,8 +104,41 @@ def generate_stats(regen: bool = False): 'anthropicKeys': '∞', }, 'backend_info': redis.get_dict('backend_info') if opts.show_backend_info else None, - 'nvidia': netdata_stats } + + if opts.show_backends: + for backend_url, v in cluster_config.all().items(): + backend_info = cluster_config.get_backend(backend_url) + if not backend_info['online']: + continue + + # TODO: have this fetch the data from VLLM which will display GPU utalization + # if opts.netdata_root: + # netdata_stats = {} + # power_states = get_power_states() + # for gpu, power_state in power_states.items(): + # netdata_stats[gpu] = { + # 'power_state': power_state, + # # 'wh_wasted_1_hr': get_gpu_wh(int(gpu.strip('gpu'))) + # } + # else: + # netdata_stats = {} + netdata_stats = {} + + # TODO: use value returned by VLLM backend here + # backend_uptime = int((datetime.now() - backend_info['start_time']).total_seconds()) if opts.show_uptime else None + backend_uptime = -1 + + output['backend_info'][backend_info['hash']] = { + 'uptime': backend_uptime, + # 'context_size': opts.context_size, + 'model': opts.manual_model_name if opts.manual_model_name else backend_info.get('running_model', 'ERROR'), + 'mode': backend_info['mode'], + 'nvidia': netdata_stats + } + else: + output['backend_info'] = {} + result = deep_sort(output) # It may take a bit to get the base client API, so don't cache until then. diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 45fbf12..e3aeeb0 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -1,5 +1,4 @@ import json -import threading import time import traceback from typing import Union @@ -10,10 +9,11 @@ from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts +from ...cluster.backend import get_a_cluster_backend from ...database.database import log_prompt from ...llm.generator import generator from ...llm.vllm import tokenize -from ...stream import sock +from ...sock import sock # TODO: have workers process streaming requests @@ -35,19 +35,13 @@ def stream(ws): log_in_bg(quitting_err_msg, is_error=True) def log_in_bg(generated_text_bg, elapsed_time_bg: Union[int, float] = None, is_error: bool = False, status_code: int = None): - - def background_task_exception(): - generated_tokens = tokenize(generated_text_bg) - log_prompt(handler.client_ip, handler.token, input_prompt, generated_text_bg, elapsed_time_bg, handler.parameters, r_headers, status_code, r_url, response_tokens=generated_tokens, is_error=is_error) - - # TODO: use async/await instead of threads - thread = threading.Thread(target=background_task_exception) - thread.start() - thread.join() + generated_tokens = tokenize(generated_text_bg) + log_prompt(handler.client_ip, handler.token, input_prompt, generated_text_bg, elapsed_time_bg, handler.parameters, r_headers, status_code, r_url, cluster_backend, response_tokens=generated_tokens, is_error=is_error) if not opts.enable_streaming: return 'Streaming is disabled', 401 + cluster_backend = None r_headers = dict(request.headers) r_url = request.url message_num = 0 @@ -90,14 +84,15 @@ def stream(ws): } # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None), handler.token_priority) + event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority) if not event: r, _ = handler.handle_ratelimited() err_msg = r.json['results'][0]['text'] send_err_and_quit(err_msg) return try: - response = generator(llm_request) + cluster_backend = get_a_cluster_backend() + response = generator(llm_request, cluster_backend) if not response: error_msg = 'Failed to reach backend while streaming.' print('Streaming failed:', error_msg) @@ -142,7 +137,7 @@ def stream(ws): ws.close() end_time = time.time() elapsed_time = end_time - start_time - log_prompt(handler.client_ip, handler.token, input_prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, response_tokens=tokenize(generated_text)) + log_prompt(handler.client_ip, handler.token, input_prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, cluster_backend, response_tokens=tokenize(generated_text)) return message_num += 1 @@ -181,5 +176,5 @@ def stream(ws): # The client closed the stream. end_time = time.time() elapsed_time = end_time - start_time - log_prompt(handler.client_ip, handler.token, input_prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, response_tokens=tokenize(generated_text)) + log_prompt(handler.client_ip, handler.token, input_prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, cluster_backend, response_tokens=tokenize(generated_text)) ws.close() # this is important if we encountered and error and exited early. diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v1/info.py index 7cdbf0f..90778e5 100644 --- a/llm_server/routes/v1/info.py +++ b/llm_server/routes/v1/info.py @@ -2,22 +2,21 @@ import time from flask import jsonify, request +from llm_server.custom_redis import flask_cache from . import bp from ..auth import requires_auth -from llm_server.custom_redis import flask_cache from ... import opts -from ...llm.info import get_running_model - - -# @bp.route('/info', methods=['GET']) -# # @cache.cached(timeout=3600, query_string=True) -# def get_info(): -# # requests.get() -# return 'yes' +from ...cluster.backend import get_a_cluster_backend, get_backends, get_backends_from_model, is_valid_model +from ...cluster.cluster_config import cluster_config @bp.route('/model', methods=['GET']) -def get_model(): +@bp.route('//model', methods=['GET']) +def get_model(model_name=None): + if not model_name: + b = get_a_cluster_backend() + model_name = cluster_config.get_backend(b)['running_model'] + # We will manage caching ourself since we don't want to cache # when the backend is down. Also, Cloudflare won't cache 500 errors. cache_key = 'model_cache::' + request.url @@ -26,16 +25,17 @@ def get_model(): if cached_response: return cached_response - model_name, error = get_running_model() - if not model_name: + if not is_valid_model(model_name): response = jsonify({ - 'code': 502, - 'msg': 'failed to reach backend', - 'type': error.__class__.__name__ - }), 500 # return 500 so Cloudflare doesn't intercept us + 'code': 400, + 'msg': 'Model does not exist.', + }), 400 else: + num_backends = len(get_backends_from_model(model_name)) + response = jsonify({ 'result': opts.manual_model_name if opts.manual_model_name else model_name, + 'model_backend_count': num_backends, 'timestamp': int(time.time()) }), 200 flask_cache.set(cache_key, response, timeout=60) @@ -43,7 +43,11 @@ def get_model(): return response -@bp.route('/backend', methods=['GET']) +@bp.route('/backends', methods=['GET']) @requires_auth def get_backend(): - return jsonify({'backend': opts.backend_url, 'mode': opts.mode}), 200 + online, offline = get_backends() + result = [] + for i in online + offline: + result.append(cluster_config.get_backend(i)) + return jsonify(result), 200 diff --git a/llm_server/stream.py b/llm_server/sock.py similarity index 100% rename from llm_server/stream.py rename to llm_server/sock.py diff --git a/llm_server/workers/app.py b/llm_server/workers/app.py deleted file mode 100644 index fda6fb3..0000000 --- a/llm_server/workers/app.py +++ /dev/null @@ -1,35 +0,0 @@ -from threading import Thread - -from .blocking import start_workers -from .main import main_background_thread -from .moderator import start_moderation_workers -from .printer import console_printer -from .recent import recent_prompters_thread -from .threads import cache_stats -from .. import opts - - -def start_background(): - start_workers(opts.concurrent_gens) - - t = Thread(target=main_background_thread) - t.daemon = True - t.start() - print('Started the main background thread.') - - start_moderation_workers(opts.openai_moderation_workers) - - t = Thread(target=cache_stats) - t.daemon = True - t.start() - print('Started the stats cacher.') - - t = Thread(target=recent_prompters_thread) - t.daemon = True - t.start() - print('Started the recent proompters thread.') - - t = Thread(target=console_printer) - t.daemon = True - t.start() - print('Started the console printer.') diff --git a/llm_server/workers/blocking.py b/llm_server/workers/inferencer.py similarity index 88% rename from llm_server/workers/blocking.py rename to llm_server/workers/inferencer.py index dcf0047..626e34b 100644 --- a/llm_server/workers/blocking.py +++ b/llm_server/workers/inferencer.py @@ -2,15 +2,15 @@ import threading import time from llm_server import opts -from llm_server.llm.generator import generator from llm_server.custom_redis import redis +from llm_server.llm.generator import generator from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, priority_queue def worker(): while True: need_to_wait() - (request_json_body, client_ip, token, parameters), event_id = priority_queue.get() + (request_json_body, client_ip, token, parameters, cluster_backend), event_id = priority_queue.get() need_to_wait() increment_ip_count(client_ip, 'processing_ips') @@ -22,7 +22,7 @@ def worker(): continue try: - success, response, error_msg = generator(request_json_body) + success, response, error_msg = generator(request_json_body, cluster_backend) event = DataEvent(event_id) event.set((success, response, error_msg)) finally: @@ -42,7 +42,7 @@ def start_workers(num_workers: int): def need_to_wait(): # We need to check the number of active workers since the streaming endpoint may be doing something. - active_workers = redis.get('active_gen_workers', int, 0) + active_workers = redis.get('active_gen_workers', 0, dtype=int) s = time.time() while active_workers >= opts.concurrent_gens: time.sleep(0.01) diff --git a/llm_server/workers/main.py b/llm_server/workers/main.py deleted file mode 100644 index f592c5e..0000000 --- a/llm_server/workers/main.py +++ /dev/null @@ -1,55 +0,0 @@ -import time - -from llm_server import opts -from llm_server.database.database import weighted_average_column_for_model -from llm_server.llm.info import get_running_model -from llm_server.custom_redis import redis - - -def main_background_thread(): - redis.set('average_generation_elapsed_sec', 0) - redis.set('estimated_avg_tps', 0) - redis.set('average_output_tokens', 0) - redis.set('backend_online', 0) - redis.set_dict('backend_info', {}) - - while True: - # TODO: unify this - if opts.mode == 'oobabooga': - running_model, err = get_running_model() - if err: - print(err) - redis.set('backend_online', 0) - else: - redis.set('running_model', running_model) - redis.set('backend_online', 1) - elif opts.mode == 'vllm': - running_model, err = get_running_model() - if err: - print(err) - redis.set('backend_online', 0) - else: - redis.set('running_model', running_model) - redis.set('backend_online', 1) - else: - raise Exception - - # exclude_zeros=True filters out rows where an error message was returned. Previously, if there was an error, 0 - # was entered into the column. The new code enters null instead but we need to be backwards compatible for now. - average_generation_elapsed_sec = weighted_average_column_for_model('prompts', 'generation_time', running_model, opts.mode, opts.backend_url, exclude_zeros=True, include_system_tokens=opts.include_system_tokens_in_stats) or 0 - if average_generation_elapsed_sec: # returns None on exception - redis.set('average_generation_elapsed_sec', average_generation_elapsed_sec) - - # overall = average_column_for_model('prompts', 'generation_time', opts.running_model) - # print(f'Weighted: {average_generation_elapsed_sec}, overall: {overall}') - - average_output_tokens = weighted_average_column_for_model('prompts', 'response_tokens', running_model, opts.mode, opts.backend_url, exclude_zeros=True, include_system_tokens=opts.include_system_tokens_in_stats) or 0 - if average_generation_elapsed_sec: - redis.set('average_output_tokens', average_output_tokens) - - # overall = average_column_for_model('prompts', 'response_tokens', opts.running_model) - # print(f'Weighted: {average_output_tokens}, overall: {overall}') - - estimated_avg_tps = round(average_output_tokens / average_generation_elapsed_sec, 2) if average_generation_elapsed_sec > 0 else 0 # Avoid division by zero - redis.set('estimated_avg_tps', estimated_avg_tps) - time.sleep(60) diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py new file mode 100644 index 0000000..447046f --- /dev/null +++ b/llm_server/workers/mainer.py @@ -0,0 +1,56 @@ +import time + +from llm_server import opts +from llm_server.cluster.backend import get_a_cluster_backend, get_backends +from llm_server.cluster.cluster_config import cluster_config +from llm_server.custom_redis import redis +from llm_server.database.database import weighted_average_column_for_model +from llm_server.llm.info import get_running_model + + +def main_background_thread(): + while True: + online, offline = get_backends() + for backend_url in online: + backend_info = cluster_config.get_backend(backend_url) + backend_mode = backend_info['mode'] + running_model, err = get_running_model(backend_url, backend_mode) + if err: + continue + + average_generation_elapsed_sec, average_output_tokens, estimated_avg_tps = calc_stats_for_backend(backend_url, running_model, backend_mode) + if average_generation_elapsed_sec: # returns None on exception + cluster_config.set_backend_value(backend_url, 'average_generation_elapsed_sec', average_generation_elapsed_sec) + if average_output_tokens: + cluster_config.set_backend_value(backend_url, 'average_output_tokens', average_output_tokens) + if average_generation_elapsed_sec and average_output_tokens: + cluster_config.set_backend_value(backend_url, 'estimated_avg_tps', estimated_avg_tps) + + default_backend_url = get_a_cluster_backend() + default_backend_info = cluster_config.get_backend(default_backend_url) + default_backend_mode = default_backend_info['mode'] + default_running_model, err = get_running_model(default_backend_url, default_backend_mode) + if err: + continue + + default_average_generation_elapsed_sec, default_average_output_tokens, default_estimated_avg_tps = calc_stats_for_backend(default_running_model, default_running_model, default_backend_mode) + if default_average_generation_elapsed_sec: + redis.set('average_generation_elapsed_sec', default_average_generation_elapsed_sec) + if default_average_output_tokens: + redis.set('average_output_tokens', default_average_output_tokens) + if default_average_generation_elapsed_sec and default_average_output_tokens: + redis.set('estimated_avg_tps', default_estimated_avg_tps) + time.sleep(30) + + +def calc_stats_for_backend(backend_url, running_model, backend_mode): + # exclude_zeros=True filters out rows where an error message was returned. Previously, if there was an error, 0 + # was entered into the column. The new code enters null instead but we need to be backwards compatible for now. + average_generation_elapsed_sec = weighted_average_column_for_model('prompts', 'generation_time', + running_model, backend_mode, backend_url, exclude_zeros=True, + include_system_tokens=opts.include_system_tokens_in_stats) or 0 + average_output_tokens = weighted_average_column_for_model('prompts', 'response_tokens', + running_model, backend_mode, backend_url, exclude_zeros=True, + include_system_tokens=opts.include_system_tokens_in_stats) or 0 + estimated_avg_tps = round(average_output_tokens / average_generation_elapsed_sec, 2) if average_generation_elapsed_sec > 0 else 0 # Avoid division by zero + return average_generation_elapsed_sec, average_output_tokens, estimated_avg_tps diff --git a/llm_server/workers/recent.py b/llm_server/workers/recenter.py similarity index 100% rename from llm_server/workers/recent.py rename to llm_server/workers/recenter.py diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py new file mode 100644 index 0000000..83bac2d --- /dev/null +++ b/llm_server/workers/threader.py @@ -0,0 +1,50 @@ +import time +from threading import Thread + +from llm_server import opts +from llm_server.cluster.stores import redis_running_models +from llm_server.cluster.worker import cluster_worker +from llm_server.routes.v1.generate_stats import generate_stats +from llm_server.workers.inferencer import start_workers +from llm_server.workers.mainer import main_background_thread +from llm_server.workers.moderator import start_moderation_workers +from llm_server.workers.printer import console_printer +from llm_server.workers.recenter import recent_prompters_thread + + +def cache_stats(): + while True: + generate_stats(regen=True) + time.sleep(1) + + +def start_background(): + start_workers(opts.concurrent_gens) + + t = Thread(target=main_background_thread) + t.daemon = True + t.start() + print('Started the main background thread.') + + start_moderation_workers(opts.openai_moderation_workers) + + t = Thread(target=cache_stats) + t.daemon = True + t.start() + print('Started the stats cacher.') + + t = Thread(target=recent_prompters_thread) + t.daemon = True + t.start() + print('Started the recent proompters thread.') + + t = Thread(target=console_printer) + t.daemon = True + t.start() + print('Started the console printer.') + + redis_running_models.flush() + t = Thread(target=cluster_worker) + t.daemon = True + t.start() + print('Started the cluster worker.') diff --git a/llm_server/workers/threads.py b/llm_server/workers/threads.py deleted file mode 100644 index d1c5183..0000000 --- a/llm_server/workers/threads.py +++ /dev/null @@ -1,9 +0,0 @@ -import time - -from llm_server.routes.v1.generate_stats import generate_stats - - -def cache_stats(): - while True: - generate_stats(regen=True) - time.sleep(5) diff --git a/gunicorn.py b/other/gunicorn.py similarity index 60% rename from gunicorn.py rename to other/gunicorn.py index 30f9274..099e9ce 100644 --- a/gunicorn.py +++ b/other/gunicorn.py @@ -1,3 +1,8 @@ +""" +This file is used to run certain tasks when the HTTP server starts. +It's located here so it doesn't get imported with daemon.py +""" + try: import gevent.monkey diff --git a/server.py b/server.py index 3c334bc..0214b49 100644 --- a/server.py +++ b/server.py @@ -1,4 +1,4 @@ -from llm_server.config.config import mode_ui_names +from llm_server.cluster.cluster_config import cluster_config try: import gevent.monkey @@ -7,8 +7,6 @@ try: except ImportError: pass -from llm_server.pre_fork import server_startup -from llm_server.config.load import load_config, parse_backends import os import sys from pathlib import Path @@ -16,14 +14,17 @@ from pathlib import Path import simplejson as json from flask import Flask, jsonify, render_template, request -import llm_server +from llm_server.cluster.backend import get_a_cluster_backend, get_backends +from llm_server.cluster.redis_cycle import load_backend_cycle +from llm_server.config.config import mode_ui_names +from llm_server.config.load import load_config, parse_backends from llm_server.database.conn import database from llm_server.database.create import create_db -from llm_server.llm import get_token_count +from llm_server.pre_fork import server_startup from llm_server.routes.openai import openai_bp from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp -from llm_server.stream import init_socketio +from llm_server.sock import init_socketio # TODO: set VLLM to stream ALL data using socket.io. If the socket disconnects, cancel generation. # TODO: add backend fallbacks. Backends at the bottom of the list are higher priority and are fallbacks if the upper ones fail @@ -37,6 +38,8 @@ from llm_server.stream import init_socketio # TODO: use coloredlogs # TODO: need to update opts. for workers # TODO: add a healthcheck to VLLM +# TODO: allow choosing the model by the URL path +# TODO: have VLLM report context size, uptime # Lower priority # TODO: estiamted wait time needs to account for full concurrent_gens but the queue is less than concurrent_gens @@ -64,7 +67,7 @@ import config from llm_server import opts from llm_server.helpers import auto_set_base_client_api from llm_server.llm.vllm.info import vllm_info -from llm_server.custom_redis import RedisCustom, flask_cache +from llm_server.custom_redis import flask_cache from llm_server.llm import redis from llm_server.routes.stats import get_active_gen_workers from llm_server.routes.v1.generate_stats import generate_stats @@ -83,20 +86,18 @@ if config_path_environ: else: config_path = Path(script_path, 'config', 'config.yml') -success, config, msg = load_config(config_path, script_path) +success, config, msg = load_config(config_path) if not success: print('Failed to load config:', msg) sys.exit(1) database.init_db(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database']) create_db() -llm_server.llm.redis = RedisCustom('local_llm') -create_db() -x = parse_backends(config) -print(x) - -# print(app.url_map) +cluster_config.clear() +cluster_config.load(parse_backends(config)) +on, off = get_backends() +load_backend_cycle('backend_cycler', on + off) @app.route('/') @@ -104,12 +105,18 @@ print(x) @app.route('/api/openai') @flask_cache.cached(timeout=10) def home(): - stats = generate_stats() + # Use the default backend + backend_url = get_a_cluster_backend() + if backend_url: + backend_info = cluster_config.get_backend(backend_url) + stats = generate_stats(backend_url) + else: + backend_info = stats = None if not stats['online']: running_model = estimated_wait_sec = 'offline' else: - running_model = redis.get('running_model', str, 'ERROR') + running_model = backend_info['running_model'] active_gen_workers = get_active_gen_workers() if stats['queue']['queued'] == 0 and active_gen_workers >= opts.concurrent_gens: @@ -130,10 +137,16 @@ def home(): info_html = '' mode_info = '' - if opts.mode == 'vllm': + using_vllm = False + for k, v in cluster_config.all().items(): + if v['mode'] == vllm: + using_vllm = True + break + + if using_vllm == 'vllm': mode_info = vllm_info - base_client_api = redis.get('base_client_api', str) + base_client_api = redis.get('base_client_api', dtype=str) return render_template('home.html', llm_middleware_name=opts.llm_middleware_name, diff --git a/test-cluster.py b/test-cluster.py index 531892b..ec1773a 100644 --- a/test-cluster.py +++ b/test-cluster.py @@ -7,23 +7,33 @@ except ImportError: import time from threading import Thread +from llm_server.cluster.redis_cycle import load_backend_cycle -from llm_server.cluster.funcs.backend import get_best_backends -from llm_server.cluster.redis_config_cache import RedisClusterStore +from llm_server.cluster.backend import get_backends, get_a_cluster_backend from llm_server.cluster.worker import cluster_worker from llm_server.config.load import parse_backends, load_config +from llm_server.cluster.redis_config_cache import RedisClusterStore -success, config, msg = load_config('./config/config.yml').resolve().absolute() +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument('config') +args = parser.parse_args() + +success, config, msg = load_config(args.config) cluster_config = RedisClusterStore('cluster_config') cluster_config.clear() cluster_config.load(parse_backends(config)) +on, off = get_backends() +load_backend_cycle('backend_cycler', on + off) t = Thread(target=cluster_worker) t.daemon = True t.start() while True: - x = get_best_backends() - print(x) + # online, offline = get_backends() + # print(online, offline) + # print(get_a_cluster_backend()) time.sleep(3) -- 2.34.1 From 114f36e70984f00b680b41f79662fd6909e2c68b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 19:41:50 -0600 Subject: [PATCH 003/163] functional --- daemon.py | 25 +++++- llm_server/cluster/backend.py | 78 ++++++++++++------- llm_server/cluster/model_choices.py | 88 +++++++++++++++++++++ llm_server/cluster/redis_config_cache.py | 3 + llm_server/cluster/redis_cycle.py | 42 ++++++---- llm_server/cluster/worker.py | 37 +++++---- llm_server/config/config.py | 3 +- llm_server/config/load.py | 2 +- llm_server/custom_redis.py | 31 +++++++- llm_server/database/database.py | 14 ++-- llm_server/llm/__init__.py | 4 +- llm_server/llm/generator.py | 5 +- llm_server/llm/info.py | 15 ++++ llm_server/llm/llm_backend.py | 13 +++- llm_server/llm/oobabooga/ooba_backend.py | 9 ++- llm_server/llm/vllm/generate.py | 69 +++------------- llm_server/llm/vllm/info.py | 6 +- llm_server/llm/vllm/tokenize.py | 8 +- llm_server/llm/vllm/vllm_backend.py | 2 +- llm_server/opts.py | 2 +- llm_server/pre_fork.py | 12 --- llm_server/routes/helpers/client.py | 11 +-- llm_server/routes/ooba_request_handler.py | 4 +- llm_server/routes/queue.py | 39 +++++++--- llm_server/routes/request_handler.py | 27 ++++--- llm_server/routes/stats.py | 27 ++++++- llm_server/routes/v1/generate.py | 10 ++- llm_server/routes/v1/generate_stats.py | 95 ++++------------------- llm_server/routes/v1/info.py | 17 ++-- llm_server/workers/inferencer.py | 25 +++--- llm_server/workers/mainer.py | 22 +----- llm_server/workers/printer.py | 11 ++- llm_server/workers/threader.py | 4 +- other/vllm/vllm_api_server.py | 0 requirements.txt | 11 +-- server.py | 75 +++++++----------- templates/home.html | 35 ++++++++- test-cluster.py | 39 ---------- 38 files changed, 505 insertions(+), 415 deletions(-) create mode 100644 llm_server/cluster/model_choices.py mode change 100755 => 100644 other/vllm/vllm_api_server.py delete mode 100644 test-cluster.py diff --git a/daemon.py b/daemon.py index 82635f0..0fa3601 100644 --- a/daemon.py +++ b/daemon.py @@ -3,9 +3,14 @@ import sys import time from pathlib import Path -from llm_server.config.load import load_config +from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.redis_cycle import redis_cycler_db +from llm_server.cluster.stores import redis_running_models +from llm_server.config.load import load_config, parse_backends from llm_server.custom_redis import redis from llm_server.database.create import create_db +from llm_server.routes.queue import priority_queue +from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.threader import start_background script_path = os.path.dirname(os.path.realpath(__file__)) @@ -19,16 +24,30 @@ if __name__ == "__main__": flushed_keys = redis.flush() print('Flushed', len(flushed_keys), 'keys from Redis.') + redis_cycler_db.flushall() + redis_running_models.flush() + success, config, msg = load_config(config_path) if not success: print('Failed to load config:', msg) sys.exit(1) create_db() + + priority_queue.flush() + cluster_config.clear() + cluster_config.load(parse_backends(config)) + + print('Loading backend stats...') + generate_stats() + start_background() redis.set('daemon_started', 1) print('== Daemon Setup Complete ==\n') - while True: - time.sleep(3600) + try: + while True: + time.sleep(3600) + except KeyboardInterrupt: + redis.set('daemon_started', 0) diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index 7b28e86..bb3e6d4 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -1,23 +1,34 @@ -from llm_server.cluster.redis_config_cache import RedisClusterStore -from llm_server.cluster.redis_cycle import redis_cycle +from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle from llm_server.cluster.stores import redis_running_models -from llm_server.llm.info import get_running_model +from llm_server.llm.generator import generator +from llm_server.llm.info import get_info -def test_backend(backend_url: str, mode: str): - running_model, err = get_running_model(backend_url, mode) - if not running_model: - return False - return True +def test_backend(backend_url: str, test_prompt: bool = False): + backend_info = cluster_config.get_backend(backend_url) + if test_prompt: + data = { + "prompt": "Test prompt", + "stream": False, + "temperature": 0, + "max_new_tokens": 16, + } + success, response, err = generator(data, backend_url, timeout=10) + if not success or not response or err: + return False, {} + i = get_info(backend_url, backend_info['mode']) + if not i.get('model'): + return False, {} + return True, i def get_backends(): - cluster_config = RedisClusterStore('cluster_config') backends = cluster_config.all() result = {} for k, v in backends.items(): b = cluster_config.get_backend(k) - status = b['online'] + status = b.get('online', False) priority = b['priority'] result[k] = {'status': status, 'priority': priority} online_backends = sorted( @@ -33,30 +44,43 @@ def get_backends(): return [url for url, info in online_backends], [url for url, info in offline_backends] -def get_a_cluster_backend(): +def get_a_cluster_backend(model=None): """ Get a backend from Redis. If there are no online backends, return None. + If `model` is not supplied, we will pick one ourself. """ - online, offline = get_backends() - cycled = redis_cycle('backend_cycler') - c = cycled.copy() - for i in range(len(cycled)): - if cycled[i] in offline: - del c[c.index(cycled[i])] - if len(c): - return c[0] + if model: + # First, determine if there are multiple backends hosting the same model. + backends_hosting_model = [i.decode('utf-8') for i in redis_running_models.smembers(model)] + + # If so, create an iterator for those backends + if len(backends_hosting_model): + add_backend_cycler(model, backends_hosting_model) + cycled = redis_cycle(model) + if len(cycled): + return cycled[0] + else: + # No backend hosting that model + return None else: - return None + online, _ = get_backends() + if len(online): + return online[0] def get_backends_from_model(model_name: str): - cluster_config = RedisClusterStore('cluster_config') - a = cluster_config.all() - matches = [] - for k, v in a.items(): - if v['online'] and v['running_model'] == model_name: - matches.append(k) - return matches + return [x.decode('utf-8') for x in redis_running_models.smembers(model_name)] + + +# def verify_context_size(model_name:str): +# b = get_backends_from_model(model_name) +# for backend_url in b: +# backend_info = cluster_config.get_backend(backend_url) +# backend_info.get() + + +def get_running_models(): + return redis_running_models.keys() def purge_backend_from_running_models(backend_url: str): diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py new file mode 100644 index 0000000..c9a94fd --- /dev/null +++ b/llm_server/cluster/model_choices.py @@ -0,0 +1,88 @@ +import numpy as np + +from llm_server import opts +from llm_server.cluster.backend import get_a_cluster_backend, get_backends_from_model, get_running_models +from llm_server.cluster.cluster_config import cluster_config +from llm_server.custom_redis import redis +from llm_server.routes.queue import priority_queue +from llm_server.routes.stats import calculate_wait_time, get_active_gen_workers + + +# TODO: give this a better name! +def get_model_choices(regen: bool = False): + if not regen: + c = redis.getp('model_choices') + if c: + return c + + base_client_api = redis.get('base_client_api', dtype=str) + running_models = get_running_models() + model_choices = {} + for model in running_models: + b = get_backends_from_model(model) + + context_size = [] + avg_gen_per_worker = [] + for backend_url in b: + backend_info = cluster_config.get_backend(backend_url) + if backend_info.get('model_config'): + context_size.append(backend_info['model_config']['max_position_embeddings']) + if backend_info.get('average_generation_elapsed_sec'): + avg_gen_per_worker.append(backend_info['average_generation_elapsed_sec']) + + active_gen_workers = get_active_gen_workers(model) + proompters_in_queue = priority_queue.len(model) + + if len(avg_gen_per_worker): + average_generation_elapsed_sec = np.average(avg_gen_per_worker) + else: + average_generation_elapsed_sec = 0 + estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, opts.concurrent_gens, active_gen_workers) + + if proompters_in_queue == 0 and active_gen_workers >= opts.concurrent_gens: + # There will be a wait if the queue is empty but prompts are processing, but we don't + # know how long. + estimated_wait_sec = f"less than {estimated_wait_sec} seconds" + else: + estimated_wait_sec = f"{estimated_wait_sec} seconds" + + model_choices[model] = { + 'client_api': f'https://{base_client_api}/v2/{model}', + 'ws_client_api': f'wss://{base_client_api}/v2/{model}/stream' if opts.enable_streaming else None, + 'openai_client_api': f'https://{base_client_api}/openai/v2/{model}' if opts.enable_openi_compatible_backend else 'disabled', + 'backend_count': len(b), + 'estimated_wait': estimated_wait_sec, + 'queued': proompters_in_queue, + 'processing': active_gen_workers, + 'avg_generation_time': average_generation_elapsed_sec + } + + if len(context_size): + model_choices[model]['context_size'] = min(context_size) + + model_choices = dict(sorted(model_choices.items())) + + default_backend = get_a_cluster_backend() + default_backend_info = cluster_config.get_backend(default_backend) + default_context_size = default_backend_info['model_config']['max_position_embeddings'] + default_average_generation_elapsed_sec = default_backend_info.get('average_generation_elapsed_sec') + default_active_gen_workers = redis.get(f'active_gen_workers:{default_backend}', dtype=int, default=0) + default_proompters_in_queue = priority_queue.len(default_backend_info['model']) + default_estimated_wait_sec = calculate_wait_time(default_average_generation_elapsed_sec, default_proompters_in_queue, default_backend_info['concurrent_gens'], default_active_gen_workers) + + default_backend_dict = { + 'client_api': f'https://{base_client_api}/v2', + 'ws_client_api': f'wss://{base_client_api}/v2' if opts.enable_streaming else None, + 'openai_client_api': f'https://{base_client_api}/openai/v2' if opts.enable_openi_compatible_backend else 'disabled', + 'estimated_wait': default_estimated_wait_sec, + 'queued': default_proompters_in_queue, + 'processing': default_active_gen_workers, + 'context_size': default_context_size, + 'hash': default_backend_info['hash'], + 'model': default_backend_info['model'], + 'avg_generation_time': default_average_generation_elapsed_sec + } + + redis.setp('model_choices', (model_choices, default_backend_dict)) + + return model_choices, default_backend_dict diff --git a/llm_server/cluster/redis_config_cache.py b/llm_server/cluster/redis_config_cache.py index ebb6099..3bab915 100644 --- a/llm_server/cluster/redis_config_cache.py +++ b/llm_server/cluster/redis_config_cache.py @@ -44,3 +44,6 @@ class RedisClusterStore: return result else: return {} + + # def get(self, name: str): + # return self.all().get(name) diff --git a/llm_server/cluster/redis_cycle.py b/llm_server/cluster/redis_cycle.py index 87893ba..7cff2c4 100644 --- a/llm_server/cluster/redis_cycle.py +++ b/llm_server/cluster/redis_cycle.py @@ -1,21 +1,35 @@ import redis -r = redis.Redis(host='localhost', port=6379, db=9) +redis_cycler_db = redis.Redis(host='localhost', port=6379, db=9) def redis_cycle(list_name): - while True: - pipe = r.pipeline() - pipe.lpop(list_name) - popped_element = pipe.execute()[0] - if popped_element is None: - return None - r.rpush(list_name, popped_element) - new_list = r.lrange(list_name, 0, -1) - return [x.decode('utf-8') for x in new_list] + """ + Emulates itertools.cycle() but returns the complete shuffled list. + :param list_name: + :return: + """ + to_move = redis_cycler_db.rpop(list_name) + if not to_move: + return [] + redis_cycler_db.lpush(list_name, to_move) + new_list = redis_cycler_db.lrange(list_name, 0, -1) + return [x.decode('utf-8') for x in new_list] -def load_backend_cycle(list_name: str, elements: list): - r.delete(list_name) - for element in elements: - r.rpush(list_name, element) +def add_backend_cycler(list_name: str, new_elements: list): + existing_elements = [i.decode('utf-8') for i in redis_cycler_db.lrange(list_name, 0, -1)] + existing_set = set(existing_elements) + + with redis_cycler_db.pipeline() as pipe: + # Add elements + for element in new_elements: + if element not in existing_set: + pipe.rpush(list_name, element) + + # Remove elements + for element in existing_set: + if element not in new_elements: + pipe.lrem(list_name, 0, element) + + pipe.execute() diff --git a/llm_server/cluster/worker.py b/llm_server/cluster/worker.py index bee280a..7956198 100644 --- a/llm_server/cluster/worker.py +++ b/llm_server/cluster/worker.py @@ -1,31 +1,42 @@ -from datetime import datetime +import time from threading import Thread -from llm_server.cluster.backend import purge_backend_from_running_models, test_backend +from llm_server.cluster.backend import test_backend from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.stores import redis_running_models -from llm_server.llm.info import get_running_model def cluster_worker(): + counter = 0 while True: + test_prompt = False + if counter % 4 == 0: + # Only send a test prompt every 120 seconds. + test_prompt = True threads = [] for n, v in cluster_config.all().items(): - thread = Thread(target=check_backend, args=(n, v)) + thread = Thread(target=check_backend, args=(n, v, test_prompt)) thread.start() threads.append(thread) for thread in threads: thread.join() + time.sleep(15) + counter += 1 -def check_backend(n, v): - # Check if backends are online - # TODO: also have test_backend() get the uptime - online = test_backend(v['backend_url'], v['mode']) +def check_backend(n, v, test_prompt): + online, backend_info = test_backend(v['backend_url'], test_prompt=test_prompt) + # purge_backend_from_running_models(n) if online: - running_model, err = get_running_model(v['backend_url'], v['mode']) - if not err: - cluster_config.set_backend_value(n, 'running_model', running_model) - purge_backend_from_running_models(n) - redis_running_models.sadd(running_model, n) + running_model = backend_info['model'] + for k, v in backend_info.items(): + cluster_config.set_backend_value(n, k, v) + redis_running_models.sadd(running_model, n) + else: + for model in redis_running_models.keys(): + redis_running_models.srem(model, n) + + # redis_running_models.srem(backend_info['model'], n) + # backend_cycler_store.lrem(backend_info['model'], 1, n) + cluster_config.set_backend_value(n, 'online', online) diff --git a/llm_server/config/config.py b/llm_server/config/config.py index b98ea49..645e81e 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -34,8 +34,9 @@ config_default_vars = { 'openai_moderation_enabled': True, 'netdata_root': None, 'show_backends': True, + 'cluster_workers': 30 } -config_required_vars = ['token_limit', 'concurrent_gens', 'mode', 'llm_middleware_name'] +config_required_vars = ['cluster', 'mode', 'llm_middleware_name'] mode_ui_names = { 'oobabooga': ('Text Gen WebUI (ooba)', 'Blocking API url', 'Streaming API url'), diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 09fb127..9c2e7f3 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -26,7 +26,6 @@ def load_config(config_path): opts.log_prompts = config['log_prompts'] opts.concurrent_gens = config['concurrent_gens'] opts.frontend_api_client = config['frontend_api_client'] - opts.context_size = config['token_limit'] opts.show_num_prompts = config['show_num_prompts'] opts.show_uptime = config['show_uptime'] opts.cluster = config['cluster'] @@ -53,6 +52,7 @@ def load_config(config_path): opts.openai_silent_trim = config['openai_silent_trim'] opts.openai_moderation_enabled = config['openai_moderation_enabled'] opts.show_backends = config['show_backends'] + opts.cluster_workers = config['cluster_workers'] if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index b0db49e..d5d278f 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -9,17 +9,18 @@ from flask_caching import Cache from redis import Redis from redis.typing import AnyKeyT, EncodableT, ExpiryT, FieldT, KeyT, PatternT, ZScoreBoundT -flask_cache = Cache(config={'CACHE_TYPE': 'RedisCache', 'CACHE_REDIS_URL': 'redis://localhost:6379/0', 'CACHE_KEY_PREFIX': 'local_llm_flask'}) +flask_cache = Cache(config={'CACHE_TYPE': 'RedisCache', 'CACHE_REDIS_URL': 'redis://localhost:6379/15', 'CACHE_KEY_PREFIX': 'local_llm_flask'}) ONE_MONTH_SECONDS = 2678000 -class RedisCustom: +class RedisCustom(Redis): """ A wrapper class to set prefixes to keys. """ def __init__(self, prefix, **kwargs): + super().__init__() self.redis = Redis(**kwargs) self.prefix = prefix try: @@ -108,6 +109,9 @@ class RedisCustom: ): return self.redis.hincrby(self._key(name), key, amount) + def zcard(self, name: KeyT): + return self.redis.zcard(self._key(name)) + def hdel(self, name: str, *keys: List): return self.redis.hdel(self._key(name), *keys) @@ -129,6 +133,9 @@ class RedisCustom: ): return self.redis.zadd(self._key(name), mapping, nx, xx, ch, incr, gt, lt) + def lpush(self, name: str, *values: FieldT): + return self.redis.lpush(self._key(name), *values) + def hset( self, name: str, @@ -164,6 +171,18 @@ class RedisCustom: def pipeline(self, transaction=True, shard_hint=None): return self.redis.pipeline(transaction, shard_hint) + def smembers(self, name: str): + return self.redis.smembers(self._key(name)) + + def spop(self, name: str, count: Optional[int] = None): + return self.redis.spop(self._key(name), count) + + def rpoplpush(self, src, dst): + return self.redis.rpoplpush(src, dst) + + def zpopmin(self, name: KeyT, count: Union[int, None] = None): + return self.redis.zpopmin(self._key(name), count) + def exists(self, *names: KeyT): n = [] for name in names: @@ -196,5 +215,13 @@ class RedisCustom: self.redis.delete(key) return flushed + def flushall(self, asynchronous: bool = ..., **kwargs) -> bool: + self.flush() + return True + + def flushdb(self, asynchronous: bool = ..., **kwargs) -> bool: + self.flush() + return True + redis = RedisCustom('local_llm') diff --git a/llm_server/database/database.py b/llm_server/database/database.py index bf5f537..1dc2145 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -5,14 +5,14 @@ from threading import Thread import llm_server from llm_server import opts -from llm_server.custom_redis import redis +from llm_server.cluster.cluster_config import cluster_config from llm_server.database.conn import database from llm_server.llm.vllm import tokenize -def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, cluster_backend, response_tokens: int = None, is_error: bool = False): +def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, backend_url, response_tokens: int = None, is_error: bool = False): def background_task(): - nonlocal ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, cluster_backend, response_tokens, is_error + nonlocal ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, backend_url, response_tokens, is_error # Try not to shove JSON into the database. if isinstance(response, dict) and response.get('results'): response = response['results'][0]['text'] @@ -23,10 +23,10 @@ def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backe except: pass - prompt_tokens = llm_server.llm.get_token_count(prompt) + prompt_tokens = llm_server.llm.get_token_count(prompt, backend_url) if not is_error: if not response_tokens: - response_tokens = llm_server.llm.get_token_count(response) + response_tokens = llm_server.llm.get_token_count(response, backend_url) else: response_tokens = None @@ -47,7 +47,7 @@ def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backe if token: increment_token_uses(token) - running_model = redis.get('running_model', str, 'ERROR') + running_model = cluster_config.get_backend(backend_url).get('model') timestamp = int(time.time()) cursor = database.cursor() try: @@ -56,7 +56,7 @@ def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backe (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, - (ip, token, running_model, opts.mode, cluster_backend, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) + (ip, token, running_model, opts.mode, backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) finally: cursor.close() diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 6e39b42..3feb027 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -2,10 +2,10 @@ from llm_server.llm import oobabooga, vllm from llm_server.custom_redis import redis -def get_token_count(prompt: str): +def get_token_count(prompt: str, backend_url: str): backend_mode = redis.get('backend_mode', dtype=str) if backend_mode == 'vllm': - return vllm.tokenize(prompt) + return vllm.tokenize(prompt, backend_url) elif backend_mode == 'ooba': return oobabooga.tokenize(prompt) else: diff --git a/llm_server/llm/generator.py b/llm_server/llm/generator.py index 42c3bb7..f05b37c 100644 --- a/llm_server/llm/generator.py +++ b/llm_server/llm/generator.py @@ -1,14 +1,13 @@ from llm_server import opts -def generator(request_json_body, cluster_backend): +def generator(request_json_body, cluster_backend, timeout: int = None): if opts.mode == 'oobabooga': # from .oobabooga.generate import generate # return generate(request_json_body) raise NotImplementedError elif opts.mode == 'vllm': from .vllm.generate import generate - r = generate(request_json_body, cluster_backend) - return r + return generate(request_json_body, cluster_backend, timeout=timeout) else: raise Exception diff --git a/llm_server/llm/info.py b/llm_server/llm/info.py index 117da3f..d1218e2 100644 --- a/llm_server/llm/info.py +++ b/llm_server/llm/info.py @@ -20,3 +20,18 @@ def get_running_model(backend_url: str, mode: str): return False, e else: raise Exception + + +def get_info(backend_url: str, mode: str): + if mode == 'ooba': + return {} + # raise NotImplementedError + elif mode == 'vllm': + try: + r = requests.get(f'{backend_url}/info', verify=opts.verify_ssl, timeout=opts.backend_request_timeout) + j = r.json() + except Exception as e: + return {} + return j + else: + raise Exception diff --git a/llm_server/llm/llm_backend.py b/llm_server/llm/llm_backend.py index e8268b1..e69f8fc 100644 --- a/llm_server/llm/llm_backend.py +++ b/llm_server/llm/llm_backend.py @@ -3,13 +3,17 @@ from typing import Tuple, Union import flask from llm_server import opts -from llm_server.llm import get_token_count +from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis +from llm_server.llm import get_token_count class LLMBackend: _default_params: dict + def __init__(self, backend_url: str): + self.backend_url = backend_url + def handle_response(self, success, request: flask.Request, response_json_body: dict, response_status_code: int, client_ip, token, prompt, elapsed_time, parameters, headers): raise NotImplementedError @@ -38,8 +42,9 @@ class LLMBackend: return True, None def validate_prompt(self, prompt: str) -> Tuple[bool, Union[str, None]]: - prompt_len = get_token_count(prompt) - if prompt_len > opts.context_size - 10: + prompt_len = get_token_count(prompt, self.backend_url) + token_limit = cluster_config.get_backend(self.backend_url)['model_config']['max_position_embeddings'] + if prompt_len > token_limit - 10: model_name = redis.get('running_model', 'NO MODEL ERROR', dtype=str) - return False, f'Token indices sequence length is longer than the specified maximum sequence length for this model ({prompt_len} > {opts.context_size}, model: {model_name}). Please lower your context size' + return False, f'Token indices sequence length is longer than the specified maximum sequence length for this model ({prompt_len} > {token_limit}, model: {model_name}). Please lower your context size' return True, None diff --git a/llm_server/llm/oobabooga/ooba_backend.py b/llm_server/llm/oobabooga/ooba_backend.py index 78f2190..fe450bf 100644 --- a/llm_server/llm/oobabooga/ooba_backend.py +++ b/llm_server/llm/oobabooga/ooba_backend.py @@ -1,9 +1,9 @@ from flask import jsonify +from llm_server.custom_redis import redis from ..llm_backend import LLMBackend from ...database.database import log_prompt from ...helpers import safe_list_get -from llm_server.custom_redis import redis from ...routes.helpers.client import format_sillytavern_err from ...routes.helpers.http import validate_json @@ -33,7 +33,7 @@ class OobaboogaBackend(LLMBackend): error_msg = 'Unknown error.' else: error_msg = error_msg.strip('.') + '.' - backend_response = format_sillytavern_err(error_msg, 'error') + backend_response = format_sillytavern_err(error_msg, error_type='error', backend_url=self.backend_url) log_prompt(client_ip, token, prompt, backend_response, None, parameters, headers, response_status_code, request.url, is_error=True) return jsonify({ 'code': 500, @@ -50,7 +50,8 @@ class OobaboogaBackend(LLMBackend): backend_err = True backend_response = format_sillytavern_err( f'Backend (oobabooga) returned an empty string. This is usually due to an error on the backend during inference. Please check your parameters and try again.', - 'error') + error_type='error', + backend_url=self.backend_url) response_json_body['results'][0]['text'] = backend_response if not backend_err: @@ -61,7 +62,7 @@ class OobaboogaBackend(LLMBackend): **response_json_body }), 200 else: - backend_response = format_sillytavern_err(f'The backend did not return valid JSON.', 'error') + backend_response = format_sillytavern_err(f'The backend did not return valid JSON.', error_type='error', backend_url=self.backend_url) log_prompt(client_ip, token, prompt, backend_response, elapsed_time, parameters, headers, response.status_code, request.url, is_error=True) return jsonify({ 'code': 500, diff --git a/llm_server/llm/vllm/generate.py b/llm_server/llm/vllm/generate.py index caac445..72b0243 100644 --- a/llm_server/llm/vllm/generate.py +++ b/llm_server/llm/vllm/generate.py @@ -24,57 +24,6 @@ def prepare_json(json_data: dict): return json_data -def transform_to_text(json_request, api_response): - """ - This is to convert a streaming request to a non-streamed request. Don't think this is nessesary. - :param json_request: - :param api_response: - :return: - """ - prompt = transform_prompt_to_text(json_request['messages']) - text = '' - finish_reason = None - for line in api_response.split('\n'): - if line.startswith('data:'): - try: - data = json.loads(line[5:].strip()) - except json.decoder.JSONDecodeError: - break - if 'choices' in data: - for choice in data['choices']: - if 'delta' in choice and 'content' in choice['delta']: - text += choice['delta']['content'] - if data['choices'][0]['finish_reason']: - finish_reason = data['choices'][0]['finish_reason'] - - prompt_tokens = len(llm_server.llm.get_token_count(prompt)) - completion_tokens = len(llm_server.llm.get_token_count(text)) - running_model = redis.get('running_model', 'ERROR', dtype=str) - - # https://platform.openai.com/docs/api-reference/making-requests?lang=python - return { - "id": str(uuid4()), - "object": "chat.completion", - "created": int(time.time()), - "model": running_model, - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": completion_tokens, - "total_tokens": prompt_tokens + completion_tokens - }, - "choices": [ - { - "message": { - "role": "assistant", - "content": text - }, - "finish_reason": finish_reason, - "index": 0 - } - ] - } - - def transform_prompt_to_text(prompt: list): text = '' for item in prompt: @@ -82,26 +31,26 @@ def transform_prompt_to_text(prompt: list): return text.strip('\n') -def handle_blocking_request(json_data: dict, cluster_backend): +def handle_blocking_request(json_data: dict, cluster_backend, timeout: int = 10): try: - r = requests.post(f'{cluster_backend}/generate', json=prepare_json(json_data), verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) + r = requests.post(f'{cluster_backend}/generate', json=prepare_json(json_data), verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout if not timeout else timeout) except requests.exceptions.ReadTimeout: - print(f'Failed to reach VLLM inference endpoint - request to backend timed out') + # print(f'Failed to reach VLLM inference endpoint - request to backend timed out') return False, None, 'Request to backend timed out' except Exception as e: - print(f'Failed to reach VLLM inference endpoint -', f'{e.__class__.__name__}: {e}') + # print(f'Failed to reach VLLM inference endpoint -', f'{e.__class__.__name__}: {e}') return False, None, 'Request to backend encountered error' if r.status_code != 200: - print(f'Failed to reach VLLM inference endpoint - got code {r.status_code}') + # print(f'Failed to reach VLLM inference endpoint - got code {r.status_code}') return False, r, f'Backend returned {r.status_code}' return True, r, None -def generate(json_data: dict, cluster_backend): +def generate(json_data: dict, cluster_backend, timeout: int = None): if json_data.get('stream'): try: - return requests.post(f'{cluster_backend}/generate', json=prepare_json(json_data), stream=True, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) + return requests.post(f'{cluster_backend}/generate', json=prepare_json(json_data), stream=True, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout if not timeout else timeout) except Exception as e: - print(f'Failed to reach VLLM inference endpoint -', f'{e.__class__.__name__}: {e}') + return False else: - return handle_blocking_request(json_data, cluster_backend) + return handle_blocking_request(json_data, cluster_backend, timeout=timeout) diff --git a/llm_server/llm/vllm/info.py b/llm_server/llm/vllm/info.py index 996c614..0142301 100644 --- a/llm_server/llm/vllm/info.py +++ b/llm_server/llm/vllm/info.py @@ -1,3 +1,7 @@ +import requests + +from llm_server import opts + vllm_info = """

Important: This endpoint is running vllm and not all Oobabooga parameters are supported.

Supported Parameters: """ \ No newline at end of file +""" diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index a698fd6..747a8b8 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -2,19 +2,21 @@ import requests import tiktoken from llm_server import opts +from llm_server.cluster.cluster_config import cluster_config -def tokenize(prompt: str) -> int: +def tokenize(prompt: str, backend_url: str) -> int: if not prompt: # The tokenizers have issues when the prompt is None. return 0 tokenizer = tiktoken.get_encoding("cl100k_base") + token_limit = cluster_config.get_backend(backend_url)['model_config']['max_position_embeddings'] # First we tokenize it locally to determine if it's worth sending it to the backend. initial_estimate = len(tokenizer.encode(prompt)) - if initial_estimate <= opts.context_size + 200: + if initial_estimate <= token_limit + 200: try: - r = requests.post(f'{opts.backend_url}/tokenize', json={'input': prompt}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) + r = requests.post(f'{backend_url}/tokenize', json={'input': prompt}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) j = r.json() return j['length'] except Exception as e: diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index 3db99d9..a28e59a 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -20,7 +20,7 @@ class VLLMBackend(LLMBackend): backend_response = '' log_prompt(ip=client_ip, token=token, prompt=prompt, response=backend_response, gen_time=elapsed_time, parameters=parameters, headers=headers, backend_response_code=response_status_code, request_url=request.url, - response_tokens=response_json_body.get('details', {}).get('generated_tokens')) + response_tokens=response_json_body.get('details', {}).get('generated_tokens'), backend_url=self.backend_url) return jsonify({'results': [{'text': backend_response}]}), 200 diff --git a/llm_server/opts.py b/llm_server/opts.py index 0d13979..bbd6201 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -5,7 +5,6 @@ concurrent_gens = 3 mode = 'oobabooga' backend_url = None -context_size = 5555 max_new_tokens = 500 auth_required = False log_prompts = False @@ -38,3 +37,4 @@ openai_silent_trim = False openai_moderation_enabled = True cluster = {} show_backends = True +cluster_workers = 30 diff --git a/llm_server/pre_fork.py b/llm_server/pre_fork.py index 900210c..6e8c1ad 100644 --- a/llm_server/pre_fork.py +++ b/llm_server/pre_fork.py @@ -1,21 +1,9 @@ import sys -from redis import Redis - from llm_server.custom_redis import redis -from llm_server.routes.v1.generate_stats import generate_stats def server_startup(s): if not redis.get('daemon_started', dtype=bool): print('Could not find the key daemon_started in Redis. Did you forget to start the daemon process?') sys.exit(1) - - # Flush the RedisPriorityQueue database. - queue_redis = Redis(host='localhost', port=6379, db=15) - for key in queue_redis.scan_iter('*'): - queue_redis.delete(key) - - # Cache the initial stats - print('Loading backend stats...') - generate_stats() diff --git a/llm_server/routes/helpers/client.py b/llm_server/routes/helpers/client.py index a914362..040a129 100644 --- a/llm_server/routes/helpers/client.py +++ b/llm_server/routes/helpers/client.py @@ -2,13 +2,14 @@ from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis -def format_sillytavern_err(msg: str, backend_url: str, level: str = 'info'): - cluster_backend_hash = cluster_config.get_backend_handler(backend_url)['hash'] +def format_sillytavern_err(msg: str, backend_url: str = 'none', error_type: str = 'info'): + cluster_backend_hash = cluster_config.get_backend(backend_url)['hash'] http_host = redis.get('http_host', dtype=str) return f"""``` === MESSAGE FROM LLM MIDDLEWARE AT {http_host} === --> {level.upper()} <- +-> {error_type.upper()} <- {msg} - -BACKEND HASH: {cluster_backend_hash} +``` +``` +BACKEND: {cluster_backend_hash} ```""" diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 8e0036c..a272960 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -31,7 +31,7 @@ class OobaRequestHandler(RequestHandler): msg = f'Ratelimited: you are only allowed to have {opts.simultaneous_requests_per_ip} simultaneous requests at a time. Please complete your other requests before sending another.' backend_response = self.handle_error(msg) if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.cluster_backend, is_error=True) + log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) return backend_response[0], 200 # We only return the response from handle_error(), not the error code def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: @@ -40,7 +40,7 @@ class OobaRequestHandler(RequestHandler): # TODO: how to format this response_msg = error_msg else: - response_msg = format_sillytavern_err(error_msg, error_type, self.cluster_backend) + response_msg = format_sillytavern_err(error_msg, error_type=error_type, backend_url=self.backend_url) return jsonify({ 'results': [{'text': response_msg}] diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 09ed06c..f058298 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -6,7 +6,7 @@ from uuid import uuid4 from redis import Redis from llm_server import opts -from llm_server.custom_redis import redis +from llm_server.custom_redis import RedisCustom, redis def increment_ip_count(client_ip: str, redis_key): @@ -20,12 +20,12 @@ def decrement_ip_count(client_ip: str, redis_key): class RedisPriorityQueue: - def __init__(self): - self.redis = Redis(host='localhost', port=6379, db=15) + def __init__(self, name: str = 'priority_queue', db: int = 12): + self.redis = RedisCustom(name, db=db) self.pubsub = self.redis.pubsub() self.pubsub.subscribe('events') - def put(self, item, priority): + def put(self, item, priority, selected_model): event = DataEvent() # Check if the IP is already in the dictionary and if it has reached the limit @@ -36,7 +36,7 @@ class RedisPriorityQueue: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request - self.redis.zadd('queue', {json.dumps((item, event.event_id)): -priority}) + self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event @@ -61,12 +61,23 @@ class RedisPriorityQueue: def __len__(self): return self.redis.zcard('queue') + def len(self, model_name): + count = 0 + for key in self.redis.zrange('queue', 0, -1): + item = json.loads(key) + if item[2] == model_name: + count += 1 + return count + def get_queued_ip_count(self, client_ip: str): q = self.redis.hget('queued_ip_count', client_ip) if not q: return 0 return 0 + def flush(self): + self.redis.flush() + class DataEvent: def __init__(self, event_id=None): @@ -87,12 +98,16 @@ class DataEvent: priority_queue = RedisPriorityQueue() -def incr_active_workers(): - redis.incr('active_gen_workers') +def incr_active_workers(selected_model: str, backend_url: str): + redis.incr(f'active_gen_workers:{selected_model}') + redis.incr(f'active_gen_workers:{backend_url}') -def decr_active_workers(): - redis.decr('active_gen_workers') - new_count = redis.get('active_gen_workers', 0, dtype=int) - if new_count < 0: - redis.set('active_gen_workers', 0) +def decr_active_workers(selected_model: str, backend_url: str): + redis.decr(f'active_gen_workers:{selected_model}') + if redis.get(f'active_gen_workers:{selected_model}', 0, dtype=int) < 0: + redis.set(f'active_gen_workers:{selected_model}', 0) + + redis.decr(f'active_gen_workers:{backend_url}') + if redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) < 0: + redis.set(f'active_gen_workers:{backend_url}', 0) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index ecae085..83f510a 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -15,13 +15,13 @@ from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.auth import parse_token from llm_server.routes.helpers.http import require_api_key, validate_json -from llm_server.routes.queue import priority_queue +from llm_server.routes.queue import RedisPriorityQueue, priority_queue DEFAULT_PRIORITY = 9999 class RequestHandler: - def __init__(self, incoming_request: flask.Request, incoming_json: Union[dict, str] = None): + def __init__(self, incoming_request: flask.Request, selected_model: str, incoming_json: Union[dict, str] = None): self.request = incoming_request self.enable_backend_blind_rrd = request.headers.get('LLM-Blind-RRD', False) == 'true' @@ -37,11 +37,12 @@ class RequestHandler: self.client_ip = self.get_client_ip() self.token = self.get_auth_token() self.token_priority, self.token_simultaneous_ip = self.get_token_ratelimit() - self.cluster_backend = get_a_cluster_backend() - self.cluster_backend_info = cluster_config.get_backend(self.cluster_backend) - self.backend = get_backend_handler(self.cluster_backend) + self.backend_url = get_a_cluster_backend(selected_model) + self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None self.used = False + self.selected_model = selected_model redis.zadd('recent_prompters', {self.client_ip: time.time()}) def get_auth_token(self): @@ -123,7 +124,7 @@ class RequestHandler: backend_response = self.handle_error(combined_error_message, 'Validation Error') if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), 0, self.parameters, dict(self.request.headers), 0, self.request.url, self.cluster_backend, is_error=True) + log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), 0, self.parameters, dict(self.request.headers), 0, self.request.url, self.backend_url, is_error=True) return False, backend_response return True, (None, 0) @@ -135,14 +136,16 @@ class RequestHandler: request_valid, invalid_response = self.validate_request(prompt, do_log=True) if not request_valid: return (False, None, None, 0), invalid_response - event = priority_queue.put((llm_request, self.client_ip, self.token, self.parameters, self.cluster_backend), self.token_priority) + event = priority_queue.put((llm_request, self.client_ip, self.token, self.parameters, self.backend_url), self.token_priority, self.selected_model) else: event = None if not event: return (False, None, None, 0), self.handle_ratelimited() + # TODO: add wait timeout success, response, error_msg = event.wait() + end_time = time.time() elapsed_time = end_time - self.start_time @@ -164,7 +167,7 @@ class RequestHandler: else: error_msg = error_msg.strip('.') + '.' backend_response = self.handle_error(error_msg) - log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.cluster_backend, is_error=True) + log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.backend_url, is_error=True) return (False, None, None, 0), backend_response # =============================================== @@ -184,7 +187,7 @@ class RequestHandler: if return_json_err: error_msg = 'The backend did not return valid JSON.' backend_response = self.handle_error(error_msg) - log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), elapsed_time, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.cluster_backend, is_error=True) + log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), elapsed_time, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.backend_url, is_error=True) return (False, None, None, 0), backend_response # =============================================== @@ -218,11 +221,11 @@ class RequestHandler: raise NotImplementedError -def get_backend_handler(mode): +def get_backend_handler(mode, backend_url: str): if mode == 'oobabooga': - return OobaboogaBackend() + return OobaboogaBackend(backend_url) elif mode == 'vllm': - return VLLMBackend() + return VLLMBackend(backend_url) else: raise Exception diff --git a/llm_server/routes/stats.py b/llm_server/routes/stats.py index b4dea54..9e1f291 100644 --- a/llm_server/routes/stats.py +++ b/llm_server/routes/stats.py @@ -1,6 +1,7 @@ from datetime import datetime from llm_server.custom_redis import redis +from llm_server.helpers import round_up_base server_start_time = datetime.now() @@ -14,10 +15,32 @@ def get_total_proompts(): return count -def get_active_gen_workers(): - active_gen_workers = redis.get('active_gen_workers') +def get_active_gen_workers(selected_model: str = None, ): + active_gen_workers = redis.get(f'active_gen_workers:{selected_model}') if active_gen_workers is None: count = 0 else: count = int(active_gen_workers) return count + + +def calculate_wait_time(gen_time_calc, proompters_in_queue, concurrent_gens, active_gen_workers): + if active_gen_workers < concurrent_gens: + return 0 + elif active_gen_workers >= concurrent_gens: + # Calculate how long it will take to complete the currently running gens and the queued requests. + # If the proompters in the queue are equal to the number of workers, just use the calculated generation time. + # Otherwise, use how many requests we can process concurrently times the calculated generation time. Then, round + # that number up to the nearest base gen_time_calc (ie. if gen_time_calc is 8 and the calculated number is 11.6, we will get 18). Finally, + # Add gen_time_calc to the time to account for the currently running generations. + # This assumes that all active workers will finish at the same time, which is unlikely. + # Regardless, this is the most accurate estimate we can get without tracking worker elapsed times. + proompters_in_queue_wait_time = gen_time_calc if (proompters_in_queue / concurrent_gens) <= 1 \ + else round_up_base(((proompters_in_queue / concurrent_gens) * gen_time_calc), base=gen_time_calc) + return proompters_in_queue_wait_time + gen_time_calc if active_gen_workers > 0 else 0 + elif proompters_in_queue == 0 and active_gen_workers == 0: + # No queue, no workers + return 0 + else: + # No queue + return gen_time_calc diff --git a/llm_server/routes/v1/generate.py b/llm_server/routes/v1/generate.py index 715288f..39db078 100644 --- a/llm_server/routes/v1/generate.py +++ b/llm_server/routes/v1/generate.py @@ -3,18 +3,20 @@ import traceback from flask import jsonify, request from . import bp -from ..helpers.client import format_sillytavern_err from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler +from ...cluster.backend import get_a_cluster_backend +from ...cluster.cluster_config import cluster_config -@bp.route('/generate', methods=['POST']) -def generate(): +@bp.route('/v1/generate', methods=['POST']) +@bp.route('//v1/generate', methods=['POST']) +def generate(model_name=None): request_valid_json, request_json_body = validate_json(request) if not request_valid_json or not request_json_body.get('prompt'): return jsonify({'code': 400, 'msg': 'Invalid JSON'}), 400 else: - handler = OobaRequestHandler(request) + handler = OobaRequestHandler(request, model_name) try: return handler.handle_request() except Exception: diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index 66dd316..30e0967 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -2,74 +2,32 @@ import time from datetime import datetime from llm_server import opts -from llm_server.cluster.backend import get_a_cluster_backend, test_backend +from llm_server.cluster.backend import get_a_cluster_backend from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.model_choices import get_model_choices from llm_server.custom_redis import redis from llm_server.database.database import get_distinct_ips_24h, sum_column -from llm_server.helpers import deep_sort, round_up_base -from llm_server.llm.info import get_running_model -from llm_server.routes.queue import priority_queue -from llm_server.routes.stats import get_active_gen_workers, get_total_proompts, server_start_time - - -def calculate_wait_time(gen_time_calc, proompters_in_queue, concurrent_gens, active_gen_workers): - if active_gen_workers < concurrent_gens: - return 0 - elif active_gen_workers >= concurrent_gens: - # Calculate how long it will take to complete the currently running gens and the queued requests. - # If the proompters in the queue are equal to the number of workers, just use the calculated generation time. - # Otherwise, use how many requests we can process concurrently times the calculated generation time. Then, round - # that number up to the nearest base gen_time_calc (ie. if gen_time_calc is 8 and the calculated number is 11.6, we will get 18). Finally, - # Add gen_time_calc to the time to account for the currently running generations. - # This assumes that all active workers will finish at the same time, which is unlikely. - # Regardless, this is the most accurate estimate we can get without tracking worker elapsed times. - proompters_in_queue_wait_time = gen_time_calc if (proompters_in_queue / concurrent_gens) <= 1 \ - else round_up_base(((proompters_in_queue / concurrent_gens) * gen_time_calc), base=gen_time_calc) - return proompters_in_queue_wait_time + gen_time_calc if active_gen_workers > 0 else 0 - elif proompters_in_queue == 0 and active_gen_workers == 0: - # No queue, no workers - return 0 - else: - # No queue - return gen_time_calc +from llm_server.helpers import deep_sort +from llm_server.routes.stats import get_total_proompts, server_start_time def generate_stats(regen: bool = False): if not regen: - c = redis.get('proxy_stats', dtype=dict) + c = redis.getp('proxy_stats') if c: return c default_backend_url = get_a_cluster_backend() default_backend_info = cluster_config.get_backend(default_backend_url) if not default_backend_info.get('mode'): - # TODO: remove - print('DAEMON NOT FINISHED STARTING') return base_client_api = redis.get('base_client_api', dtype=str) proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf')) - average_generation_elapsed_sec = redis.get('average_generation_elapsed_sec', 0) - - online = test_backend(default_backend_url, default_backend_info['mode']) - if online: - running_model, err = get_running_model(default_backend_url, default_backend_info['mode']) - cluster_config.set_backend_value(default_backend_url, 'running_model', running_model) - else: - running_model = None - - active_gen_workers = get_active_gen_workers() - proompters_in_queue = len(priority_queue) - - # This is so wildly inaccurate it's disabled. - # estimated_avg_tps = redis.get('estimated_avg_tps', float, default=0) - - # TODO: make this for the currently selected backend - estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, opts.concurrent_gens, active_gen_workers) output = { 'default': { - 'model': running_model, - 'backend': default_backend_info['hash'], + 'model': default_backend_info['model'], + 'backend': default_backend_url, }, 'stats': { 'proompters': { @@ -78,21 +36,14 @@ def generate_stats(regen: bool = False): }, 'proompts_total': get_total_proompts() if opts.show_num_prompts else None, 'uptime': int((datetime.now() - server_start_time).total_seconds()) if opts.show_uptime else None, - 'average_generation_elapsed_sec': int(average_generation_elapsed_sec), # 'estimated_avg_tps': estimated_avg_tps, 'tokens_generated': sum_column('prompts', 'response_tokens') if opts.show_total_output_tokens else None, 'num_backends': len(cluster_config.all()) if opts.show_backends else None, }, - 'online': online, 'endpoints': { 'blocking': f'https://{base_client_api}', 'streaming': f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else None, }, - 'queue': { - 'processing': active_gen_workers, - 'queued': proompters_in_queue, - 'estimated_wait_sec': int(estimated_wait_sec), - }, 'timestamp': int(time.time()), 'config': { 'gatekeeper': 'none' if opts.auth_required is False else 'token', @@ -106,42 +57,30 @@ def generate_stats(regen: bool = False): 'backend_info': redis.get_dict('backend_info') if opts.show_backend_info else None, } + # TODO: have get_model_choices() return all the info so we don't have to loop over the backends ourself + if opts.show_backends: for backend_url, v in cluster_config.all().items(): backend_info = cluster_config.get_backend(backend_url) if not backend_info['online']: continue - - # TODO: have this fetch the data from VLLM which will display GPU utalization - # if opts.netdata_root: - # netdata_stats = {} - # power_states = get_power_states() - # for gpu, power_state in power_states.items(): - # netdata_stats[gpu] = { - # 'power_state': power_state, - # # 'wh_wasted_1_hr': get_gpu_wh(int(gpu.strip('gpu'))) - # } - # else: - # netdata_stats = {} - netdata_stats = {} - - # TODO: use value returned by VLLM backend here - # backend_uptime = int((datetime.now() - backend_info['start_time']).total_seconds()) if opts.show_uptime else None - backend_uptime = -1 - + backend_uptime = int((datetime.now() - datetime.fromtimestamp(backend_info['startup_time'])).total_seconds()) if opts.show_uptime else None output['backend_info'][backend_info['hash']] = { 'uptime': backend_uptime, - # 'context_size': opts.context_size, - 'model': opts.manual_model_name if opts.manual_model_name else backend_info.get('running_model', 'ERROR'), + 'max_tokens': backend_info['model_config']['max_position_embeddings'], + 'model': backend_info['model'], 'mode': backend_info['mode'], - 'nvidia': netdata_stats + 'nvidia': backend_info['nvidia'], } else: output['backend_info'] = {} + output['default'] = get_model_choices(regen=True)[1] + result = deep_sort(output) # It may take a bit to get the base client API, so don't cache until then. if base_client_api: - redis.set_dict('proxy_stats', result) # Cache with no expiry + redis.setp('proxy_stats', result) + return result diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v1/info.py index 90778e5..355b415 100644 --- a/llm_server/routes/v1/info.py +++ b/llm_server/routes/v1/info.py @@ -10,13 +10,9 @@ from ...cluster.backend import get_a_cluster_backend, get_backends, get_backends from ...cluster.cluster_config import cluster_config -@bp.route('/model', methods=['GET']) -@bp.route('//model', methods=['GET']) +@bp.route('/v1/model', methods=['GET']) +@bp.route('//v1/model', methods=['GET']) def get_model(model_name=None): - if not model_name: - b = get_a_cluster_backend() - model_name = cluster_config.get_backend(b)['running_model'] - # We will manage caching ourself since we don't want to cache # when the backend is down. Also, Cloudflare won't cache 500 errors. cache_key = 'model_cache::' + request.url @@ -25,6 +21,9 @@ def get_model(model_name=None): if cached_response: return cached_response + if not model_name: + model_name = cluster_config.get_backend(get_a_cluster_backend()).get('model') + if not is_valid_model(model_name): response = jsonify({ 'code': 400, @@ -32,7 +31,6 @@ def get_model(model_name=None): }), 400 else: num_backends = len(get_backends_from_model(model_name)) - response = jsonify({ 'result': opts.manual_model_name if opts.manual_model_name else model_name, 'model_backend_count': num_backends, @@ -47,7 +45,8 @@ def get_model(model_name=None): @requires_auth def get_backend(): online, offline = get_backends() - result = [] + result = {} for i in online + offline: - result.append(cluster_config.get_backend(i)) + info = cluster_config.get_backend(i) + result[info['hash']] = info return jsonify(result), 200 diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 626e34b..e92052e 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,7 +1,7 @@ import threading import time -from llm_server import opts +from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.llm.generator import generator from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, priority_queue @@ -9,12 +9,16 @@ from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip def worker(): while True: - need_to_wait() - (request_json_body, client_ip, token, parameters, cluster_backend), event_id = priority_queue.get() - need_to_wait() + (request_json_body, client_ip, token, parameters, backend_url), event_id, selected_model = priority_queue.get() + if not selected_model: + selected_model = cluster_config.get_backend(backend_url)['model'] + + # This wait time is "invisible", meaning the worker may as + # well be still waiting to get an item from the queue. + need_to_wait(backend_url) increment_ip_count(client_ip, 'processing_ips') - incr_active_workers() + incr_active_workers(selected_model, backend_url) if not request_json_body: # This was a dummy request from the websocket handler. @@ -22,12 +26,12 @@ def worker(): continue try: - success, response, error_msg = generator(request_json_body, cluster_backend) + success, response, error_msg = generator(request_json_body, backend_url) event = DataEvent(event_id) event.set((success, response, error_msg)) finally: decrement_ip_count(client_ip, 'processing_ips') - decr_active_workers() + decr_active_workers(selected_model, backend_url) def start_workers(num_workers: int): @@ -40,11 +44,12 @@ def start_workers(num_workers: int): print(f'Started {i} inference workers.') -def need_to_wait(): +def need_to_wait(backend_url: str): # We need to check the number of active workers since the streaming endpoint may be doing something. - active_workers = redis.get('active_gen_workers', 0, dtype=int) + active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) + concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) s = time.time() - while active_workers >= opts.concurrent_gens: + while active_workers >= concurrent_gens: time.sleep(0.01) e = time.time() if e - s > 0.5: diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py index 447046f..ca82d60 100644 --- a/llm_server/workers/mainer.py +++ b/llm_server/workers/mainer.py @@ -5,7 +5,7 @@ from llm_server.cluster.backend import get_a_cluster_backend, get_backends from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.database.database import weighted_average_column_for_model -from llm_server.llm.info import get_running_model +from llm_server.llm.info import get_info, get_running_model def main_background_thread(): @@ -14,8 +14,9 @@ def main_background_thread(): for backend_url in online: backend_info = cluster_config.get_backend(backend_url) backend_mode = backend_info['mode'] - running_model, err = get_running_model(backend_url, backend_mode) - if err: + backend_info = get_info(backend_url, backend_mode) + running_model = backend_info.get('model') + if not running_model: continue average_generation_elapsed_sec, average_output_tokens, estimated_avg_tps = calc_stats_for_backend(backend_url, running_model, backend_mode) @@ -25,21 +26,6 @@ def main_background_thread(): cluster_config.set_backend_value(backend_url, 'average_output_tokens', average_output_tokens) if average_generation_elapsed_sec and average_output_tokens: cluster_config.set_backend_value(backend_url, 'estimated_avg_tps', estimated_avg_tps) - - default_backend_url = get_a_cluster_backend() - default_backend_info = cluster_config.get_backend(default_backend_url) - default_backend_mode = default_backend_info['mode'] - default_running_model, err = get_running_model(default_backend_url, default_backend_mode) - if err: - continue - - default_average_generation_elapsed_sec, default_average_output_tokens, default_estimated_avg_tps = calc_stats_for_backend(default_running_model, default_running_model, default_backend_mode) - if default_average_generation_elapsed_sec: - redis.set('average_generation_elapsed_sec', default_average_generation_elapsed_sec) - if default_average_output_tokens: - redis.set('average_output_tokens', default_average_output_tokens) - if default_average_generation_elapsed_sec and default_average_output_tokens: - redis.set('estimated_avg_tps', default_estimated_avg_tps) time.sleep(30) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index 6a33835..ed6ff65 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -1,6 +1,7 @@ import logging import time +from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue @@ -17,9 +18,11 @@ if not logger.handlers: def console_printer(): time.sleep(3) while True: - processing = redis.hkeys('processing_ips') + processing = redis.keys('active_gen_workers:http*') # backends always start with http processing_count = 0 - for ip in processing: - processing_count += int(redis.hget('processing_ips', ip)) - logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)}') + if len(processing): + for k in processing: + processing_count += redis.get(k, default=0, dtype=int) + backends = [k for k, v in cluster_config.all().items() if v['online']] + logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') time.sleep(10) diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index 83bac2d..0c82559 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -15,11 +15,11 @@ from llm_server.workers.recenter import recent_prompters_thread def cache_stats(): while True: generate_stats(regen=True) - time.sleep(1) + time.sleep(5) def start_background(): - start_workers(opts.concurrent_gens) + start_workers(opts.cluster_workers) t = Thread(target=main_background_thread) t.daemon = True diff --git a/other/vllm/vllm_api_server.py b/other/vllm/vllm_api_server.py old mode 100755 new mode 100644 diff --git a/requirements.txt b/requirements.txt index 7b49eed..6057884 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,17 @@ flask~=2.3.3 -flask_cors pyyaml~=6.0.1 flask_caching requests~=2.31.0 tiktoken~=0.5.0 -gunicorn gevent~=23.9.0.post1 -async-timeout -flask-sock -uvicorn~=0.23.2 -fastapi~=0.103.1 torch~=2.0.1 PyMySQL~=1.1.0 -DBUtils~=3.0.3 simplejson~=3.19.1 websockets~=11.0.3 basicauth~=1.0.0 openai~=0.28.0 urllib3~=2.0.4 +flask-sock==0.6.0 +gunicorn==21.2.0 +redis==5.0.1 +git+https://github.com/vllm-project/vllm \ No newline at end of file diff --git a/server.py b/server.py index 0214b49..699290f 100644 --- a/server.py +++ b/server.py @@ -1,5 +1,3 @@ -from llm_server.cluster.cluster_config import cluster_config - try: import gevent.monkey @@ -14,10 +12,10 @@ from pathlib import Path import simplejson as json from flask import Flask, jsonify, render_template, request -from llm_server.cluster.backend import get_a_cluster_backend, get_backends -from llm_server.cluster.redis_cycle import load_backend_cycle +from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.model_choices import get_model_choices from llm_server.config.config import mode_ui_names -from llm_server.config.load import load_config, parse_backends +from llm_server.config.load import load_config from llm_server.database.conn import database from llm_server.database.create import create_db from llm_server.pre_fork import server_startup @@ -26,10 +24,7 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio -# TODO: set VLLM to stream ALL data using socket.io. If the socket disconnects, cancel generation. -# TODO: add backend fallbacks. Backends at the bottom of the list are higher priority and are fallbacks if the upper ones fail -# TODO: implement background thread to test backends via sending test prompts -# TODO: if backend fails request, mark it as down +# TODO: per-backend workers # TODO: allow setting concurrent gens per-backend # TODO: set the max tokens to that of the lowest backend # TODO: implement RRD backend loadbalancer option @@ -42,6 +37,7 @@ from llm_server.sock import init_socketio # TODO: have VLLM report context size, uptime # Lower priority +# TODO: set VLLM to stream ALL data using socket.io. If the socket disconnects, cancel generation. # TODO: estiamted wait time needs to account for full concurrent_gens but the queue is less than concurrent_gens # TODO: the estiamted wait time lags behind the stats # TODO: simulate OpenAI error messages regardless of endpoint @@ -69,12 +65,11 @@ from llm_server.helpers import auto_set_base_client_api from llm_server.llm.vllm.info import vllm_info from llm_server.custom_redis import flask_cache from llm_server.llm import redis -from llm_server.routes.stats import get_active_gen_workers from llm_server.routes.v1.generate_stats import generate_stats app = Flask(__name__) init_socketio(app) -app.register_blueprint(bp, url_prefix='/api/v1/') +app.register_blueprint(bp, url_prefix='/api/v2/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') flask_cache.init_app(app) flask_cache.clear() @@ -94,37 +89,23 @@ if not success: database.init_db(config['mysql']['host'], config['mysql']['username'], config['mysql']['password'], config['mysql']['database']) create_db() -cluster_config.clear() -cluster_config.load(parse_backends(config)) -on, off = get_backends() -load_backend_cycle('backend_cycler', on + off) - @app.route('/') @app.route('/api') @app.route('/api/openai') @flask_cache.cached(timeout=10) def home(): - # Use the default backend - backend_url = get_a_cluster_backend() - if backend_url: - backend_info = cluster_config.get_backend(backend_url) - stats = generate_stats(backend_url) - else: - backend_info = stats = None + base_client_api = redis.get('base_client_api', dtype=str) + stats = generate_stats() - if not stats['online']: - running_model = estimated_wait_sec = 'offline' - else: - running_model = backend_info['running_model'] + model_choices, default_backend_info = get_model_choices() - active_gen_workers = get_active_gen_workers() - if stats['queue']['queued'] == 0 and active_gen_workers >= opts.concurrent_gens: - # There will be a wait if the queue is empty but prompts are processing, but we don't - # know how long. - estimated_wait_sec = f"less than {stats['stats']['average_generation_elapsed_sec']} seconds" - else: - estimated_wait_sec = f"{stats['queue']['estimated_wait_sec']} seconds" + if default_backend_info['queued'] == 0 and default_backend_info['queued'] >= opts.concurrent_gens: + # There will be a wait if the queue is empty but prompts are processing, but we don't + # know how long. + default_estimated_wait_sec = f"less than {default_backend_info['estimated_wait']} seconds" + else: + default_estimated_wait_sec = f"{default_backend_info['estimated_wait']} seconds" if len(config['analytics_tracking_code']): analytics_tracking_code = f"" @@ -137,39 +118,35 @@ def home(): info_html = '' mode_info = '' - using_vllm = False for k, v in cluster_config.all().items(): - if v['mode'] == vllm: - using_vllm = True + if v['mode'] == 'vllm': + mode_info = vllm_info break - if using_vllm == 'vllm': - mode_info = vllm_info - - base_client_api = redis.get('base_client_api', dtype=str) - return render_template('home.html', llm_middleware_name=opts.llm_middleware_name, analytics_tracking_code=analytics_tracking_code, info_html=info_html, - current_model=opts.manual_model_name if opts.manual_model_name else running_model, - client_api=f'https://{base_client_api}', - ws_client_api=f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else None, - estimated_wait=estimated_wait_sec, + default_model=default_backend_info['model'], + default_active_gen_workers=default_backend_info['processing'], + default_proompters_in_queue=default_backend_info['queued'], + current_model=opts.manual_model_name if opts.manual_model_name else None, # else running_model, + client_api=f'https://{base_client_api}/v2', + ws_client_api=f'wss://{base_client_api}/v2/stream' if opts.enable_streaming else 'disabled', + default_estimated_wait=default_estimated_wait_sec, mode_name=mode_ui_names[opts.mode][0], api_input_textbox=mode_ui_names[opts.mode][1], streaming_input_textbox=mode_ui_names[opts.mode][2], - context_size=opts.context_size, + default_context_size=default_backend_info['context_size'], stats_json=json.dumps(stats, indent=4, ensure_ascii=False), extra_info=mode_info, openai_client_api=f'https://{base_client_api}/openai/v1' if opts.enable_openi_compatible_backend else 'disabled', expose_openai_system_prompt=opts.expose_openai_system_prompt, enable_streaming=opts.enable_streaming, + model_choices=model_choices ) -# TODO: add authenticated route to get the current backend URL. Add it to /v1/backend - @app.route('/') @app.route('//') def fallback(first=None, rest=None): diff --git a/templates/home.html b/templates/home.html index 4b9c153..fb6f3e9 100644 --- a/templates/home.html +++ b/templates/home.html @@ -65,6 +65,10 @@ .hidden { display: none; } + + .header-workers { + font-weight: normal; + } @@ -76,8 +80,12 @@

{{ llm_middleware_name }}

-

Current Model: {{ current_model }}

-

Estimated Wait Time: {{ estimated_wait }}

+

Current Model: {{ default_model }}

+

+ Estimated Wait Time: {{ default_estimated_wait }}
+ Processing: {{ default_active_gen_workers }}
+ Queued: {{ default_proompters_in_queue }} +


Client API URL: {{ client_api }}

Streaming API URL: {{ ws_client_api if enable_streaming else 'Disabled' }}

@@ -101,7 +109,7 @@ API key textbox.
  • Click Connect to test the connection.
    • -
  • Open your preset config and set Context Size to {{ context_size }}.
    • +
  • Open your preset config and set Context Size to {{ default_context_size }}.
  • Follow this guide to get set up: rentry.org/freellamas
    • @@ -119,9 +127,30 @@
    + {% for key, value in model_choices.items() %} +
    +

    {{ key }} - {{ value.backend_count }} workers

    +

    + Estimated Wait Time: {{ value.estimated_wait }}
    + Processing: {{ value.processing }}
    + Queued: {{ value.queued }}
    +

    +

    + Client API URL: {{ value.client_api }}
    + Streaming API URL: {{ value.ws_client_api }}
    + OpenAI-Compatible API URL: {{ value.openai_client_api }} +

    +

    Context Size: {{ value.context_size }}

    +

    Average Generation Time: {{ value.avg_generation_time | int }} seconds

    +
    +
    + {% endfor %} + +
    git.evulid.cc/cyberes/local-llm-server diff --git a/test-cluster.py b/test-cluster.py deleted file mode 100644 index ec1773a..0000000 --- a/test-cluster.py +++ /dev/null @@ -1,39 +0,0 @@ -try: - import gevent.monkey - - gevent.monkey.patch_all() -except ImportError: - pass - -import time -from threading import Thread -from llm_server.cluster.redis_cycle import load_backend_cycle - -from llm_server.cluster.backend import get_backends, get_a_cluster_backend -from llm_server.cluster.worker import cluster_worker -from llm_server.config.load import parse_backends, load_config -from llm_server.cluster.redis_config_cache import RedisClusterStore - -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('config') -args = parser.parse_args() - -success, config, msg = load_config(args.config) - -cluster_config = RedisClusterStore('cluster_config') -cluster_config.clear() -cluster_config.load(parse_backends(config)) -on, off = get_backends() -load_backend_cycle('backend_cycler', on + off) - -t = Thread(target=cluster_worker) -t.daemon = True -t.start() - -while True: - # online, offline = get_backends() - # print(online, offline) - # print(get_a_cluster_backend()) - time.sleep(3) -- 2.34.1 From e0f86d053a70d14afb37760237f5a17119257743 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 19:42:41 -0600 Subject: [PATCH 004/163] reorganize to api v2 --- daemon.py | 2 +- llm_server/routes/{v1 => v2}/__init__.py | 0 llm_server/routes/{v1 => v2}/generate.py | 0 llm_server/routes/{v1 => v2}/generate_stats.py | 0 llm_server/routes/{v1 => v2}/generate_stream.py | 0 llm_server/routes/{v1 => v2}/info.py | 0 llm_server/routes/{v1 => v2}/proxy.py | 0 llm_server/workers/threader.py | 2 +- server.py | 4 ++-- 9 files changed, 4 insertions(+), 4 deletions(-) rename llm_server/routes/{v1 => v2}/__init__.py (100%) rename llm_server/routes/{v1 => v2}/generate.py (100%) rename llm_server/routes/{v1 => v2}/generate_stats.py (100%) rename llm_server/routes/{v1 => v2}/generate_stream.py (100%) rename llm_server/routes/{v1 => v2}/info.py (100%) rename llm_server/routes/{v1 => v2}/proxy.py (100%) diff --git a/daemon.py b/daemon.py index 0fa3601..aac2657 100644 --- a/daemon.py +++ b/daemon.py @@ -10,7 +10,7 @@ from llm_server.config.load import load_config, parse_backends from llm_server.custom_redis import redis from llm_server.database.create import create_db from llm_server.routes.queue import priority_queue -from llm_server.routes.v1.generate_stats import generate_stats +from llm_server.routes.v2.generate_stats import generate_stats from llm_server.workers.threader import start_background script_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v2/__init__.py similarity index 100% rename from llm_server/routes/v1/__init__.py rename to llm_server/routes/v2/__init__.py diff --git a/llm_server/routes/v1/generate.py b/llm_server/routes/v2/generate.py similarity index 100% rename from llm_server/routes/v1/generate.py rename to llm_server/routes/v2/generate.py diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v2/generate_stats.py similarity index 100% rename from llm_server/routes/v1/generate_stats.py rename to llm_server/routes/v2/generate_stats.py diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v2/generate_stream.py similarity index 100% rename from llm_server/routes/v1/generate_stream.py rename to llm_server/routes/v2/generate_stream.py diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v2/info.py similarity index 100% rename from llm_server/routes/v1/info.py rename to llm_server/routes/v2/info.py diff --git a/llm_server/routes/v1/proxy.py b/llm_server/routes/v2/proxy.py similarity index 100% rename from llm_server/routes/v1/proxy.py rename to llm_server/routes/v2/proxy.py diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index 0c82559..89a6770 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -4,7 +4,7 @@ from threading import Thread from llm_server import opts from llm_server.cluster.stores import redis_running_models from llm_server.cluster.worker import cluster_worker -from llm_server.routes.v1.generate_stats import generate_stats +from llm_server.routes.v2.generate_stats import generate_stats from llm_server.workers.inferencer import start_workers from llm_server.workers.mainer import main_background_thread from llm_server.workers.moderator import start_moderation_workers diff --git a/server.py b/server.py index 699290f..8c287be 100644 --- a/server.py +++ b/server.py @@ -21,7 +21,7 @@ from llm_server.database.create import create_db from llm_server.pre_fork import server_startup from llm_server.routes.openai import openai_bp from llm_server.routes.server_error import handle_server_error -from llm_server.routes.v1 import bp +from llm_server.routes.v2 import bp from llm_server.sock import init_socketio # TODO: per-backend workers @@ -65,7 +65,7 @@ from llm_server.helpers import auto_set_base_client_api from llm_server.llm.vllm.info import vllm_info from llm_server.custom_redis import flask_cache from llm_server.llm import redis -from llm_server.routes.v1.generate_stats import generate_stats +from llm_server.routes.v2.generate_stats import generate_stats app = Flask(__name__) init_socketio(app) -- 2.34.1 From e6267a7d467f363176d4e47fb63a0c39eaad4a71 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 19:45:04 -0600 Subject: [PATCH 005/163] remove vllm from requirements.txt --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 6057884..df16bea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,5 +13,4 @@ openai~=0.28.0 urllib3~=2.0.4 flask-sock==0.6.0 gunicorn==21.2.0 -redis==5.0.1 -git+https://github.com/vllm-project/vllm \ No newline at end of file +redis==5.0.1 \ No newline at end of file -- 2.34.1 From 11a10f85c1b30e3b4b6a1efa26f3df36475db5aa Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 19:59:30 -0600 Subject: [PATCH 006/163] adjust home page --- templates/home.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/templates/home.html b/templates/home.html index fb6f3e9..e447e90 100644 --- a/templates/home.html +++ b/templates/home.html @@ -69,6 +69,10 @@ .header-workers { font-weight: normal; } + + h3 { + font-size: 16pt; + } @@ -99,8 +103,8 @@
    -
    - Instructions: +

    Instructions

    +
    1. Set your API type to {{ mode_name }}
    2. Enter {{ client_api }} in the {{ api_input_textbox }} textbox.
      3. -- 2.34.1 From e553fa6e9fda6d437a98d2f50cbb06da1d5482fb Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 20:01:36 -0600 Subject: [PATCH 007/163] adjust home page fontsize --- templates/home.html | 1 + 1 file changed, 1 insertion(+) diff --git a/templates/home.html b/templates/home.html index e447e90..03f58c2 100644 --- a/templates/home.html +++ b/templates/home.html @@ -68,6 +68,7 @@ .header-workers { font-weight: normal; + font-size: 14pt; } h3 { -- 2.34.1 From 91ba2fad1ba4030bc5dccabe07d87c72266cf97a Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 20:11:14 -0600 Subject: [PATCH 008/163] add proompter stats back in --- server.py | 4 +++- templates/home.html | 20 ++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/server.py b/server.py index 8c287be..4a896af 100644 --- a/server.py +++ b/server.py @@ -143,7 +143,9 @@ def home(): openai_client_api=f'https://{base_client_api}/openai/v1' if opts.enable_openi_compatible_backend else 'disabled', expose_openai_system_prompt=opts.expose_openai_system_prompt, enable_streaming=opts.enable_streaming, - model_choices=model_choices + model_choices=model_choices, + proompters_5_min=stats['stats']['proompters']['5_min'], + proompters_24_hrs=stats['stats']['proompters']['24_hrs'], ) diff --git a/templates/home.html b/templates/home.html index 03f58c2..c366263 100644 --- a/templates/home.html +++ b/templates/home.html @@ -74,6 +74,10 @@ h3 { font-size: 16pt; } + + .no-marker { + list-style: none; + } @@ -109,7 +113,9 @@
      1. Set your API type to {{ mode_name }}
      2. Enter {{ client_api }} in the {{ api_input_textbox }} textbox.
        3. - {% if enable_streaming %}
      3. Enter {{ ws_client_api }} in the {{ streaming_input_textbox }} textbox.
        4. {% endif %} + {% if enable_streaming %} +
      4. Enter {{ ws_client_api }} in the {{ streaming_input_textbox }} textbox.
        5. + {% endif %}
      5. If you have a token, check the Mancer AI checkbox and enter your token in the Mancer API key textbox.
        6. @@ -132,6 +138,16 @@
        +
        +

        Statistics

        + Proompters: +
          +
        • 5 minutes: {{ proompters_5_min }}
          • +
        • 24 hours: {{ proompters_24_hrs }}
          • +
        +
        +
        + {% for key, value in model_choices.items() %}

        {{ key }} - {{ value.backend_count }} workers

        @@ -146,7 +162,7 @@ OpenAI-Compatible API URL: {{ value.openai_client_api }}

        Context Size: {{ value.context_size }}

        -

        Average Generation Time: {{ value.avg_generation_time | int }} seconds

        +

        Average Generation Time: {{ value.avg_generation_time | int }}


        {% endfor %} -- 2.34.1 From 1151bb547574325dc658e41c4e090e1e3ce1cb52 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 20:42:48 -0600 Subject: [PATCH 009/163] adjust stats --- llm_server/cluster/model_choices.py | 41 +++++++++++++++------------- llm_server/routes/request_handler.py | 4 +++ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index c9a94fd..ec78e2f 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -54,7 +54,7 @@ def get_model_choices(regen: bool = False): 'estimated_wait': estimated_wait_sec, 'queued': proompters_in_queue, 'processing': active_gen_workers, - 'avg_generation_time': average_generation_elapsed_sec + 'avg_generation_time': average_generation_elapsed_sec, } if len(context_size): @@ -63,25 +63,28 @@ def get_model_choices(regen: bool = False): model_choices = dict(sorted(model_choices.items())) default_backend = get_a_cluster_backend() - default_backend_info = cluster_config.get_backend(default_backend) - default_context_size = default_backend_info['model_config']['max_position_embeddings'] - default_average_generation_elapsed_sec = default_backend_info.get('average_generation_elapsed_sec') - default_active_gen_workers = redis.get(f'active_gen_workers:{default_backend}', dtype=int, default=0) - default_proompters_in_queue = priority_queue.len(default_backend_info['model']) - default_estimated_wait_sec = calculate_wait_time(default_average_generation_elapsed_sec, default_proompters_in_queue, default_backend_info['concurrent_gens'], default_active_gen_workers) + default_backend_dict = {} + if default_backend: + default_backend_info = cluster_config.get_backend(default_backend) + default_context_size = default_backend_info['model_config']['max_position_embeddings'] + default_average_generation_elapsed_sec = default_backend_info.get('average_generation_elapsed_sec') + default_active_gen_workers = redis.get(f'active_gen_workers:{default_backend}', dtype=int, default=0) + default_proompters_in_queue = priority_queue.len(default_backend_info['model']) + default_estimated_wait_sec = calculate_wait_time(default_average_generation_elapsed_sec, default_proompters_in_queue, default_backend_info['concurrent_gens'], default_active_gen_workers) - default_backend_dict = { - 'client_api': f'https://{base_client_api}/v2', - 'ws_client_api': f'wss://{base_client_api}/v2' if opts.enable_streaming else None, - 'openai_client_api': f'https://{base_client_api}/openai/v2' if opts.enable_openi_compatible_backend else 'disabled', - 'estimated_wait': default_estimated_wait_sec, - 'queued': default_proompters_in_queue, - 'processing': default_active_gen_workers, - 'context_size': default_context_size, - 'hash': default_backend_info['hash'], - 'model': default_backend_info['model'], - 'avg_generation_time': default_average_generation_elapsed_sec - } + default_backend_dict = { + 'client_api': f'https://{base_client_api}/v2', + 'ws_client_api': f'wss://{base_client_api}/v2' if opts.enable_streaming else None, + 'openai_client_api': f'https://{base_client_api}/openai/v2' if opts.enable_openi_compatible_backend else 'disabled', + 'estimated_wait': default_estimated_wait_sec, + 'queued': default_proompters_in_queue, + 'processing': default_active_gen_workers, + 'context_size': default_context_size, + 'hash': default_backend_info['hash'], + 'model': default_backend_info['model'], + 'avg_generation_time': default_average_generation_elapsed_sec, + 'online': True + } redis.setp('model_choices', (model_choices, default_backend_dict)) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 83f510a..0dd862a 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -39,6 +39,10 @@ class RequestHandler: self.token_priority, self.token_simultaneous_ip = self.get_token_ratelimit() self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + + if not self.cluster_backend_info.get('mode'): + print(self.backend_url, self.cluster_backend_info) + self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None self.used = False -- 2.34.1 From 166b2316e803f08ce12ab5f6d2cd639ce61a1cea Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 20:59:24 -0600 Subject: [PATCH 010/163] depricate v1 --- llm_server/routes/v1/__init__.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 llm_server/routes/v1/__init__.py diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py new file mode 100644 index 0000000..2e02d4c --- /dev/null +++ b/llm_server/routes/v1/__init__.py @@ -0,0 +1,17 @@ +from flask import Blueprint, jsonify + +from llm_server.custom_redis import redis +from llm_server.routes.helpers.client import format_sillytavern_err + +bp = Blueprint('old_v1', __name__) + + +@bp.route('/') +@bp.route('//') +def fallback(first=None, rest=None): + base_client_api = redis.get('base_client_api', dtype=str) + error_msg = f'The /v1/ endpoint has been depreciated. Please see [{base_client_api}]({base_client_api}) for the new endpoint.' + response_msg = format_sillytavern_err(error_msg, error_type='API') + return jsonify({ + 'results': [{'text': response_msg}] + }), 200 # return 200 so we don't trigger an error message in the client's ST -- 2.34.1 From 592eb08cb196b288c707a319c5ea1dc9ce430d75 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 21:07:12 -0600 Subject: [PATCH 011/163] add message for /v1/ --- llm_server/routes/helpers/client.py | 7 +++++-- llm_server/routes/v1/__init__.py | 13 +++++++------ llm_server/routes/v2/__init__.py | 2 +- server.py | 3 +++ 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/llm_server/routes/helpers/client.py b/llm_server/routes/helpers/client.py index 040a129..5031b8b 100644 --- a/llm_server/routes/helpers/client.py +++ b/llm_server/routes/helpers/client.py @@ -2,8 +2,11 @@ from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis -def format_sillytavern_err(msg: str, backend_url: str = 'none', error_type: str = 'info'): - cluster_backend_hash = cluster_config.get_backend(backend_url)['hash'] +def format_sillytavern_err(msg: str, backend_url: str = None, error_type: str = 'info'): + if backend_url: + cluster_backend_hash = cluster_config.get_backend(backend_url)['hash'] + else: + cluster_backend_hash = 'none' http_host = redis.get('http_host', dtype=str) return f"""``` === MESSAGE FROM LLM MIDDLEWARE AT {http_host} === diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py index 2e02d4c..c812f85 100644 --- a/llm_server/routes/v1/__init__.py +++ b/llm_server/routes/v1/__init__.py @@ -3,15 +3,16 @@ from flask import Blueprint, jsonify from llm_server.custom_redis import redis from llm_server.routes.helpers.client import format_sillytavern_err -bp = Blueprint('old_v1', __name__) +old_v1_bp = Blueprint('v1', __name__) -@bp.route('/') -@bp.route('//') -def fallback(first=None, rest=None): +@old_v1_bp.route('/', defaults={'path': ''}, methods=['GET', 'POST']) +@old_v1_bp.route('/', methods=['GET', 'POST']) +def fallback(path): base_client_api = redis.get('base_client_api', dtype=str) - error_msg = f'The /v1/ endpoint has been depreciated. Please see [{base_client_api}]({base_client_api}) for the new endpoint.' + error_msg = f'The /v1/ endpoint has been depreciated. Please visit {base_client_api} for more information.' response_msg = format_sillytavern_err(error_msg, error_type='API') return jsonify({ - 'results': [{'text': response_msg}] + 'results': [{'text': response_msg}], + 'result': base_client_api }), 200 # return 200 so we don't trigger an error message in the client's ST diff --git a/llm_server/routes/v2/__init__.py b/llm_server/routes/v2/__init__.py index 123683d..1860473 100644 --- a/llm_server/routes/v2/__init__.py +++ b/llm_server/routes/v2/__init__.py @@ -3,7 +3,7 @@ from flask import Blueprint from ..request_handler import before_request from ..server_error import handle_server_error -bp = Blueprint('v1', __name__) +bp = Blueprint('v2', __name__) @bp.before_request diff --git a/server.py b/server.py index 4a896af..dd508cc 100644 --- a/server.py +++ b/server.py @@ -1,3 +1,5 @@ +from llm_server.routes.v1 import old_v1_bp + try: import gevent.monkey @@ -70,6 +72,7 @@ from llm_server.routes.v2.generate_stats import generate_stats app = Flask(__name__) init_socketio(app) app.register_blueprint(bp, url_prefix='/api/v2/') +app.register_blueprint(old_v1_bp, url_prefix='/api/v1/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') flask_cache.init_app(app) flask_cache.clear() -- 2.34.1 From 7af3dbd76b1b79de683c362d4033ae4e50180a98 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 21:31:25 -0600 Subject: [PATCH 012/163] add message about settings --- llm_server/routes/v1/__init__.py | 2 +- server.py | 3 +-- templates/home.html | 1 + 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py index c812f85..cdfba2f 100644 --- a/llm_server/routes/v1/__init__.py +++ b/llm_server/routes/v1/__init__.py @@ -10,7 +10,7 @@ old_v1_bp = Blueprint('v1', __name__) @old_v1_bp.route('/', methods=['GET', 'POST']) def fallback(path): base_client_api = redis.get('base_client_api', dtype=str) - error_msg = f'The /v1/ endpoint has been depreciated. Please visit {base_client_api} for more information.' + error_msg = f'The /v1/ endpoint has been depreciated. Please visit {base_client_api} for more information.\nAlso, you must enable "Relaxed API URLS" in settings.' response_msg = format_sillytavern_err(error_msg, error_type='API') return jsonify({ 'results': [{'text': response_msg}], diff --git a/server.py b/server.py index dd508cc..380bef0 100644 --- a/server.py +++ b/server.py @@ -1,5 +1,3 @@ -from llm_server.routes.v1 import old_v1_bp - try: import gevent.monkey @@ -23,6 +21,7 @@ from llm_server.database.create import create_db from llm_server.pre_fork import server_startup from llm_server.routes.openai import openai_bp from llm_server.routes.server_error import handle_server_error +from llm_server.routes.v1 import old_v1_bp from llm_server.routes.v2 import bp from llm_server.sock import init_socketio diff --git a/templates/home.html b/templates/home.html index c366263..a706801 100644 --- a/templates/home.html +++ b/templates/home.html @@ -111,6 +111,7 @@

        Instructions

          +
        1. In Settings > Power User Options, enable Relaxed API URLS
        2. Set your API type to {{ mode_name }}
        3. Enter {{ client_api }} in the {{ api_input_textbox }} textbox.
          4. {% if enable_streaming %} -- 2.34.1 From 61856b43836f555cf44794351473e3555873625e Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 21:34:32 -0600 Subject: [PATCH 013/163] adjust message --- llm_server/routes/v1/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py index cdfba2f..18aaeee 100644 --- a/llm_server/routes/v1/__init__.py +++ b/llm_server/routes/v1/__init__.py @@ -14,5 +14,5 @@ def fallback(path): response_msg = format_sillytavern_err(error_msg, error_type='API') return jsonify({ 'results': [{'text': response_msg}], - 'result': base_client_api + 'result': f'Wrong API path, visit {base_client_api}' }), 200 # return 200 so we don't trigger an error message in the client's ST -- 2.34.1 From 9235725bdd54f79fbea1c617afe3f79a9aa3d3e4 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 21:35:55 -0600 Subject: [PATCH 014/163] adjust message --- llm_server/routes/v1/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py index 18aaeee..c492726 100644 --- a/llm_server/routes/v1/__init__.py +++ b/llm_server/routes/v1/__init__.py @@ -14,5 +14,5 @@ def fallback(path): response_msg = format_sillytavern_err(error_msg, error_type='API') return jsonify({ 'results': [{'text': response_msg}], - 'result': f'Wrong API path, visit {base_client_api}' + 'result': f'Wrong API path, visit {base_client_api} for more info' }), 200 # return 200 so we don't trigger an error message in the client's ST -- 2.34.1 From bc25d92c95fd287a9753d4f3b5074b60ae0304c4 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 21:48:16 -0600 Subject: [PATCH 015/163] reduce tokens for backend tester --- llm_server/cluster/backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index bb3e6d4..f95970c 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -12,7 +12,7 @@ def test_backend(backend_url: str, test_prompt: bool = False): "prompt": "Test prompt", "stream": False, "temperature": 0, - "max_new_tokens": 16, + "max_new_tokens": 3, } success, response, err = generator(data, backend_url, timeout=10) if not success or not response or err: -- 2.34.1 From c5b30d985c9c4b73fb681874e4b29b8d2db5ab0d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 22:11:51 -0600 Subject: [PATCH 016/163] adjust jinja template --- server.py | 4 ++-- templates/home.html | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/server.py b/server.py index 380bef0..7cb5a88 100644 --- a/server.py +++ b/server.py @@ -105,9 +105,9 @@ def home(): if default_backend_info['queued'] == 0 and default_backend_info['queued'] >= opts.concurrent_gens: # There will be a wait if the queue is empty but prompts are processing, but we don't # know how long. - default_estimated_wait_sec = f"less than {default_backend_info['estimated_wait']} seconds" + default_estimated_wait_sec = f"less than {int(default_backend_info['estimated_wait'])} seconds" else: - default_estimated_wait_sec = f"{default_backend_info['estimated_wait']} seconds" + default_estimated_wait_sec = f"{int(default_backend_info['estimated_wait'])} seconds" if len(config['analytics_tracking_code']): analytics_tracking_code = f"" diff --git a/templates/home.html b/templates/home.html index a706801..fad0429 100644 --- a/templates/home.html +++ b/templates/home.html @@ -163,7 +163,7 @@ OpenAI-Compatible API URL: {{ value.openai_client_api }}

          Context Size: {{ value.context_size }}

          -

          Average Generation Time: {{ value.avg_generation_time | int }}

          +

          Average Generation Time: {{ value.avg_generation_time | int }} seconds


        {% endfor %} -- 2.34.1 From 3ecb7bcf88116f5072b12e09b72d5a1384ca1183 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 22:14:50 -0600 Subject: [PATCH 017/163] adjust jinja template --- templates/home.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/home.html b/templates/home.html index fad0429..d599aea 100644 --- a/templates/home.html +++ b/templates/home.html @@ -151,7 +151,7 @@ {% for key, value in model_choices.items() %}
        -

        {{ key }} - {{ value.backend_count }} workers

        +

        {{ key }} - {{ value.backend_count }} {% if value.backend_count == 1 %}worker{% else %}workers{% endif %}

        Estimated Wait Time: {{ value.estimated_wait }}
        Processing: {{ value.processing }}
        -- 2.34.1 From b10d22ca0db53f9df69bfb76430a77d3e5569e5a Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sat, 30 Sep 2023 23:03:42 -0600 Subject: [PATCH 018/163] cache the home page in the background --- llm_server/config/config.py | 3 ++- llm_server/config/load.py | 1 + llm_server/opts.py | 1 + llm_server/routes/v1/__init__.py | 2 +- llm_server/workers/mainer.py | 14 ++++++++++++-- templates/home.html | 9 +-------- 6 files changed, 18 insertions(+), 12 deletions(-) diff --git a/llm_server/config/config.py b/llm_server/config/config.py index 645e81e..5308827 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -34,7 +34,8 @@ config_default_vars = { 'openai_moderation_enabled': True, 'netdata_root': None, 'show_backends': True, - 'cluster_workers': 30 + 'cluster_workers': 30, + 'background_homepage_cacher': True } config_required_vars = ['cluster', 'mode', 'llm_middleware_name'] diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 9c2e7f3..edc5991 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -53,6 +53,7 @@ def load_config(config_path): opts.openai_moderation_enabled = config['openai_moderation_enabled'] opts.show_backends = config['show_backends'] opts.cluster_workers = config['cluster_workers'] + opts.background_homepage_cacher = config['background_homepage_cacher'] if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') diff --git a/llm_server/opts.py b/llm_server/opts.py index bbd6201..ae07ca4 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -38,3 +38,4 @@ openai_moderation_enabled = True cluster = {} show_backends = True cluster_workers = 30 +background_homepage_cacher = True diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py index c492726..a52cb2e 100644 --- a/llm_server/routes/v1/__init__.py +++ b/llm_server/routes/v1/__init__.py @@ -14,5 +14,5 @@ def fallback(path): response_msg = format_sillytavern_err(error_msg, error_type='API') return jsonify({ 'results': [{'text': response_msg}], - 'result': f'Wrong API path, visit {base_client_api} for more info' + 'result': f'Wrong API path, visit {base_client_api} for more info.' }), 200 # return 200 so we don't trigger an error message in the client's ST diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py index ca82d60..580060d 100644 --- a/llm_server/workers/mainer.py +++ b/llm_server/workers/mainer.py @@ -1,11 +1,13 @@ import time +import requests + from llm_server import opts -from llm_server.cluster.backend import get_a_cluster_backend, get_backends +from llm_server.cluster.backend import get_backends from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.database.database import weighted_average_column_for_model -from llm_server.llm.info import get_info, get_running_model +from llm_server.llm.info import get_info def main_background_thread(): @@ -26,6 +28,14 @@ def main_background_thread(): cluster_config.set_backend_value(backend_url, 'average_output_tokens', average_output_tokens) if average_generation_elapsed_sec and average_output_tokens: cluster_config.set_backend_value(backend_url, 'estimated_avg_tps', estimated_avg_tps) + + if opts.background_homepage_cacher: + try: + base_client_api = redis.get('base_client_api', dtype=str) + r = requests.get('https://' + base_client_api, timeout=5) + except Exception as e: + print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}') + time.sleep(30) diff --git a/templates/home.html b/templates/home.html index d599aea..3a020a4 100644 --- a/templates/home.html +++ b/templates/home.html @@ -111,7 +111,7 @@

        Instructions

          -
        1. In Settings > Power User Options, enable Relaxed API URLS
          2. +
        2. In Settings > Power User Options, enable Relaxed API URLS.
        3. Set your API type to {{ mode_name }}
        4. Enter {{ client_api }} in the {{ api_input_textbox }} textbox.
          5. {% if enable_streaming %} @@ -167,17 +167,10 @@

        {% endfor %} - -
        git.evulid.cc/cyberes/local-llm-server
        - -- 2.34.1 From 25ec56a5efb22846246c8d90a40db2cfcf546bce Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 00:20:00 -0600 Subject: [PATCH 019/163] get streaming working, remove /v2/ --- daemon.py | 2 +- llm_server/cluster/model_choices.py | 12 +- llm_server/llm/vllm/tokenize.py | 1 + llm_server/routes/v1/__init__.py | 29 ++-- llm_server/routes/{v2 => v1}/generate.py | 0 .../routes/{v2 => v1}/generate_stats.py | 0 .../routes/{v2 => v1}/generate_stream.py | 124 ++++++++++++++---- llm_server/routes/{v2 => v1}/info.py | 0 llm_server/routes/{v2 => v1}/proxy.py | 0 llm_server/routes/v2/__init__.py | 19 --- llm_server/workers/threader.py | 2 +- server.py | 12 +- 12 files changed, 129 insertions(+), 72 deletions(-) rename llm_server/routes/{v2 => v1}/generate.py (100%) rename llm_server/routes/{v2 => v1}/generate_stats.py (100%) rename llm_server/routes/{v2 => v1}/generate_stream.py (56%) rename llm_server/routes/{v2 => v1}/info.py (100%) rename llm_server/routes/{v2 => v1}/proxy.py (100%) delete mode 100644 llm_server/routes/v2/__init__.py diff --git a/daemon.py b/daemon.py index aac2657..0fa3601 100644 --- a/daemon.py +++ b/daemon.py @@ -10,7 +10,7 @@ from llm_server.config.load import load_config, parse_backends from llm_server.custom_redis import redis from llm_server.database.create import create_db from llm_server.routes.queue import priority_queue -from llm_server.routes.v2.generate_stats import generate_stats +from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.threader import start_background script_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index ec78e2f..f8383fe 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -47,9 +47,9 @@ def get_model_choices(regen: bool = False): estimated_wait_sec = f"{estimated_wait_sec} seconds" model_choices[model] = { - 'client_api': f'https://{base_client_api}/v2/{model}', - 'ws_client_api': f'wss://{base_client_api}/v2/{model}/stream' if opts.enable_streaming else None, - 'openai_client_api': f'https://{base_client_api}/openai/v2/{model}' if opts.enable_openi_compatible_backend else 'disabled', + 'client_api': f'https://{base_client_api}/{model}', + 'ws_client_api': f'wss://{base_client_api}/{model}/v1/stream' if opts.enable_streaming else None, + 'openai_client_api': f'https://{base_client_api}/openai/{model}' if opts.enable_openi_compatible_backend else 'disabled', 'backend_count': len(b), 'estimated_wait': estimated_wait_sec, 'queued': proompters_in_queue, @@ -73,9 +73,9 @@ def get_model_choices(regen: bool = False): default_estimated_wait_sec = calculate_wait_time(default_average_generation_elapsed_sec, default_proompters_in_queue, default_backend_info['concurrent_gens'], default_active_gen_workers) default_backend_dict = { - 'client_api': f'https://{base_client_api}/v2', - 'ws_client_api': f'wss://{base_client_api}/v2' if opts.enable_streaming else None, - 'openai_client_api': f'https://{base_client_api}/openai/v2' if opts.enable_openi_compatible_backend else 'disabled', + 'client_api': f'https://{base_client_api}/v1', + 'ws_client_api': f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else None, + 'openai_client_api': f'https://{base_client_api}/openai' if opts.enable_openi_compatible_backend else 'disabled', 'estimated_wait': default_estimated_wait_sec, 'queued': default_proompters_in_queue, 'processing': default_active_gen_workers, diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 747a8b8..5cad1a4 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -6,6 +6,7 @@ from llm_server.cluster.cluster_config import cluster_config def tokenize(prompt: str, backend_url: str) -> int: + assert backend_url if not prompt: # The tokenizers have issues when the prompt is None. return 0 diff --git a/llm_server/routes/v1/__init__.py b/llm_server/routes/v1/__init__.py index a52cb2e..123683d 100644 --- a/llm_server/routes/v1/__init__.py +++ b/llm_server/routes/v1/__init__.py @@ -1,18 +1,19 @@ -from flask import Blueprint, jsonify +from flask import Blueprint -from llm_server.custom_redis import redis -from llm_server.routes.helpers.client import format_sillytavern_err +from ..request_handler import before_request +from ..server_error import handle_server_error -old_v1_bp = Blueprint('v1', __name__) +bp = Blueprint('v1', __name__) -@old_v1_bp.route('/', defaults={'path': ''}, methods=['GET', 'POST']) -@old_v1_bp.route('/', methods=['GET', 'POST']) -def fallback(path): - base_client_api = redis.get('base_client_api', dtype=str) - error_msg = f'The /v1/ endpoint has been depreciated. Please visit {base_client_api} for more information.\nAlso, you must enable "Relaxed API URLS" in settings.' - response_msg = format_sillytavern_err(error_msg, error_type='API') - return jsonify({ - 'results': [{'text': response_msg}], - 'result': f'Wrong API path, visit {base_client_api} for more info.' - }), 200 # return 200 so we don't trigger an error message in the client's ST +@bp.before_request +def before_bp_request(): + return before_request() + + +@bp.errorhandler(500) +def handle_error(e): + return handle_server_error(e) + + +from . import generate, info, proxy, generate_stream diff --git a/llm_server/routes/v2/generate.py b/llm_server/routes/v1/generate.py similarity index 100% rename from llm_server/routes/v2/generate.py rename to llm_server/routes/v1/generate.py diff --git a/llm_server/routes/v2/generate_stats.py b/llm_server/routes/v1/generate_stats.py similarity index 100% rename from llm_server/routes/v2/generate_stats.py rename to llm_server/routes/v1/generate_stats.py diff --git a/llm_server/routes/v2/generate_stream.py b/llm_server/routes/v1/generate_stream.py similarity index 56% rename from llm_server/routes/v2/generate_stream.py rename to llm_server/routes/v1/generate_stream.py index e3aeeb0..c0a0927 100644 --- a/llm_server/routes/v2/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -1,26 +1,39 @@ import json import time import traceback -from typing import Union from flask import request +from . import bp from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts -from ...cluster.backend import get_a_cluster_backend from ...database.database import log_prompt from ...llm.generator import generator from ...llm.vllm import tokenize from ...sock import sock -# TODO: have workers process streaming requests -# TODO: make sure to log the token as well (seems to be missing in the DB right now) +# Stacking the @sock.route() creates a TypeError error on the /v1/stream endpoint. +# We solve this by splitting the routes -@sock.route('/api/v1/stream') -def stream(ws): +@bp.route('/stream') +def stream(): + return 'This is a websocket endpoint.', 400 + + +@sock.route('/stream', bp=bp) +def stream_without_model(ws): + do_stream(ws, model_name=None) + + +@sock.route('//v1/stream', bp=bp) +def stream_with_model(ws, model_name=None): + do_stream(ws, model_name) + + +def do_stream(ws, model_name): def send_err_and_quit(quitting_err_msg): ws.send(json.dumps({ 'event': 'text_stream', @@ -32,23 +45,33 @@ def stream(ws): 'message_num': 1 })) ws.close() - log_in_bg(quitting_err_msg, is_error=True) - - def log_in_bg(generated_text_bg, elapsed_time_bg: Union[int, float] = None, is_error: bool = False, status_code: int = None): - generated_tokens = tokenize(generated_text_bg) - log_prompt(handler.client_ip, handler.token, input_prompt, generated_text_bg, elapsed_time_bg, handler.parameters, r_headers, status_code, r_url, cluster_backend, response_tokens=generated_tokens, is_error=is_error) + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=quitting_err_msg, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.cluster_backend_info, + response_tokens=tokenize(generated_text, handler.backend_url), + is_error=True + ) if not opts.enable_streaming: - return 'Streaming is disabled', 401 + return 'Streaming is disabled', 500 - cluster_backend = None r_headers = dict(request.headers) r_url = request.url message_num = 0 + while ws.connected: message = ws.receive() request_valid_json, request_json_body = validate_json(message) + if not request_valid_json or not request_json_body.get('prompt'): + ws.close() return 'Invalid JSON', 400 else: if opts.mode != 'vllm': @@ -57,9 +80,10 @@ def stream(ws): auth_failure = require_api_key(request_json_body) if auth_failure: + ws.close() return auth_failure - handler = OobaRequestHandler(request, request_json_body) + handler = OobaRequestHandler(request, model_name, request_json_body) generated_text = '' input_prompt = request_json_body['prompt'] response_status_code = 0 @@ -84,15 +108,14 @@ def stream(ws): } # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority) + event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority, handler.backend_url) if not event: r, _ = handler.handle_ratelimited() err_msg = r.json['results'][0]['text'] send_err_and_quit(err_msg) return try: - cluster_backend = get_a_cluster_backend() - response = generator(llm_request, cluster_backend) + response = generator(llm_request, handler.backend_url) if not response: error_msg = 'Failed to reach backend while streaming.' print('Streaming failed:', error_msg) @@ -134,10 +157,25 @@ def stream(ws): # The has client closed the stream. if request: request.close() - ws.close() + try: + ws.close() + except: + pass end_time = time.time() elapsed_time = end_time - start_time - log_prompt(handler.client_ip, handler.token, input_prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, cluster_backend, response_tokens=tokenize(generated_text)) + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=tokenize(generated_text, handler.backend_url) + ) + return message_num += 1 @@ -149,7 +187,19 @@ def stream(ws): end_time = time.time() elapsed_time = end_time - start_time - log_in_bg(generated_text, elapsed_time_bg=elapsed_time, is_error=not response, status_code=response_status_code) + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=tokenize(generated_text, handler.backend_url), + is_error=not response + ) except: traceback.print_exc() generated_text = generated_text + '\n\n' + handler.handle_error('Encountered error while streaming.', 'exception')[0].json['results'][0]['text'] @@ -161,12 +211,24 @@ def stream(ws): if request: request.close() ws.close() - log_in_bg(generated_text, is_error=True, status_code=response_status_code) + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=None, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=tokenize(generated_text, handler.backend_url), + is_error=True + ) return finally: # The worker incremented it, we'll decrement it. decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers() + decr_active_workers(handler.selected_model, handler.backend_url) try: ws.send(json.dumps({ 'event': 'stream_end', @@ -176,5 +238,19 @@ def stream(ws): # The client closed the stream. end_time = time.time() elapsed_time = end_time - start_time - log_prompt(handler.client_ip, handler.token, input_prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, cluster_backend, response_tokens=tokenize(generated_text)) - ws.close() # this is important if we encountered and error and exited early. + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=tokenize(generated_text, handler.backend_url) + ) + try: + ws.close() # this is important if we encountered and error and exited early. + except: + pass diff --git a/llm_server/routes/v2/info.py b/llm_server/routes/v1/info.py similarity index 100% rename from llm_server/routes/v2/info.py rename to llm_server/routes/v1/info.py diff --git a/llm_server/routes/v2/proxy.py b/llm_server/routes/v1/proxy.py similarity index 100% rename from llm_server/routes/v2/proxy.py rename to llm_server/routes/v1/proxy.py diff --git a/llm_server/routes/v2/__init__.py b/llm_server/routes/v2/__init__.py deleted file mode 100644 index 1860473..0000000 --- a/llm_server/routes/v2/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -from flask import Blueprint - -from ..request_handler import before_request -from ..server_error import handle_server_error - -bp = Blueprint('v2', __name__) - - -@bp.before_request -def before_bp_request(): - return before_request() - - -@bp.errorhandler(500) -def handle_error(e): - return handle_server_error(e) - - -from . import generate, info, proxy, generate_stream diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index 89a6770..0c82559 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -4,7 +4,7 @@ from threading import Thread from llm_server import opts from llm_server.cluster.stores import redis_running_models from llm_server.cluster.worker import cluster_worker -from llm_server.routes.v2.generate_stats import generate_stats +from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.inferencer import start_workers from llm_server.workers.mainer import main_background_thread from llm_server.workers.moderator import start_moderation_workers diff --git a/server.py b/server.py index 7cb5a88..0eba490 100644 --- a/server.py +++ b/server.py @@ -21,8 +21,7 @@ from llm_server.database.create import create_db from llm_server.pre_fork import server_startup from llm_server.routes.openai import openai_bp from llm_server.routes.server_error import handle_server_error -from llm_server.routes.v1 import old_v1_bp -from llm_server.routes.v2 import bp +from llm_server.routes.v1 import bp from llm_server.sock import init_socketio # TODO: per-backend workers @@ -66,12 +65,11 @@ from llm_server.helpers import auto_set_base_client_api from llm_server.llm.vllm.info import vllm_info from llm_server.custom_redis import flask_cache from llm_server.llm import redis -from llm_server.routes.v2.generate_stats import generate_stats +from llm_server.routes.v1.generate_stats import generate_stats app = Flask(__name__) init_socketio(app) -app.register_blueprint(bp, url_prefix='/api/v2/') -app.register_blueprint(old_v1_bp, url_prefix='/api/v1/') +app.register_blueprint(bp, url_prefix='/api/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') flask_cache.init_app(app) flask_cache.clear() @@ -133,8 +131,8 @@ def home(): default_active_gen_workers=default_backend_info['processing'], default_proompters_in_queue=default_backend_info['queued'], current_model=opts.manual_model_name if opts.manual_model_name else None, # else running_model, - client_api=f'https://{base_client_api}/v2', - ws_client_api=f'wss://{base_client_api}/v2/stream' if opts.enable_streaming else 'disabled', + client_api=f'https://{base_client_api}/v1', + ws_client_api=f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else 'disabled', default_estimated_wait=default_estimated_wait_sec, mode_name=mode_ui_names[opts.mode][0], api_input_textbox=mode_ui_names[opts.mode][1], -- 2.34.1 From d203973e80d56702159af6e6946a683c7bf1294f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 01:13:13 -0600 Subject: [PATCH 020/163] fix routes --- README.md | 2 ++ llm_server/cluster/model_choices.py | 2 +- llm_server/routes/v1/generate_stream.py | 7 ++++--- requirements.txt | 2 +- server.py | 4 ++-- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 429e246..4e827ca 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ To set up token auth, add rows to the `token_auth` table in the SQLite database. ### Use +Flask may give unusual errors when running `python server.py`. I think this is coming from Flask-Socket. Running with Gunicorn seems to fix the issue: `gunicorn -b :5000 --worker-class gevent server:app` + ### To Do diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index f8383fe..4b02b97 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -73,7 +73,7 @@ def get_model_choices(regen: bool = False): default_estimated_wait_sec = calculate_wait_time(default_average_generation_elapsed_sec, default_proompters_in_queue, default_backend_info['concurrent_gens'], default_active_gen_workers) default_backend_dict = { - 'client_api': f'https://{base_client_api}/v1', + 'client_api': f'https://{base_client_api}', 'ws_client_api': f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else None, 'openai_client_api': f'https://{base_client_api}/openai' if opts.enable_openi_compatible_backend else 'disabled', 'estimated_wait': default_estimated_wait_sec, diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index c0a0927..d6328d1 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -18,12 +18,13 @@ from ...sock import sock # Stacking the @sock.route() creates a TypeError error on the /v1/stream endpoint. # We solve this by splitting the routes -@bp.route('/stream') -def stream(): +@bp.route('/v1/stream') +@bp.route('//v1/stream') +def stream(model_name=None): return 'This is a websocket endpoint.', 400 -@sock.route('/stream', bp=bp) +@sock.route('/v1/stream', bp=bp) def stream_without_model(ws): do_stream(ws, model_name=None) diff --git a/requirements.txt b/requirements.txt index df16bea..28e818f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ flask~=2.3.3 pyyaml~=6.0.1 -flask_caching +Flask-Caching==2.0.2 requests~=2.31.0 tiktoken~=0.5.0 gevent~=23.9.0.post1 diff --git a/server.py b/server.py index 0eba490..6c06303 100644 --- a/server.py +++ b/server.py @@ -68,9 +68,9 @@ from llm_server.llm import redis from llm_server.routes.v1.generate_stats import generate_stats app = Flask(__name__) -init_socketio(app) app.register_blueprint(bp, url_prefix='/api/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') +init_socketio(app) flask_cache.init_app(app) flask_cache.clear() @@ -131,7 +131,7 @@ def home(): default_active_gen_workers=default_backend_info['processing'], default_proompters_in_queue=default_backend_info['queued'], current_model=opts.manual_model_name if opts.manual_model_name else None, # else running_model, - client_api=f'https://{base_client_api}/v1', + client_api=f'https://{base_client_api}', ws_client_api=f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else 'disabled', default_estimated_wait=default_estimated_wait_sec, mode_name=mode_ui_names[opts.mode][0], -- 2.34.1 From 93d19fb95b9ebf9a8df11e6bbad885eb298122b9 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 10:25:32 -0600 Subject: [PATCH 021/163] fix exception --- llm_server/routes/v1/generate_stream.py | 2 +- server.py | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index d6328d1..24d5bc6 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -50,7 +50,7 @@ def do_stream(ws, model_name): token=handler.token, prompt=input_prompt, response=quitting_err_msg, - gen_time=elapsed_time, + gen_time=None, parameters=handler.parameters, headers=r_headers, backend_response_code=response_status_code, diff --git a/server.py b/server.py index 6c06303..71685a4 100644 --- a/server.py +++ b/server.py @@ -24,17 +24,9 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio -# TODO: per-backend workers -# TODO: allow setting concurrent gens per-backend -# TODO: set the max tokens to that of the lowest backend -# TODO: implement RRD backend loadbalancer option -# TODO: have VLLM reject a request if it already has n == concurrent_gens running # TODO: add a way to cancel VLLM gens. Maybe use websockets? -# TODO: use coloredlogs # TODO: need to update opts. for workers # TODO: add a healthcheck to VLLM -# TODO: allow choosing the model by the URL path -# TODO: have VLLM report context size, uptime # Lower priority # TODO: set VLLM to stream ALL data using socket.io. If the socket disconnects, cancel generation. -- 2.34.1 From 2a3ff7e21ef37e3fe467ca97ae67616c2bb1dd37 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 14:15:01 -0600 Subject: [PATCH 022/163] update openai endpoints --- README.md | 2 + llm_server/cluster/model_choices.py | 3 +- llm_server/llm/llm_backend.py | 3 +- llm_server/llm/openai/oai_to_vllm.py | 63 +++++++ llm_server/llm/openai/transform.py | 89 ++++------ llm_server/llm/vllm/vllm_backend.py | 7 +- llm_server/routes/openai/chat_completions.py | 66 ++++---- llm_server/routes/openai/completions.py | 167 +++++++++++++++---- llm_server/routes/openai/models.py | 19 ++- llm_server/routes/openai/simulated.py | 2 +- llm_server/routes/openai_request_handler.py | 72 ++++++-- llm_server/routes/request_handler.py | 6 +- llm_server/routes/v1/generate.py | 2 - llm_server/routes/v1/generate_stats.py | 1 + llm_server/routes/v1/generate_stream.py | 2 +- llm_server/routes/v1/info.py | 15 +- llm_server/routes/v1/proxy.py | 18 +- server.py | 8 +- 18 files changed, 384 insertions(+), 161 deletions(-) create mode 100644 llm_server/llm/openai/oai_to_vllm.py diff --git a/README.md b/README.md index 4e827ca..c95e083 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,8 @@ To set up token auth, add rows to the `token_auth` table in the SQLite database. ### Use +If you see unexpected errors in the console, make sure `daemon.py` is running or else the required data will be missing from Redis. + Flask may give unusual errors when running `python server.py`. I think this is coming from Flask-Socket. Running with Gunicorn seems to fix the issue: `gunicorn -b :5000 --worker-class gevent server:app` diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index 4b02b97..31cd8cb 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -60,7 +60,8 @@ def get_model_choices(regen: bool = False): if len(context_size): model_choices[model]['context_size'] = min(context_size) - model_choices = dict(sorted(model_choices.items())) + # Python wants to sort lowercase vs. uppercase letters differently. + model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper())) default_backend = get_a_cluster_backend() default_backend_dict = {} diff --git a/llm_server/llm/llm_backend.py b/llm_server/llm/llm_backend.py index e69f8fc..ccc8db8 100644 --- a/llm_server/llm/llm_backend.py +++ b/llm_server/llm/llm_backend.py @@ -2,7 +2,6 @@ from typing import Tuple, Union import flask -from llm_server import opts from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.llm import get_token_count @@ -36,6 +35,8 @@ class LLMBackend: """ If a backend needs to do other checks not related to the prompt or parameters. Default is no extra checks preformed. + :param request: + :param prompt: :param parameters: :return: """ diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py new file mode 100644 index 0000000..5f58da5 --- /dev/null +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -0,0 +1,63 @@ +from flask import jsonify + +from llm_server import opts + + +def oai_to_vllm(request_json_body, hashes: bool, mode): + if not request_json_body.get('stop'): + request_json_body['stop'] = [] + + if hashes: + request_json_body['stop'].extend(['\n### INSTRUCTION', '\n### USER', '\n### ASSISTANT', '\n### RESPONSE']) + if opts.openai_force_no_hashes: + request_json_body['stop'].append('### ') + else: + request_json_body['stop'].extend(['\nuser:', '\nassistant:']) + + if request_json_body.get('frequency_penalty', 0) < -2: + request_json_body['frequency_penalty'] = -2 + elif request_json_body.get('frequency_penalty', 0) > 2: + request_json_body['frequency_penalty'] = 2 + + if mode == 'vllm' and request_json_body.get('top_p') == 0: + request_json_body['top_p'] = 0.01 + + return request_json_body + + +def format_oai_err(err_msg): + return jsonify({ + "error": { + "message": err_msg, + "type": "invalid_request_error", + "param": None, + "code": None + } + }), 400 + + +def validate_oai(parameters): + if parameters['temperature'] > 2: + return format_oai_err(f"{parameters['temperature']} is greater than the maximum of 2 - 'temperature'") + if parameters['temperature'] < 0: + return format_oai_err(f"{parameters['temperature']} less than the minimum of 0 - 'temperature'") + + if parameters.get('top_p', 1) > 2: + return format_oai_err(f"{parameters['top_p']} is greater than the maximum of 1 - 'top_p'") + if parameters.get('top_p', 1) < 0: + return format_oai_err(f"{parameters['top_p']} less than the minimum of 0 - 'top_p'") + + if parameters.get('presence_penalty', 1) > 2: + return format_oai_err(f"{parameters['presence_penalty']} is greater than the maximum of 2 - 'presence_penalty'") + if parameters.get('presence_penalty', 1) < -2: + return format_oai_err(f"{parameters['presence_penalty']} less than the minimum of -2 - 'presence_penalty'") + + if parameters.get('top_p', 1) > 2: + return format_oai_err(f"{parameters['top_p']} is greater than the maximum of 1 - 'top_p'") + if parameters.get('top_p', 1) < 0: + return format_oai_err(f"{parameters['top_p']} less than the minimum of 0 - 'top_p'") + + if parameters.get('top_p', 1) > 2: + return format_oai_err(f"{parameters['top_p']} is greater than the maximum of 1 - 'top_p'") + if parameters.get('top_p', 1) < 0: + return format_oai_err(f"{parameters['top_p']} less than the minimum of 0 - 'top_p'") diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index 62e0ed8..0100c7f 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -2,73 +2,24 @@ import concurrent.futures import re import secrets import string -import time import traceback from typing import Dict, List import tiktoken -from flask import jsonify, make_response -import llm_server from llm_server import opts from llm_server.llm import get_token_count -from llm_server.custom_redis import redis ANTI_RESPONSE_RE = re.compile(r'^### (.*?)(?:\:)?\s') # Match a "### XXX" line. ANTI_CONTINUATION_RE = re.compile(r'(.*?### .*?(?:\:)?(.|\n)*)') # Match everything after a "### XXX" line. -def build_openai_response(prompt, response, model=None): - # Seperate the user's prompt from the context - x = prompt.split('### USER:') - if len(x) > 1: - prompt = re.sub(r'\n$', '', x[-1].strip(' ')) - - # Make sure the bot doesn't put any other instructions in its response - # y = response.split('\n### ') - # if len(y) > 1: - # response = re.sub(r'\n$', '', y[0].strip(' ')) - response = re.sub(ANTI_RESPONSE_RE, '', response) - response = re.sub(ANTI_CONTINUATION_RE, '', response) - - # TODO: async/await - prompt_tokens = llm_server.llm.get_token_count(prompt) - response_tokens = llm_server.llm.get_token_count(response) - running_model = redis.get('running_model', 'ERROR', dtype=str) - - response = make_response(jsonify({ - "id": f"chatcmpl-{generate_oai_string(30)}", - "object": "chat.completion", - "created": int(time.time()), - "model": running_model if opts.openai_expose_our_model else model, - "choices": [{ - "index": 0, - "message": { - "role": "assistant", - "content": response, - }, - "logprobs": None, - "finish_reason": "stop" - }], - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": response_tokens, - "total_tokens": prompt_tokens + response_tokens - } - }), 200) - - stats = redis.get('proxy_stats', dtype=dict) - if stats: - response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] - return response - - def generate_oai_string(length=24): alphabet = string.ascii_letters + string.digits return ''.join(secrets.choice(alphabet) for i in range(length)) -def trim_prompt_to_fit(prompt: List[Dict[str, str]], context_token_limit: int) -> List[Dict[str, str]]: +def trim_messages_to_fit(prompt: List[Dict[str, str]], context_token_limit: int, backend_url: str) -> List[Dict[str, str]]: tokenizer = tiktoken.get_encoding("cl100k_base") def get_token_count_tiktoken_thread(msg): @@ -95,13 +46,13 @@ def trim_prompt_to_fit(prompt: List[Dict[str, str]], context_token_limit: int) - break def get_token_count_thread(msg): - return get_token_count(msg["content"]) + return get_token_count(msg["content"], backend_url) with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: token_counts = list(executor.map(get_token_count_thread, prompt)) total_tokens = sum(token_counts) - formatting_tokens = get_token_count(transform_messages_to_prompt(prompt)) - total_tokens + formatting_tokens = get_token_count(transform_messages_to_prompt(prompt), backend_url) - total_tokens if total_tokens + formatting_tokens > context_token_limit: # Start over, but this time calculate the token count using the backend @@ -109,6 +60,40 @@ def trim_prompt_to_fit(prompt: List[Dict[str, str]], context_token_limit: int) - token_counts = list(executor.map(get_token_count_thread, prompt)) else: break + return prompt + + +def trim_string_to_fit(prompt: str, context_token_limit: int, backend_url: str) -> str: + tokenizer = tiktoken.get_encoding("cl100k_base") + + def get_token_count_tiktoken_thread(msg): + return len(tokenizer.encode(msg)) + + token_count = get_token_count_tiktoken_thread(prompt) + + # If total tokens exceed the limit, start trimming + if token_count > context_token_limit: + while True: + while token_count > context_token_limit: + # Calculate the index to start removing characters from + remove_index = len(prompt) // 3 + + while remove_index < len(prompt): + prompt = prompt[:remove_index] + prompt[remove_index + 100:] + token_count = get_token_count_tiktoken_thread(prompt) + if token_count <= context_token_limit or remove_index == len(prompt): + break + + def get_token_count_thread(msg): + return get_token_count(msg, backend_url) + + token_count = get_token_count_thread(prompt) + + if token_count > context_token_limit: + # Start over, but this time calculate the token count using the backend + token_count = get_token_count_thread(prompt) + else: + break return prompt diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index a28e59a..abc1cbb 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -1,4 +1,3 @@ -import threading from typing import Tuple from flask import jsonify @@ -35,9 +34,11 @@ class VLLMBackend(LLMBackend): top_p=parameters.get('top_p', self._default_params['top_p']), top_k=top_k, use_beam_search=True if parameters.get('num_beams', 0) > 1 else False, - stop=parameters.get('stopping_strings', self._default_params['stop']), + stop=list(set(parameters.get('stopping_strings', self._default_params['stop']) or parameters.get('stop', self._default_params['stop']))), ignore_eos=parameters.get('ban_eos_token', False), - max_tokens=parameters.get('max_new_tokens', self._default_params['max_tokens']) + max_tokens=parameters.get('max_new_tokens', self._default_params['max_tokens']) or parameters.get('max_tokens', self._default_params['max_tokens']), + presence_penalty=parameters.get('presence_penalty', self._default_params['presence_penalty']), + frequency_penalty=parameters.get('frequency_penalty', self._default_params['frequency_penalty']) ) except ValueError as e: return None, str(e).strip('.') diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index b3159a5..0e716e9 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -1,5 +1,4 @@ import json -import threading import time import traceback @@ -10,11 +9,10 @@ from . import openai_bp from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler from ... import opts -from ...cluster.backend import get_a_cluster_backend from ...database.database import log_prompt from ...llm.generator import generator -from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt -from ...llm.vllm import tokenize +from ...llm.openai.oai_to_vllm import oai_to_vllm +from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit # TODO: add rate-limit headers? @@ -25,32 +23,46 @@ def openai_chat_completions(): if not request_valid_json or not request_json_body.get('messages') or not request_json_body.get('model'): return jsonify({'code': 400, 'msg': 'invalid JSON'}), 400 else: - handler = OpenAIRequestHandler(request, request_json_body) - if request_json_body.get('stream'): + handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body) + + if handler.cluster_backend_info['mode'] != 'vllm': + # TODO: implement other backends + raise NotImplementedError + + if not request_json_body.get('stream'): + try: + return handler.handle_request() + except Exception: + traceback.print_exc() + return 'Internal server error', 500 + else: if not opts.enable_streaming: # TODO: return a proper OAI error message return 'disabled', 401 - if opts.mode != 'vllm': - # TODO: implement other backends - raise NotImplementedError + if opts.openai_silent_trim: + handler.request_json_body['messages'] = trim_messages_to_fit(request_json_body['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) response_status_code = 0 start_time = time.time() request_valid, invalid_response = handler.validate_request() if not request_valid: - # TODO: simulate OAI here - raise Exception('TODO: simulate OAI here') + return invalid_response else: - handler.prompt = transform_messages_to_prompt(request_json_body['messages']) + if opts.openai_silent_trim: + oai_messages = trim_messages_to_fit(handler.request.json['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) + else: + oai_messages = handler.request.json['messages'] + + handler.prompt = transform_messages_to_prompt(oai_messages) + handler.parameters = oai_to_vllm(handler.parameters, hashes=True, mode=handler.cluster_backend_info['mode']) msg_to_backend = { **handler.parameters, 'prompt': handler.prompt, 'stream': True, } try: - cluster_backend = get_a_cluster_backend() - response = generator(msg_to_backend, cluster_backend) + response = generator(msg_to_backend, handler.backend_url) r_headers = dict(request.headers) r_url = request.url model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') @@ -94,22 +106,20 @@ def openai_chat_completions(): end_time = time.time() elapsed_time = end_time - start_time - def background_task(): - generated_tokens = tokenize(generated_text) - log_prompt(handler.client_ip, handler.token, handler.prompt, generated_text, elapsed_time, handler.parameters, r_headers, response_status_code, r_url, cluster_backend, response_tokens=generated_tokens) - - # TODO: use async/await instead of threads - thread = threading.Thread(target=background_task) - thread.start() - thread.join() + log_prompt( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) return Response(generate(), mimetype='text/event-stream') except: # TODO: simulate OAI here raise Exception - else: - try: - return handler.handle_request() - except Exception: - traceback.print_exc() - return 'Internal server error', 500 diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 8950927..41d1d3b 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -1,15 +1,19 @@ import time import traceback -from flask import jsonify, make_response, request +import simplejson as json +from flask import Response, jsonify, request -from . import openai_bp from llm_server.custom_redis import redis +from . import openai_bp from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler from ... import opts +from ...database.database import log_prompt from ...llm import get_token_count -from ...llm.openai.transform import generate_oai_string +from ...llm.generator import generator +from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai +from ...llm.openai.transform import generate_oai_string, trim_string_to_fit # TODO: add rate-limit headers? @@ -21,40 +25,137 @@ def openai_completions(): return jsonify({'code': 400, 'msg': 'Invalid JSON'}), 400 else: try: - response, status_code = OobaRequestHandler(request).handle_request() - if status_code != 200: - return status_code - output = response.json['results'][0]['text'] + handler = OobaRequestHandler(incoming_request=request) - # TODO: async/await - prompt_tokens = get_token_count(request_json_body['prompt']) - response_tokens = get_token_count(output) - running_model = redis.get('running_model', 'ERROR', dtype=str) + if handler.cluster_backend_info['mode'] != 'vllm': + # TODO: implement other backends + raise NotImplementedError - response = make_response(jsonify({ - "id": f"cmpl-{generate_oai_string(30)}", - "object": "text_completion", - "created": int(time.time()), - "model": running_model if opts.openai_expose_our_model else request_json_body.get('model'), - "choices": [ - { - "text": output, - "index": 0, - "logprobs": None, - "finish_reason": None + invalid_oai_err_msg = validate_oai(handler.request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg + handler.request_json_body = oai_to_vllm(handler.request_json_body, hashes=False, mode=handler.cluster_backend_info['mode']) + + # Convert parameters to the selected backend type + if opts.openai_silent_trim: + handler.request_json_body['prompt'] = trim_string_to_fit(request_json_body['prompt'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) + else: + # The handle_request() call below will load the prompt so we don't have + # to do anything else here. + pass + + if not request_json_body.get('stream'): + response, status_code = handler.handle_request() + if status_code != 200: + return status_code + output = response.json['results'][0]['text'] + + # TODO: async/await + prompt_tokens = get_token_count(request_json_body['prompt'], handler.backend_url) + response_tokens = get_token_count(output, handler.backend_url) + running_model = redis.get('running_model', 'ERROR', dtype=str) + + response = jsonify({ + "id": f"cmpl-{generate_oai_string(30)}", + "object": "text_completion", + "created": int(time.time()), + "model": running_model if opts.openai_expose_our_model else request_json_body.get('model'), + "choices": [ + { + "text": output, + "index": 0, + "logprobs": None, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": response_tokens, + "total_tokens": prompt_tokens + response_tokens } - ], - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": response_tokens, - "total_tokens": prompt_tokens + response_tokens - } - }), 200) + }) - stats = redis.get('proxy_stats', dtype=dict) - if stats: - response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] - return response + stats = redis.get('proxy_stats', dtype=dict) + if stats: + response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] + return response, 200 + else: + if not opts.enable_streaming: + # TODO: return a proper OAI error message + return 'disabled', 401 + + response_status_code = 0 + start_time = time.time() + + request_valid, invalid_response = handler.validate_request() + if not request_valid: + # TODO: simulate OAI here + raise Exception('TODO: simulate OAI here') + else: + handler.prompt = handler.request_json_body['prompt'] + msg_to_backend = { + **handler.parameters, + 'prompt': handler.prompt, + 'stream': True, + } + response = generator(msg_to_backend, handler.backend_url) + r_headers = dict(request.headers) + r_url = request.url + model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') + oai_string = generate_oai_string(30) + + def generate(): + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue + + data = { + "id": f"chatcmpl-{oai_string}", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + + log_prompt( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) + + return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'Internal Server Error', 500 diff --git a/llm_server/routes/openai/models.py b/llm_server/routes/openai/models.py index 657f084..39931f8 100644 --- a/llm_server/routes/openai/models.py +++ b/llm_server/routes/openai/models.py @@ -3,24 +3,24 @@ import traceback import requests from flask import jsonify -from . import openai_bp from llm_server.custom_redis import ONE_MONTH_SECONDS, flask_cache, redis +from . import openai_bp from ..stats import server_start_time from ... import opts from ...cluster.backend import get_a_cluster_backend +from ...cluster.cluster_config import cluster_config from ...helpers import jsonify_pretty -from ...llm.info import get_running_model +from ...llm.openai.transform import generate_oai_string @openai_bp.route('/models', methods=['GET']) @flask_cache.cached(timeout=60, query_string=True) def openai_list_models(): - model, error = get_running_model() - if not model: + model_name = cluster_config.get_backend(get_a_cluster_backend()).get('model') + if not model_name: response = jsonify({ 'code': 502, 'msg': 'failed to reach backend', - 'type': error.__class__.__name__ }), 500 # return 500 so Cloudflare doesn't intercept us else: running_model = redis.get('running_model', 'ERROR', dtype=str) @@ -65,7 +65,14 @@ def fetch_openai_models(): if opts.openai_api_key: try: response = requests.get('https://api.openai.com/v1/models', headers={'Authorization': f"Bearer {opts.openai_api_key}"}, timeout=10) - return response.json()['data'] + j = response.json()['data'] + + # The "modelperm" string appears to be user-specific, so we'll + # randomize it just to be safe. + for model in range(len(j)): + for p in range(len(j[model]['permission'])): + j[model]['permission'][p]['id'] = f'modelperm-{generate_oai_string(24)}' + return j except: traceback.print_exc() return [] diff --git a/llm_server/routes/openai/simulated.py b/llm_server/routes/openai/simulated.py index 301e8de..2dafedb 100644 --- a/llm_server/routes/openai/simulated.py +++ b/llm_server/routes/openai/simulated.py @@ -17,7 +17,7 @@ def openai_organizations(): "id": f"org-{generate_oai_string(24)}", "created": int(server_start_time.timestamp()), "title": "Personal", - "name": "user-abcdefghijklmnopqrstuvwx", + "name": f"user-{generate_oai_string(24)}", "description": "Personal org for bobjoe@0.0.0.0", "personal": True, "is_default": True, diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index d97ea09..6b9ff98 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -1,14 +1,19 @@ import json +import re +import time import traceback from typing import Tuple from uuid import uuid4 import flask -from flask import jsonify +from flask import Response, jsonify, make_response +import llm_server from llm_server import opts +from llm_server.custom_redis import redis from llm_server.database.database import is_api_key_moderated -from llm_server.llm.openai.transform import build_openai_response, transform_messages_to_prompt, trim_prompt_to_fit +from llm_server.llm.openai.oai_to_vllm import oai_to_vllm, validate_oai +from llm_server.llm.openai.transform import ANTI_CONTINUATION_RE, ANTI_RESPONSE_RE, generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit from llm_server.routes.request_handler import RequestHandler from llm_server.workers.moderator import add_moderation_task, get_results @@ -22,7 +27,7 @@ class OpenAIRequestHandler(RequestHandler): assert not self.used if opts.openai_silent_trim: - oai_messages = trim_prompt_to_fit(self.request.json['messages'], opts.context_size) + oai_messages = trim_messages_to_fit(self.request.json['messages'], self.cluster_backend_info['model_config']['max_position_embeddings'], self.backend_url) else: oai_messages = self.request.json['messages'] @@ -51,13 +56,8 @@ class OpenAIRequestHandler(RequestHandler): print(f'OpenAI moderation endpoint failed:', f'{e.__class__.__name__}: {e}') print(traceback.format_exc()) - # Reconstruct the request JSON with the validated parameters and prompt. - self.parameters['stop'].extend(['\n### INSTRUCTION', '\n### USER', '\n### ASSISTANT', '\n### RESPONSE']) - if opts.openai_force_no_hashes: - self.parameters['stop'].append('### ') - - if opts.mode == 'vllm' and self.request_json_body.get('top_p') == 0: - self.request_json_body['top_p'] = 0.01 + # TODO: support Ooba + self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode']) llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) @@ -65,7 +65,7 @@ class OpenAIRequestHandler(RequestHandler): model = self.request_json_body.get('model') if success: - return build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code + return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code else: return backend_response, backend_response_status_code @@ -75,7 +75,6 @@ class OpenAIRequestHandler(RequestHandler): return 'Ratelimited', 429 def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: - # TODO: return a simulated OpenAI error message return jsonify({ "error": { "message": "Invalid request, check your parameters and try again.", @@ -84,3 +83,52 @@ class OpenAIRequestHandler(RequestHandler): "code": None } }), 400 + + def build_openai_response(self, prompt, response, model=None): + # Seperate the user's prompt from the context + x = prompt.split('### USER:') + if len(x) > 1: + prompt = re.sub(r'\n$', '', x[-1].strip(' ')) + + # Make sure the bot doesn't put any other instructions in its response + response = re.sub(ANTI_RESPONSE_RE, '', response) + response = re.sub(ANTI_CONTINUATION_RE, '', response) + + # TODO: async/await + prompt_tokens = llm_server.llm.get_token_count(prompt, self.backend_url) + response_tokens = llm_server.llm.get_token_count(response, self.backend_url) + running_model = redis.get('running_model', 'ERROR', dtype=str) + + response = make_response(jsonify({ + "id": f"chatcmpl-{generate_oai_string(30)}", + "object": "chat.completion", + "created": int(time.time()), + "model": running_model if opts.openai_expose_our_model else model, + "choices": [{ + "index": 0, + "message": { + "role": "assistant", + "content": response, + }, + "logprobs": None, + "finish_reason": "stop" + }], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": response_tokens, + "total_tokens": prompt_tokens + response_tokens + } + }), 200) + + stats = redis.get('proxy_stats', dtype=dict) + if stats: + response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] + return response + + def validate_request(self, prompt: str = None, do_log: bool = False) -> Tuple[bool, Tuple[Response | None, int]]: + invalid_oai_err_msg = validate_oai(self.request_json_body) + if invalid_oai_err_msg: + return False, invalid_oai_err_msg + self.request_json_body = oai_to_vllm(self.request_json_body, hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) + # If the parameters were invalid, let the superclass deal with it. + return super().validate_request(prompt, do_log) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 0dd862a..ff83e76 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -15,13 +15,13 @@ from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.auth import parse_token from llm_server.routes.helpers.http import require_api_key, validate_json -from llm_server.routes.queue import RedisPriorityQueue, priority_queue +from llm_server.routes.queue import priority_queue DEFAULT_PRIORITY = 9999 class RequestHandler: - def __init__(self, incoming_request: flask.Request, selected_model: str, incoming_json: Union[dict, str] = None): + def __init__(self, incoming_request: flask.Request, selected_model: str = None, incoming_json: Union[dict, str] = None): self.request = incoming_request self.enable_backend_blind_rrd = request.headers.get('LLM-Blind-RRD', False) == 'true' @@ -41,7 +41,7 @@ class RequestHandler: self.cluster_backend_info = cluster_config.get_backend(self.backend_url) if not self.cluster_backend_info.get('mode'): - print(self.backend_url, self.cluster_backend_info) + print(selected_model, self.backend_url, self.cluster_backend_info) self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None diff --git a/llm_server/routes/v1/generate.py b/llm_server/routes/v1/generate.py index 39db078..1a63db9 100644 --- a/llm_server/routes/v1/generate.py +++ b/llm_server/routes/v1/generate.py @@ -5,8 +5,6 @@ from flask import jsonify, request from . import bp from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler -from ...cluster.backend import get_a_cluster_backend -from ...cluster.cluster_config import cluster_config @bp.route('/v1/generate', methods=['POST']) diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index 30e0967..500f015 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -71,6 +71,7 @@ def generate_stats(regen: bool = False): 'model': backend_info['model'], 'mode': backend_info['mode'], 'nvidia': backend_info['nvidia'], + 'priority': backend_info['priority'], } else: output['backend_info'] = {} diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 24d5bc6..c9b9c0d 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -84,7 +84,7 @@ def do_stream(ws, model_name): ws.close() return auth_failure - handler = OobaRequestHandler(request, model_name, request_json_body) + handler = OobaRequestHandler(incoming_request=request, selected_model=model_name, incoming_json=request_json_body) generated_text = '' input_prompt = request_json_body['prompt'] response_status_code = 0 diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v1/info.py index 355b415..df4e3be 100644 --- a/llm_server/routes/v1/info.py +++ b/llm_server/routes/v1/info.py @@ -4,10 +4,8 @@ from flask import jsonify, request from llm_server.custom_redis import flask_cache from . import bp -from ..auth import requires_auth from ... import opts -from ...cluster.backend import get_a_cluster_backend, get_backends, get_backends_from_model, is_valid_model -from ...cluster.cluster_config import cluster_config +from ...cluster.backend import get_a_cluster_backend, get_backends_from_model, is_valid_model @bp.route('/v1/model', methods=['GET']) @@ -39,14 +37,3 @@ def get_model(model_name=None): flask_cache.set(cache_key, response, timeout=60) return response - - -@bp.route('/backends', methods=['GET']) -@requires_auth -def get_backend(): - online, offline = get_backends() - result = {} - for i in online + offline: - info = cluster_config.get_backend(i) - result[info['hash']] = info - return jsonify(result), 200 diff --git a/llm_server/routes/v1/proxy.py b/llm_server/routes/v1/proxy.py index 5ffd194..e5ff5d3 100644 --- a/llm_server/routes/v1/proxy.py +++ b/llm_server/routes/v1/proxy.py @@ -1,6 +1,11 @@ +from flask import jsonify + +from llm_server.custom_redis import flask_cache from . import bp from .generate_stats import generate_stats -from llm_server.custom_redis import flask_cache +from ..auth import requires_auth +from ...cluster.backend import get_backends +from ...cluster.cluster_config import cluster_config from ...helpers import jsonify_pretty @@ -8,3 +13,14 @@ from ...helpers import jsonify_pretty @flask_cache.cached(timeout=5, query_string=True) def get_stats(): return jsonify_pretty(generate_stats()) + + +@bp.route('/backends', methods=['GET']) +@requires_auth +def get_backend(): + online, offline = get_backends() + result = {} + for i in online + offline: + info = cluster_config.get_backend(i) + result[info['hash']] = info + return jsonify(result), 200 diff --git a/server.py b/server.py index 71685a4..4191a84 100644 --- a/server.py +++ b/server.py @@ -24,11 +24,13 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio -# TODO: add a way to cancel VLLM gens. Maybe use websockets? -# TODO: need to update opts. for workers -# TODO: add a healthcheck to VLLM +# TODO: make sure openai_moderation_enabled works on websockets, completions, and chat completions # Lower priority +# TODO: support logit_bias on OpenAI and Ooba endpoints. +# TODO: add a way to cancel VLLM gens. Maybe use websockets? +# TODO: validate openai_silent_trim works as expected and only when enabled +# TODO: rewrite config storage. Store in redis so we can reload it. # TODO: set VLLM to stream ALL data using socket.io. If the socket disconnects, cancel generation. # TODO: estiamted wait time needs to account for full concurrent_gens but the queue is less than concurrent_gens # TODO: the estiamted wait time lags behind the stats -- 2.34.1 From f7e9687527333a6e7e1fb479c8d30d2b8372e6f7 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 16:04:53 -0600 Subject: [PATCH 023/163] finish openai endpoints --- README.md | 4 +- llm_server/cluster/backend.py | 7 +- llm_server/llm/generator.py | 6 +- llm_server/llm/llm_backend.py | 6 +- llm_server/llm/openai/transform.py | 34 +-- llm_server/llm/vllm/tokenize.py | 44 ++-- llm_server/routes/ooba_request_handler.py | 11 +- llm_server/routes/openai/chat_completions.py | 153 +++++++----- llm_server/routes/openai/completions.py | 249 ++++++++++--------- llm_server/routes/openai_request_handler.py | 24 +- llm_server/routes/request_handler.py | 2 +- llm_server/routes/v1/generate_stream.py | 4 + llm_server/routes/v1/info.py | 1 + llm_server/workers/inferencer.py | 4 +- requirements.txt | 4 +- 15 files changed, 311 insertions(+), 242 deletions(-) diff --git a/README.md b/README.md index c95e083..ccfaaf4 100644 --- a/README.md +++ b/README.md @@ -43,12 +43,10 @@ To set up token auth, add rows to the `token_auth` table in the SQLite database. ### Use -If you see unexpected errors in the console, make sure `daemon.py` is running or else the required data will be missing from Redis. +If you see unexpected errors in the console, make sure `daemon.py` is running or else the required data will be missing from Redis. You may need to wait a few minutes for the daemon to populate the database. Flask may give unusual errors when running `python server.py`. I think this is coming from Flask-Socket. Running with Gunicorn seems to fix the issue: `gunicorn -b :5000 --worker-class gevent server:app` - - ### To Do - [x] Implement streaming diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index f95970c..61061bb 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -14,8 +14,11 @@ def test_backend(backend_url: str, test_prompt: bool = False): "temperature": 0, "max_new_tokens": 3, } - success, response, err = generator(data, backend_url, timeout=10) - if not success or not response or err: + try: + success, response, err = generator(data, backend_url, timeout=10) + if not success or not response or err: + return False, {} + except: return False, {} i = get_info(backend_url, backend_info['mode']) if not i.get('model'): diff --git a/llm_server/llm/generator.py b/llm_server/llm/generator.py index f05b37c..c924d38 100644 --- a/llm_server/llm/generator.py +++ b/llm_server/llm/generator.py @@ -1,12 +1,14 @@ from llm_server import opts +from llm_server.cluster.cluster_config import cluster_config def generator(request_json_body, cluster_backend, timeout: int = None): - if opts.mode == 'oobabooga': + mode = cluster_config.get_backend(cluster_backend)['mode'] + if mode == 'ooba': # from .oobabooga.generate import generate # return generate(request_json_body) raise NotImplementedError - elif opts.mode == 'vllm': + elif mode == 'vllm': from .vllm.generate import generate return generate(request_json_body, cluster_backend, timeout=timeout) else: diff --git a/llm_server/llm/llm_backend.py b/llm_server/llm/llm_backend.py index ccc8db8..2ac2beb 100644 --- a/llm_server/llm/llm_backend.py +++ b/llm_server/llm/llm_backend.py @@ -12,6 +12,7 @@ class LLMBackend: def __init__(self, backend_url: str): self.backend_url = backend_url + self.backend_info = cluster_config.get_backend(self.backend_url) def handle_response(self, success, request: flask.Request, response_json_body: dict, response_status_code: int, client_ip, token, prompt, elapsed_time, parameters, headers): raise NotImplementedError @@ -44,8 +45,7 @@ class LLMBackend: def validate_prompt(self, prompt: str) -> Tuple[bool, Union[str, None]]: prompt_len = get_token_count(prompt, self.backend_url) - token_limit = cluster_config.get_backend(self.backend_url)['model_config']['max_position_embeddings'] + token_limit = self.backend_info['model_config']['max_position_embeddings'] if prompt_len > token_limit - 10: - model_name = redis.get('running_model', 'NO MODEL ERROR', dtype=str) - return False, f'Token indices sequence length is longer than the specified maximum sequence length for this model ({prompt_len} > {token_limit}, model: {model_name}). Please lower your context size' + return False, f'Token indices sequence length is longer than the specified maximum sequence length for this model ({prompt_len} > {token_limit}, model: {self.backend_info["model"]}). Please lower your context size' return True, None diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index 0100c7f..4cf2951 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -20,19 +20,17 @@ def generate_oai_string(length=24): def trim_messages_to_fit(prompt: List[Dict[str, str]], context_token_limit: int, backend_url: str) -> List[Dict[str, str]]: - tokenizer = tiktoken.get_encoding("cl100k_base") - - def get_token_count_tiktoken_thread(msg): - return len(tokenizer.encode(msg["content"])) + def get_token_count_thread(msg): + return get_token_count(msg["content"], backend_url) with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: - token_counts = list(executor.map(get_token_count_tiktoken_thread, prompt)) + token_counts = list(executor.map(get_token_count_thread, prompt)) total_tokens = sum(token_counts) - formatting_tokens = len(tokenizer.encode(transform_messages_to_prompt(prompt))) - total_tokens + formatting_tokens = get_token_count(transform_messages_to_prompt(prompt), backend_url) - total_tokens # If total tokens exceed the limit, start trimming - if total_tokens > context_token_limit: + if total_tokens + formatting_tokens > context_token_limit: while True: while total_tokens + formatting_tokens > context_token_limit: # Calculate the index to start removing messages from @@ -45,15 +43,11 @@ def trim_messages_to_fit(prompt: List[Dict[str, str]], context_token_limit: int, if total_tokens + formatting_tokens <= context_token_limit or remove_index == len(prompt): break - def get_token_count_thread(msg): - return get_token_count(msg["content"], backend_url) - with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: token_counts = list(executor.map(get_token_count_thread, prompt)) total_tokens = sum(token_counts) formatting_tokens = get_token_count(transform_messages_to_prompt(prompt), backend_url) - total_tokens - if total_tokens + formatting_tokens > context_token_limit: # Start over, but this time calculate the token count using the backend with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor: @@ -65,11 +59,7 @@ def trim_messages_to_fit(prompt: List[Dict[str, str]], context_token_limit: int, def trim_string_to_fit(prompt: str, context_token_limit: int, backend_url: str) -> str: tokenizer = tiktoken.get_encoding("cl100k_base") - - def get_token_count_tiktoken_thread(msg): - return len(tokenizer.encode(msg)) - - token_count = get_token_count_tiktoken_thread(prompt) + token_count = get_token_count(prompt, backend_url) # If total tokens exceed the limit, start trimming if token_count > context_token_limit: @@ -80,21 +70,17 @@ def trim_string_to_fit(prompt: str, context_token_limit: int, backend_url: str) while remove_index < len(prompt): prompt = prompt[:remove_index] + prompt[remove_index + 100:] - token_count = get_token_count_tiktoken_thread(prompt) + token_count = len(tokenizer.encode(prompt)) if token_count <= context_token_limit or remove_index == len(prompt): break - def get_token_count_thread(msg): - return get_token_count(msg, backend_url) - - token_count = get_token_count_thread(prompt) - + token_count = get_token_count(prompt, backend_url) if token_count > context_token_limit: # Start over, but this time calculate the token count using the backend - token_count = get_token_count_thread(prompt) + token_count = get_token_count(prompt, backend_url) else: break - + print(token_count) return prompt diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 5cad1a4..d51b1de 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -1,29 +1,35 @@ -import requests +import asyncio + +import aiohttp import tiktoken from llm_server import opts -from llm_server.cluster.cluster_config import cluster_config def tokenize(prompt: str, backend_url: str) -> int: assert backend_url if not prompt: - # The tokenizers have issues when the prompt is None. return 0 - tokenizer = tiktoken.get_encoding("cl100k_base") - token_limit = cluster_config.get_backend(backend_url)['model_config']['max_position_embeddings'] - # First we tokenize it locally to determine if it's worth sending it to the backend. - initial_estimate = len(tokenizer.encode(prompt)) - if initial_estimate <= token_limit + 200: - try: - r = requests.post(f'{backend_url}/tokenize', json={'input': prompt}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) - j = r.json() - return j['length'] - except Exception as e: - print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') - return len(tokenizer.encode(prompt)) + 10 - else: - # If the result was greater than our context size, return the estimate. - # We won't be sending it through the backend so it does't need to be accurage. - return initial_estimate + async def run(): + tokenizer = tiktoken.get_encoding("cl100k_base") + + async def send_chunk(chunk): + try: + async with session.post(f'{backend_url}/tokenize', json={'input': chunk}, verify_ssl=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) as response: + j = await response.json() + return j['length'] + except Exception as e: + print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') + return len(tokenizer.encode(chunk)) + 10 + + chunk_size = 300 + chunks = [prompt[i:i + chunk_size] for i in range(0, len(prompt), chunk_size)] + + async with aiohttp.ClientSession() as session: + tasks = [send_chunk(chunk) for chunk in chunks] + lengths = await asyncio.gather(*tasks) + + return sum(lengths) + + return asyncio.run(run()) diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index a272960..909848e 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -13,7 +13,7 @@ class OobaRequestHandler(RequestHandler): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def handle_request(self): + def handle_request(self, return_ok: bool = True): assert not self.used request_valid, invalid_response = self.validate_request() @@ -25,14 +25,19 @@ class OobaRequestHandler(RequestHandler): llm_request = {**self.parameters, 'prompt': prompt} _, backend_response = self.generate_response(llm_request) - return backend_response + if return_ok: + # Always return 200 so ST displays our error messages + return backend_response[0], 200 + else: + # The OpenAI route needs to detect 429 errors. + return backend_response def handle_ratelimited(self, do_log: bool = True): msg = f'Ratelimited: you are only allowed to have {opts.simultaneous_requests_per_ip} simultaneous requests at a time. Please complete your other requests before sending another.' backend_response = self.handle_error(msg) if do_log: log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) - return backend_response[0], 200 # We only return the response from handle_error(), not the error code + return backend_response[0], 429 def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: disable_st_error_formatting = request.headers.get('LLM-ST-Errors', False) == 'true' diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 0e716e9..e00d665 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -8,10 +8,11 @@ from llm_server.custom_redis import redis from . import openai_bp from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler +from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts from ...database.database import log_prompt from ...llm.generator import generator -from ...llm.openai.oai_to_vllm import oai_to_vllm +from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit @@ -24,11 +25,6 @@ def openai_chat_completions(): return jsonify({'code': 400, 'msg': 'invalid JSON'}), 400 else: handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body) - - if handler.cluster_backend_info['mode'] != 'vllm': - # TODO: implement other backends - raise NotImplementedError - if not request_json_body.get('stream'): try: return handler.handle_request() @@ -37,30 +33,51 @@ def openai_chat_completions(): return 'Internal server error', 500 else: if not opts.enable_streaming: - # TODO: return a proper OAI error message - return 'disabled', 401 + return 'DISABLED', 401 + + invalid_oai_err_msg = validate_oai(handler.request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg + handler.request_json_body = oai_to_vllm(handler.request_json_body, hashes=False, mode=handler.cluster_backend_info['mode']) if opts.openai_silent_trim: - handler.request_json_body['messages'] = trim_messages_to_fit(request_json_body['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) + handler.prompt = transform_messages_to_prompt(trim_messages_to_fit(handler.request.json['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url)) + else: + handler.prompt = transform_messages_to_prompt(handler.request.json['messages']) response_status_code = 0 start_time = time.time() + request_valid, invalid_response = handler.validate_request() if not request_valid: return invalid_response else: - if opts.openai_silent_trim: - oai_messages = trim_messages_to_fit(handler.request.json['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) - else: - oai_messages = handler.request.json['messages'] - - handler.prompt = transform_messages_to_prompt(oai_messages) - handler.parameters = oai_to_vllm(handler.parameters, hashes=True, mode=handler.cluster_backend_info['mode']) msg_to_backend = { **handler.parameters, 'prompt': handler.prompt, 'stream': True, } + + # Add a dummy event to the queue and wait for it to reach a worker + event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority, handler.backend_url) + if not event: + log_prompt( + handler.client_ip, + handler.token, + handler.prompt, + None, + None, + handler.parameters, + request.headers, + response_status_code, + request.url, + handler.backend_url, + ) + return handler.handle_ratelimited() + + # Wait for a worker to get our request and discard it. + _, _, _ = event.wait() + try: response = generator(msg_to_backend, handler.backend_url) r_headers = dict(request.headers) @@ -69,57 +86,61 @@ def openai_chat_completions(): oai_string = generate_oai_string(30) def generate(): - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + try: + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"chatcmpl-{oai_string}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' + data = { + "id": f"chatcmpl-{oai_string}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - - log_prompt( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) + log_prompt( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) + finally: + # The worker incremented it, we'll decrement it. + decrement_ip_count(handler.client_ip, 'processing_ips') + decr_active_workers(handler.selected_model, handler.backend_url) return Response(generate(), mimetype='text/event-stream') - except: - # TODO: simulate OAI here - raise Exception + except Exception: + traceback.print_exc() + return 'INTERNAL SERVER', 500 diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 41d1d3b..7bed9fa 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -8,6 +8,7 @@ from llm_server.custom_redis import redis from . import openai_bp from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler +from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts from ...database.database import log_prompt from ...llm import get_token_count @@ -24,80 +25,98 @@ def openai_completions(): if not request_valid_json or not request_json_body.get('prompt'): return jsonify({'code': 400, 'msg': 'Invalid JSON'}), 400 else: - try: - handler = OobaRequestHandler(incoming_request=request) + handler = OobaRequestHandler(incoming_request=request) - if handler.cluster_backend_info['mode'] != 'vllm': - # TODO: implement other backends - raise NotImplementedError + if handler.cluster_backend_info['mode'] != 'vllm': + # TODO: implement other backends + raise NotImplementedError - invalid_oai_err_msg = validate_oai(handler.request_json_body) - if invalid_oai_err_msg: - return invalid_oai_err_msg - handler.request_json_body = oai_to_vllm(handler.request_json_body, hashes=False, mode=handler.cluster_backend_info['mode']) + invalid_oai_err_msg = validate_oai(handler.request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg + handler.request_json_body = oai_to_vllm(handler.request_json_body, hashes=False, mode=handler.cluster_backend_info['mode']) - # Convert parameters to the selected backend type - if opts.openai_silent_trim: - handler.request_json_body['prompt'] = trim_string_to_fit(request_json_body['prompt'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) - else: - # The handle_request() call below will load the prompt so we don't have - # to do anything else here. - pass + if opts.openai_silent_trim: + handler.request_json_body['prompt'] = trim_string_to_fit(request_json_body['prompt'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) + else: + # The handle_request() call below will load the prompt so we don't have + # to do anything else here. + pass - if not request_json_body.get('stream'): - response, status_code = handler.handle_request() - if status_code != 200: - return status_code - output = response.json['results'][0]['text'] + if not request_json_body.get('stream'): + response, status_code = handler.handle_request(return_ok=False) + if status_code == 429: + return handler.handle_ratelimited() + output = response.json['results'][0]['text'] - # TODO: async/await - prompt_tokens = get_token_count(request_json_body['prompt'], handler.backend_url) - response_tokens = get_token_count(output, handler.backend_url) - running_model = redis.get('running_model', 'ERROR', dtype=str) + # TODO: async/await + prompt_tokens = get_token_count(request_json_body['prompt'], handler.backend_url) + response_tokens = get_token_count(output, handler.backend_url) + running_model = redis.get('running_model', 'ERROR', dtype=str) - response = jsonify({ - "id": f"cmpl-{generate_oai_string(30)}", - "object": "text_completion", - "created": int(time.time()), - "model": running_model if opts.openai_expose_our_model else request_json_body.get('model'), - "choices": [ - { - "text": output, - "index": 0, - "logprobs": None, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": prompt_tokens, - "completion_tokens": response_tokens, - "total_tokens": prompt_tokens + response_tokens + response = jsonify({ + "id": f"cmpl-{generate_oai_string(30)}", + "object": "text_completion", + "created": int(time.time()), + "model": running_model if opts.openai_expose_our_model else request_json_body.get('model'), + "choices": [ + { + "text": output, + "index": 0, + "logprobs": None, + "finish_reason": "stop" } - }) + ], + "usage": { + "prompt_tokens": prompt_tokens, + "completion_tokens": response_tokens, + "total_tokens": prompt_tokens + response_tokens + } + }) - stats = redis.get('proxy_stats', dtype=dict) - if stats: - response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] - return response, 200 + stats = redis.get('proxy_stats', dtype=dict) + if stats: + response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] + return response, 200 + else: + if not opts.enable_streaming: + return 'DISABLED', 401 + + response_status_code = 0 + start_time = time.time() + + request_valid, invalid_response = handler.validate_request() + if not request_valid: + return invalid_response else: - if not opts.enable_streaming: - # TODO: return a proper OAI error message - return 'disabled', 401 + handler.prompt = handler.request_json_body['prompt'] + msg_to_backend = { + **handler.parameters, + 'prompt': handler.prompt, + 'stream': True, + } - response_status_code = 0 - start_time = time.time() + # Add a dummy event to the queue and wait for it to reach a worker + event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority, handler.backend_url) + if not event: + log_prompt( + handler.client_ip, + handler.token, + handler.prompt, + None, + None, + handler.parameters, + request.headers, + response_status_code, + request.url, + handler.backend_url, + ) + return handler.handle_ratelimited() - request_valid, invalid_response = handler.validate_request() - if not request_valid: - # TODO: simulate OAI here - raise Exception('TODO: simulate OAI here') - else: - handler.prompt = handler.request_json_body['prompt'] - msg_to_backend = { - **handler.parameters, - 'prompt': handler.prompt, - 'stream': True, - } + # Wait for a worker to get our request and discard it. + _, _, _ = event.wait() + + try: response = generator(msg_to_backend, handler.backend_url) r_headers = dict(request.headers) r_url = request.url @@ -105,57 +124,61 @@ def openai_completions(): oai_string = generate_oai_string(30) def generate(): - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + try: + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"chatcmpl-{oai_string}", - "object": "text_completion", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' + data = { + "id": f"cmpl-{oai_string}", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - - log_prompt( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) + log_prompt( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) + finally: + # The worker incremented it, we'll decrement it. + decrement_ip_count(handler.client_ip, 'processing_ips') + decr_active_workers(handler.selected_model, handler.backend_url) return Response(generate(), mimetype='text/event-stream') - except Exception: - traceback.print_exc() - return 'Internal Server Error', 500 + except Exception: + traceback.print_exc() + return 'INTERNAL SERVER', 500 diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 6b9ff98..541c2c9 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -10,8 +10,9 @@ from flask import Response, jsonify, make_response import llm_server from llm_server import opts +from llm_server.cluster.model_choices import get_model_choices from llm_server.custom_redis import redis -from llm_server.database.database import is_api_key_moderated +from llm_server.database.database import is_api_key_moderated, log_prompt from llm_server.llm.openai.oai_to_vllm import oai_to_vllm, validate_oai from llm_server.llm.openai.transform import ANTI_CONTINUATION_RE, ANTI_RESPONSE_RE, generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit from llm_server.routes.request_handler import RequestHandler @@ -70,9 +71,24 @@ class OpenAIRequestHandler(RequestHandler): return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): - # TODO: return a simulated OpenAI error message - # Ratelimited: you are only allowed to have {opts.simultaneous_requests_per_ip} simultaneous requests at a time. Please complete your other requests before sending another. - return 'Ratelimited', 429 + _, default_backend_info = get_model_choices() + w = int(default_backend_info['estimated_wait']) if default_backend_info['estimated_wait'] > 0 else 2 + response = jsonify({ + "error": { + "message": "Rate limit reached on tokens per min. Limit: 10000 / min. Please try again in 6s. Contact us through our help center at help.openai.com if you continue to have issues.", + "type": "rate_limit_exceeded", + "param": None, + "code": None + } + }) + response.headers['x-ratelimit-limit-requests'] = '2' + response.headers['x-ratelimit-remaining-requests'] = '0' + response.headers['x-ratelimit-reset-requests'] = f"{w}s" + + if do_log: + log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), response.data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) + + return response, 429 def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: return jsonify({ diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index ff83e76..a595b89 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -209,7 +209,7 @@ class RequestHandler: if queued_ip_count + processing_ip < self.token_simultaneous_ip or self.token_priority == 0: return False else: - print(f'Rejecting request from {self.client_ip} - {queued_ip_count + processing_ip} queued + processing.') + print(f'Rejecting request from {self.client_ip} - {queued_ip_count + processing_ip} already queued/processing.') return True def handle_request(self) -> Tuple[flask.Response, int]: diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index c9b9c0d..9417151 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -115,6 +115,10 @@ def do_stream(ws, model_name): err_msg = r.json['results'][0]['text'] send_err_and_quit(err_msg) return + + # Wait for a worker to get our request and discard it. + _, _, _ = event.wait() + try: response = generator(llm_request, handler.backend_url) if not response: diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v1/info.py index df4e3be..6e37720 100644 --- a/llm_server/routes/v1/info.py +++ b/llm_server/routes/v1/info.py @@ -6,6 +6,7 @@ from llm_server.custom_redis import flask_cache from . import bp from ... import opts from ...cluster.backend import get_a_cluster_backend, get_backends_from_model, is_valid_model +from ...cluster.cluster_config import cluster_config @bp.route('/v1/model', methods=['GET']) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index e92052e..07de40e 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -21,8 +21,10 @@ def worker(): incr_active_workers(selected_model, backend_url) if not request_json_body: - # This was a dummy request from the websocket handler. + # This was a dummy request from the websocket handlers. # We're going to let the websocket handler decrement processing_ips and active_gen_workers. + event = DataEvent(event_id) + event.set((True, None, None)) continue try: diff --git a/requirements.txt b/requirements.txt index 28e818f..bcd1eeb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,6 @@ openai~=0.28.0 urllib3~=2.0.4 flask-sock==0.6.0 gunicorn==21.2.0 -redis==5.0.1 \ No newline at end of file +redis==5.0.1 +aiohttp==3.8.5 +asyncio==3.4.3 \ No newline at end of file -- 2.34.1 From 51881ae39dae4901635a0a21f2811e300d00a61b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 17:19:34 -0600 Subject: [PATCH 024/163] fix tokenizer --- llm_server/llm/openai/transform.py | 1 - llm_server/llm/vllm/tokenize.py | 48 ++++++++++++++++-------------- server.py | 2 ++ 3 files changed, 28 insertions(+), 23 deletions(-) diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index 4cf2951..39f942a 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -80,7 +80,6 @@ def trim_string_to_fit(prompt: str, context_token_limit: int, backend_url: str) token_count = get_token_count(prompt, backend_url) else: break - print(token_count) return prompt diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index d51b1de..006842e 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -1,6 +1,6 @@ -import asyncio +import concurrent.futures -import aiohttp +import requests import tiktoken from llm_server import opts @@ -9,27 +9,31 @@ from llm_server import opts def tokenize(prompt: str, backend_url: str) -> int: assert backend_url if not prompt: + # The tokenizers have issues when the prompt is None. return 0 + tokenizer = tiktoken.get_encoding("cl100k_base") - async def run(): - tokenizer = tiktoken.get_encoding("cl100k_base") + # Split the prompt into 300 character chunks + chunk_size = 300 + chunks = [prompt[i:i + chunk_size] for i in range(0, len(prompt), chunk_size)] - async def send_chunk(chunk): + # Define a function to send a chunk to the server + def send_chunk(chunk): + try: + r = requests.post(f'{backend_url}/tokenize', json={'input': chunk}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) + j = r.json() + return j['length'] + except Exception as e: + print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') + return len(tokenizer.encode(chunk)) + 10 + + # Use a ThreadPoolExecutor to send all chunks to the server at once + with concurrent.futures.ThreadPoolExecutor() as executor: + future_to_chunk = {executor.submit(send_chunk, chunk): chunk for chunk in chunks} + for future in concurrent.futures.as_completed(future_to_chunk): + chunk = future_to_chunk[future] try: - async with session.post(f'{backend_url}/tokenize', json={'input': chunk}, verify_ssl=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) as response: - j = await response.json() - return j['length'] - except Exception as e: - print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') - return len(tokenizer.encode(chunk)) + 10 - - chunk_size = 300 - chunks = [prompt[i:i + chunk_size] for i in range(0, len(prompt), chunk_size)] - - async with aiohttp.ClientSession() as session: - tasks = [send_chunk(chunk) for chunk in chunks] - lengths = await asyncio.gather(*tasks) - - return sum(lengths) - - return asyncio.run(run()) + data = future.result() + except Exception as exc: + print('%r generated an exception: %s' % (chunk, exc)) + return sum(future.result() for future in future_to_chunk) diff --git a/server.py b/server.py index 4191a84..1d89ca2 100644 --- a/server.py +++ b/server.py @@ -24,7 +24,9 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio +# TODO: is frequency penalty the same as ooba repetition penalty??? # TODO: make sure openai_moderation_enabled works on websockets, completions, and chat completions +# TODO: if a backend is at its limit of concurrent requests, choose a different one # Lower priority # TODO: support logit_bias on OpenAI and Ooba endpoints. -- 2.34.1 From a594729d0007691ba989f93ec0b713f1d5ecf35d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 22:37:13 -0600 Subject: [PATCH 025/163] fix keyerror --- llm_server/llm/openai/oai_to_vllm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 5f58da5..237dc16 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -37,9 +37,9 @@ def format_oai_err(err_msg): def validate_oai(parameters): - if parameters['temperature'] > 2: + if parameters.get('temperature', 0) > 2: return format_oai_err(f"{parameters['temperature']} is greater than the maximum of 2 - 'temperature'") - if parameters['temperature'] < 0: + if parameters.get('temperature', 0) < 0: return format_oai_err(f"{parameters['temperature']} less than the minimum of 0 - 'temperature'") if parameters.get('top_p', 1) > 2: -- 2.34.1 From 21da2f63738953d64e7000d608941308a42f3c92 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 22:58:08 -0600 Subject: [PATCH 026/163] fix openai error message --- llm_server/llm/openai/oai_to_vllm.py | 3 +++ llm_server/routes/helpers/http.py | 2 +- llm_server/routes/openai/chat_completions.py | 3 +++ llm_server/routes/openai/completions.py | 3 +++ llm_server/routes/request_handler.py | 12 +++++++++++- 5 files changed, 21 insertions(+), 2 deletions(-) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 237dc16..ce59e9b 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -61,3 +61,6 @@ def validate_oai(parameters): return format_oai_err(f"{parameters['top_p']} is greater than the maximum of 1 - 'top_p'") if parameters.get('top_p', 1) < 0: return format_oai_err(f"{parameters['top_p']} less than the minimum of 0 - 'top_p'") + + if parameters.get('max_tokens', 2) < 1: + return format_oai_err(f"{parameters['max_tokens']} is less than the minimum of 1 - 'max_tokens'") diff --git a/llm_server/routes/helpers/http.py b/llm_server/routes/helpers/http.py index 2fa1190..a3f1906 100644 --- a/llm_server/routes/helpers/http.py +++ b/llm_server/routes/helpers/http.py @@ -100,4 +100,4 @@ def validate_json(data: Union[str, flask.Request, requests.models.Response, flas j = json.loads(str(data)) return True, j except Exception as e: - return False, e + return False, e \ No newline at end of file diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index e00d665..e59f255 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -27,6 +27,9 @@ def openai_chat_completions(): handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body) if not request_json_body.get('stream'): try: + invalid_oai_err_msg = validate_oai(request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg return handler.handle_request() except Exception: traceback.print_exc() diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 7bed9fa..e772842 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -44,6 +44,9 @@ def openai_completions(): pass if not request_json_body.get('stream'): + invalid_oai_err_msg = validate_oai(request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg response, status_code = handler.handle_request(return_ok=False) if status_code == 429: return handler.handle_ratelimited() diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index a595b89..f93547b 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -171,7 +171,17 @@ class RequestHandler: else: error_msg = error_msg.strip('.') + '.' backend_response = self.handle_error(error_msg) - log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.backend_url, is_error=True) + log_prompt(ip=self.client_ip, + token=self.token, + prompt=prompt, + response=backend_response[0].data.decode('utf-8'), + gen_time=None, + parameters=self.parameters, + headers=dict(self.request.headers), + backend_response_code=response_status_code, + request_url=self.request.url, + backend_url=self.backend_url, + is_error=True) return (False, None, None, 0), backend_response # =============================================== -- 2.34.1 From d1c4e68f8be19fb9df3c46b15d72f7e3a6293da0 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 1 Oct 2023 23:07:49 -0600 Subject: [PATCH 027/163] fix openai models response --- llm_server/routes/openai/models.py | 53 +++++++++++++++--------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/llm_server/routes/openai/models.py b/llm_server/routes/openai/models.py index 39931f8..9f2845d 100644 --- a/llm_server/routes/openai/models.py +++ b/llm_server/routes/openai/models.py @@ -25,38 +25,37 @@ def openai_list_models(): else: running_model = redis.get('running_model', 'ERROR', dtype=str) oai = fetch_openai_models() - r = [] + r = { + "object": "list", + "data": oai + } + # TODO: verify this works if opts.openai_expose_our_model: - r = [{ - "object": "list", - "data": [ + r["data"].insert(0, { + "id": running_model, + "object": "model", + "created": int(server_start_time.timestamp()), + "owned_by": opts.llm_middleware_name, + "permission": [ { "id": running_model, - "object": "model", + "object": "model_permission", "created": int(server_start_time.timestamp()), - "owned_by": opts.llm_middleware_name, - "permission": [ - { - "id": running_model, - "object": "model_permission", - "created": int(server_start_time.timestamp()), - "allow_create_engine": False, - "allow_sampling": False, - "allow_logprobs": False, - "allow_search_indices": False, - "allow_view": True, - "allow_fine_tuning": False, - "organization": "*", - "group": None, - "is_blocking": False - } - ], - "root": None, - "parent": None + "allow_create_engine": False, + "allow_sampling": False, + "allow_logprobs": False, + "allow_search_indices": False, + "allow_view": True, + "allow_fine_tuning": False, + "organization": "*", + "group": None, + "is_blocking": False } - ] - }] - response = jsonify_pretty(r + oai), 200 + ], + "root": None, + "parent": None + }) + response = jsonify_pretty(r), 200 return response -- 2.34.1 From b0089859d7671a3b7131d65df4c6a3c3e038d16f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 2 Oct 2023 02:05:15 -0600 Subject: [PATCH 028/163] fix ratelimiting --- llm_server/database/database.py | 18 ++++++++++++++ llm_server/llm/vllm/tokenize.py | 1 + llm_server/routes/queue.py | 5 ++-- llm_server/routes/request_handler.py | 36 +++++++++------------------- 4 files changed, 33 insertions(+), 27 deletions(-) diff --git a/llm_server/database/database.py b/llm_server/database/database.py index 1dc2145..dec5e98 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -188,3 +188,21 @@ def increment_token_uses(token): cursor.execute('UPDATE token_auth SET uses = uses + 1 WHERE token = %s', (token,)) finally: cursor.close() + + +def get_token_ratelimit(token): + priority = 9990 + simultaneous_ip = opts.simultaneous_requests_per_ip + if token: + cursor = database.cursor() + try: + cursor.execute("SELECT priority, simultaneous_ip FROM token_auth WHERE token = %s", (token,)) + result = cursor.fetchone() + if result: + priority, simultaneous_ip = result + if simultaneous_ip is None: + # No ratelimit for this token if null + simultaneous_ip = 999999999 + finally: + cursor.close() + return priority, simultaneous_ip diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 006842e..bd44ad8 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -25,6 +25,7 @@ def tokenize(prompt: str, backend_url: str) -> int: return j['length'] except Exception as e: print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') + raise Exception return len(tokenizer.encode(chunk)) + 10 # Use a ThreadPoolExecutor to send all chunks to the server at once diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index f058298..5d2c6b3 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -5,8 +5,8 @@ from uuid import uuid4 from redis import Redis -from llm_server import opts from llm_server.custom_redis import RedisCustom, redis +from llm_server.database.database import get_token_ratelimit def increment_ip_count(client_ip: str, redis_key): @@ -32,7 +32,8 @@ class RedisPriorityQueue: ip_count = self.redis.hget('queued_ip_count', item[1]) if ip_count: ip_count = int(ip_count) - if ip_count and int(ip_count) >= opts.simultaneous_requests_per_ip and priority != 0: + _, simultaneous_ip = get_token_ratelimit(item[2]) + if ip_count and int(ip_count) >= simultaneous_ip and priority != 0: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index f93547b..7c425dc 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -8,8 +8,7 @@ from llm_server import opts from llm_server.cluster.backend import get_a_cluster_backend from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis -from llm_server.database.conn import database -from llm_server.database.database import log_prompt +from llm_server.database.database import get_token_ratelimit, log_prompt from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend @@ -17,8 +16,6 @@ from llm_server.routes.auth import parse_token from llm_server.routes.helpers.http import require_api_key, validate_json from llm_server.routes.queue import priority_queue -DEFAULT_PRIORITY = 9999 - class RequestHandler: def __init__(self, incoming_request: flask.Request, selected_model: str = None, incoming_json: Union[dict, str] = None): @@ -36,7 +33,7 @@ class RequestHandler: self.start_time = time.time() self.client_ip = self.get_client_ip() self.token = self.get_auth_token() - self.token_priority, self.token_simultaneous_ip = self.get_token_ratelimit() + self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) @@ -58,6 +55,8 @@ class RequestHandler: return parse_token(self.request.headers['Authorization']) def get_client_ip(self): + if self.request.headers.get('Llm-Connecting-Ip'): + return self.request.headers['Llm-Connecting-Ip'] if self.request.headers.get('X-Connecting-IP'): return self.request.headers.get('X-Connecting-IP') elif self.request.headers.get('Cf-Connecting-Ip'): @@ -67,23 +66,6 @@ class RequestHandler: else: return self.request.remote_addr - def get_token_ratelimit(self): - priority = DEFAULT_PRIORITY - simultaneous_ip = opts.simultaneous_requests_per_ip - if self.token: - cursor = database.cursor() - try: - cursor.execute("SELECT priority, simultaneous_ip FROM token_auth WHERE token = %s", (self.token,)) - result = cursor.fetchone() - if result: - priority, simultaneous_ip = result - if simultaneous_ip is None: - # No ratelimit for this token if null - simultaneous_ip = 999999999 - finally: - cursor.close() - return priority, simultaneous_ip - def get_parameters(self): if self.request_json_body.get('max_tokens'): self.request_json_body['max_new_tokens'] = self.request_json_body.pop('max_tokens') @@ -210,17 +192,21 @@ class RequestHandler: return (success, response, error_msg, elapsed_time), self.backend.handle_response(success, self.request, response_json_body, response_status_code, self.client_ip, self.token, prompt, elapsed_time, self.parameters, dict(self.request.headers)) def is_client_ratelimited(self) -> bool: + if self.token_priority == 0: + return False + queued_ip_count = int(priority_queue.get_queued_ip_count(self.client_ip)) x = redis.hget('processing_ips', self.client_ip) if x: processing_ip = int(x) else: processing_ip = 0 - if queued_ip_count + processing_ip < self.token_simultaneous_ip or self.token_priority == 0: - return False - else: + + if queued_ip_count + processing_ip >= self.token_simultaneous_ip: print(f'Rejecting request from {self.client_ip} - {queued_ip_count + processing_ip} already queued/processing.') return True + else: + return False def handle_request(self) -> Tuple[flask.Response, int]: # Must include this in your child. -- 2.34.1 From 4f226ae38ebb4ecfc0879417996314e33e8e53f8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 2 Oct 2023 11:11:48 -0600 Subject: [PATCH 029/163] handle requests to offline backends --- llm_server/workers/inferencer.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 07de40e..0aff9ac 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,6 +1,7 @@ import threading import time +from llm_server.cluster.backend import get_a_cluster_backend from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.llm.generator import generator @@ -10,8 +11,17 @@ from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip def worker(): while True: (request_json_body, client_ip, token, parameters, backend_url), event_id, selected_model = priority_queue.get() + backend_info = cluster_config.get_backend(backend_url) + + if not backend_info['online']: + old = backend_url + backend_url = get_a_cluster_backend() + backend_info = cluster_config.get_backend(backend_url) + print(f'Backend {old} offline. Request was redirected to {backend_url}') + del old + if not selected_model: - selected_model = cluster_config.get_backend(backend_url)['model'] + selected_model = backend_info['model'] # This wait time is "invisible", meaning the worker may as # well be still waiting to get an item from the queue. -- 2.34.1 From 94141b8ecf71b5ac98ad39e5467b7410fd854b99 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 2 Oct 2023 20:53:08 -0600 Subject: [PATCH 030/163] fix processing not being decremented on streaming, fix confusion over queue, adjust stop sequences --- llm_server/cluster/model_choices.py | 4 +- llm_server/database/database.py | 10 +- llm_server/llm/__init__.py | 3 + llm_server/llm/openai/oai_to_vllm.py | 4 +- llm_server/llm/vllm/tokenize.py | 3 + llm_server/routes/openai/chat_completions.py | 2 +- llm_server/routes/openai/completions.py | 2 +- llm_server/routes/openai_request_handler.py | 7 +- llm_server/routes/queue.py | 23 +- llm_server/routes/request_handler.py | 2 +- llm_server/routes/stats.py | 9 +- llm_server/routes/v1/generate_stream.py | 365 +++++++++---------- llm_server/workers/inferencer.py | 11 +- other/vllm/vllm_api_server.py | 0 requirements.txt | 4 +- server.py | 2 +- 16 files changed, 226 insertions(+), 225 deletions(-) mode change 100644 => 100755 other/vllm/vllm_api_server.py diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index 31cd8cb..3df3aea 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -5,7 +5,7 @@ from llm_server.cluster.backend import get_a_cluster_backend, get_backends_from_ from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue -from llm_server.routes.stats import calculate_wait_time, get_active_gen_workers +from llm_server.routes.stats import calculate_wait_time, get_active_gen_workers_model # TODO: give this a better name! @@ -30,7 +30,7 @@ def get_model_choices(regen: bool = False): if backend_info.get('average_generation_elapsed_sec'): avg_gen_per_worker.append(backend_info['average_generation_elapsed_sec']) - active_gen_workers = get_active_gen_workers(model) + active_gen_workers = get_active_gen_workers_model(model) proompters_in_queue = priority_queue.len(model) if len(avg_gen_per_worker): diff --git a/llm_server/database/database.py b/llm_server/database/database.py index dec5e98..fc1aa21 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -2,15 +2,15 @@ import json import time import traceback from threading import Thread +from typing import Union -import llm_server from llm_server import opts from llm_server.cluster.cluster_config import cluster_config from llm_server.database.conn import database -from llm_server.llm.vllm import tokenize +from llm_server.llm import get_token_count -def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, backend_url, response_tokens: int = None, is_error: bool = False): +def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): def background_task(): nonlocal ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, backend_url, response_tokens, is_error # Try not to shove JSON into the database. @@ -23,10 +23,10 @@ def log_prompt(ip, token, prompt, response, gen_time, parameters, headers, backe except: pass - prompt_tokens = llm_server.llm.get_token_count(prompt, backend_url) + prompt_tokens = get_token_count(prompt, backend_url) if not is_error: if not response_tokens: - response_tokens = llm_server.llm.get_token_count(response, backend_url) + response_tokens = get_token_count(response, backend_url) else: response_tokens = None diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 3feb027..ba46635 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -3,6 +3,9 @@ from llm_server.custom_redis import redis def get_token_count(prompt: str, backend_url: str): + assert isinstance(prompt, str) + assert isinstance(backend_url, str) + backend_mode = redis.get('backend_mode', dtype=str) if backend_mode == 'vllm': return vllm.tokenize(prompt, backend_url) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index ce59e9b..9111389 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -8,11 +8,11 @@ def oai_to_vllm(request_json_body, hashes: bool, mode): request_json_body['stop'] = [] if hashes: - request_json_body['stop'].extend(['\n### INSTRUCTION', '\n### USER', '\n### ASSISTANT', '\n### RESPONSE']) + request_json_body['stop'].extend(['### INSTRUCTION', '### USER', '### ASSISTANT', '### RESPONSE']) if opts.openai_force_no_hashes: request_json_body['stop'].append('### ') else: - request_json_body['stop'].extend(['\nuser:', '\nassistant:']) + request_json_body['stop'].extend(['user:', 'assistant:']) if request_json_body.get('frequency_penalty', 0) < -2: request_json_body['frequency_penalty'] = -2 diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index bd44ad8..d5a1b71 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -8,6 +8,9 @@ from llm_server import opts def tokenize(prompt: str, backend_url: str) -> int: assert backend_url + assert isinstance(prompt, str) + assert isinstance(backend_url, str) + if not prompt: # The tokenizers have issues when the prompt is None. return 0 diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index e59f255..c46e89f 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -62,7 +62,7 @@ def openai_chat_completions(): } # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority, handler.backend_url) + event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: log_prompt( handler.client_ip, diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index e772842..6904348 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -100,7 +100,7 @@ def openai_completions(): } # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority, handler.backend_url) + event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: log_prompt( handler.client_ip, diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 541c2c9..8664695 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -8,11 +8,11 @@ from uuid import uuid4 import flask from flask import Response, jsonify, make_response -import llm_server from llm_server import opts from llm_server.cluster.model_choices import get_model_choices from llm_server.custom_redis import redis from llm_server.database.database import is_api_key_moderated, log_prompt +from llm_server.llm import get_token_count from llm_server.llm.openai.oai_to_vllm import oai_to_vllm, validate_oai from llm_server.llm.openai.transform import ANTI_CONTINUATION_RE, ANTI_RESPONSE_RE, generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit from llm_server.routes.request_handler import RequestHandler @@ -110,9 +110,8 @@ class OpenAIRequestHandler(RequestHandler): response = re.sub(ANTI_RESPONSE_RE, '', response) response = re.sub(ANTI_CONTINUATION_RE, '', response) - # TODO: async/await - prompt_tokens = llm_server.llm.get_token_count(prompt, self.backend_url) - response_tokens = llm_server.llm.get_token_count(response, self.backend_url) + prompt_tokens = get_token_count(prompt, self.backend_url) + response_tokens = get_token_count(response, self.backend_url) running_model = redis.get('running_model', 'ERROR', dtype=str) response = make_response(jsonify({ diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 5d2c6b3..a8a47b1 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -27,7 +27,6 @@ class RedisPriorityQueue: def put(self, item, priority, selected_model): event = DataEvent() - # Check if the IP is already in the dictionary and if it has reached the limit ip_count = self.redis.hget('queued_ip_count', item[1]) if ip_count: @@ -99,16 +98,20 @@ class DataEvent: priority_queue = RedisPriorityQueue() +def update_active_workers(key: str, operation: str): + if operation == 'incr': + redis.incr(f'active_gen_workers:{key}') + elif operation == 'decr': + redis.decr(f'active_gen_workers:{key}') + if redis.get(f'active_gen_workers:{key}', default=0, dtype=int) < 0: + redis.set(f'active_gen_workers:{key}', 0) + + def incr_active_workers(selected_model: str, backend_url: str): - redis.incr(f'active_gen_workers:{selected_model}') - redis.incr(f'active_gen_workers:{backend_url}') + update_active_workers(selected_model, 'incr') + update_active_workers(backend_url, 'incr') def decr_active_workers(selected_model: str, backend_url: str): - redis.decr(f'active_gen_workers:{selected_model}') - if redis.get(f'active_gen_workers:{selected_model}', 0, dtype=int) < 0: - redis.set(f'active_gen_workers:{selected_model}', 0) - - redis.decr(f'active_gen_workers:{backend_url}') - if redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) < 0: - redis.set(f'active_gen_workers:{backend_url}', 0) + update_active_workers(selected_model, 'decr') + update_active_workers(backend_url, 'decr') diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 7c425dc..4e8b8e4 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -36,6 +36,7 @@ class RequestHandler: self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + self.selected_model = self.cluster_backend_info['model'] if not self.cluster_backend_info.get('mode'): print(selected_model, self.backend_url, self.cluster_backend_info) @@ -43,7 +44,6 @@ class RequestHandler: self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None self.used = False - self.selected_model = selected_model redis.zadd('recent_prompters', {self.client_ip: time.time()}) def get_auth_token(self): diff --git a/llm_server/routes/stats.py b/llm_server/routes/stats.py index 9e1f291..7f3b2fe 100644 --- a/llm_server/routes/stats.py +++ b/llm_server/routes/stats.py @@ -15,13 +15,8 @@ def get_total_proompts(): return count -def get_active_gen_workers(selected_model: str = None, ): - active_gen_workers = redis.get(f'active_gen_workers:{selected_model}') - if active_gen_workers is None: - count = 0 - else: - count = int(active_gen_workers) - return count +def get_active_gen_workers_model(selected_model: str = None): + return redis.get(f'active_gen_workers:{selected_model}', dtype=int, default=0) def calculate_wait_time(gen_time_calc, proompters_in_queue, concurrent_gens, active_gen_workers): diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 9417151..ac148dd 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -11,7 +11,6 @@ from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts from ...database.database import log_prompt from ...llm.generator import generator -from ...llm.vllm import tokenize from ...sock import sock @@ -45,7 +44,6 @@ def do_stream(ws, model_name): 'event': 'stream_end', 'message_num': 1 })) - ws.close() log_prompt(ip=handler.client_ip, token=handler.token, prompt=input_prompt, @@ -56,7 +54,7 @@ def do_stream(ws, model_name): backend_response_code=response_status_code, request_url=r_url, backend_url=handler.cluster_backend_info, - response_tokens=tokenize(generated_text, handler.backend_url), + response_tokens=None, is_error=True ) @@ -67,195 +65,192 @@ def do_stream(ws, model_name): r_url = request.url message_num = 0 - while ws.connected: - message = ws.receive() - request_valid_json, request_json_body = validate_json(message) + try: + while ws.connected: + message = ws.receive() + request_valid_json, request_json_body = validate_json(message) - if not request_valid_json or not request_json_body.get('prompt'): - ws.close() - return 'Invalid JSON', 400 - else: - if opts.mode != 'vllm': - # TODO: implement other backends - raise NotImplementedError - - auth_failure = require_api_key(request_json_body) - if auth_failure: - ws.close() - return auth_failure - - handler = OobaRequestHandler(incoming_request=request, selected_model=model_name, incoming_json=request_json_body) - generated_text = '' - input_prompt = request_json_body['prompt'] - response_status_code = 0 - start_time = time.time() - - err_msg = None - if handler.is_client_ratelimited(): - r, _ = handler.handle_ratelimited(do_log=False) - err_msg = r.json['results'][0]['text'] + if not request_valid_json or not request_json_body.get('prompt'): + return 'Invalid JSON', 400 else: - request_valid, invalid_response = handler.validate_request(prompt=input_prompt) - if not request_valid: - err_msg = invalid_response[0].json['results'][0]['text'] - if err_msg: - send_err_and_quit(err_msg) - return + if opts.mode != 'vllm': + # TODO: implement other backends + raise NotImplementedError - llm_request = { - **handler.parameters, - 'prompt': input_prompt, - 'stream': True, - } + auth_failure = require_api_key(request_json_body) + if auth_failure: + return auth_failure - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, None), handler.token_priority, handler.backend_url) - if not event: - r, _ = handler.handle_ratelimited() - err_msg = r.json['results'][0]['text'] - send_err_and_quit(err_msg) - return + handler = OobaRequestHandler(incoming_request=request, selected_model=model_name, incoming_json=request_json_body) + generated_text = '' + input_prompt = request_json_body['prompt'] + response_status_code = 0 + start_time = time.time() - # Wait for a worker to get our request and discard it. - _, _, _ = event.wait() + err_msg = None + if handler.is_client_ratelimited(): + r, _ = handler.handle_ratelimited(do_log=False) + err_msg = r.json['results'][0]['text'] + else: + request_valid, invalid_response = handler.validate_request(prompt=input_prompt) + if not request_valid: + err_msg = invalid_response[0].json['results'][0]['text'] + if err_msg: + send_err_and_quit(err_msg) + return - try: - response = generator(llm_request, handler.backend_url) - if not response: - error_msg = 'Failed to reach backend while streaming.' - print('Streaming failed:', error_msg) - msg = handler.handle_error(error_msg)[0].json['results'][0]['text'] + llm_request = { + **handler.parameters, + 'prompt': input_prompt, + 'stream': True, + } + + # Add a dummy event to the queue and wait for it to reach a worker + event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + if not event: + r, _ = handler.handle_ratelimited() + err_msg = r.json['results'][0]['text'] + send_err_and_quit(err_msg) + return + + # Wait for a worker to get our request and discard it. + _, _, _ = event.wait() + + try: + response = generator(llm_request, handler.backend_url) + + if not response: + error_msg = 'Failed to reach backend while streaming.' + print('Streaming failed:', error_msg) + msg = handler.handle_error(error_msg)[0].json['results'][0]['text'] + ws.send(json.dumps({ + 'event': 'text_stream', + 'message_num': message_num, + 'text': msg + })) + else: + # Be extra careful when getting attributes from the response object + try: + response_status_code = response.status_code + except: + response_status_code = 0 + + partial_response = b'' + + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(input_prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue + try: + ws.send(json.dumps({ + 'event': 'text_stream', + 'message_num': message_num, + 'text': new + })) + except: + # The has client closed the stream. + if request: + # Cancel the backend? + request.close() + end_time = time.time() + elapsed_time = end_time - start_time + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=None + ) + return + + message_num += 1 + partial_response = b'' # Reset the partial response + + # If there is no more data, break the loop + if not chunk: + break + + end_time = time.time() + elapsed_time = end_time - start_time + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=None, + is_error=not response + ) + except: + traceback.print_exc() + generated_text = generated_text + '\n\n' + handler.handle_error('Encountered error while streaming.', 'exception')[0].json['results'][0]['text'] ws.send(json.dumps({ 'event': 'text_stream', 'message_num': message_num, - 'text': msg + 'text': generated_text })) - else: - # Be extra careful when getting attributes from the response object - try: - response_status_code = response.status_code - except: - response_status_code = 0 - - partial_response = b'' - - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(input_prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue - try: - ws.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'text': new - })) - except: - # The has client closed the stream. - if request: - request.close() - try: - ws.close() - except: - pass - end_time = time.time() - elapsed_time = end_time - start_time - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=tokenize(generated_text, handler.backend_url) - ) - - return - - message_num += 1 - partial_response = b'' # Reset the partial response - - # If there is no more data, break the loop - if not chunk: - break - - end_time = time.time() - elapsed_time = end_time - start_time - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=tokenize(generated_text, handler.backend_url), - is_error=not response - ) - except: - traceback.print_exc() - generated_text = generated_text + '\n\n' + handler.handle_error('Encountered error while streaming.', 'exception')[0].json['results'][0]['text'] - ws.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'text': generated_text - })) - if request: - request.close() - ws.close() - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=None, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=tokenize(generated_text, handler.backend_url), - is_error=True - ) - return - finally: - # The worker incremented it, we'll decrement it. - decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers(handler.selected_model, handler.backend_url) - try: - ws.send(json.dumps({ - 'event': 'stream_end', - 'message_num': message_num - })) - except: - # The client closed the stream. - end_time = time.time() - elapsed_time = end_time - start_time - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=tokenize(generated_text, handler.backend_url) - ) - try: - ws.close() # this is important if we encountered and error and exited early. - except: - pass + if request: + request.close() + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=None, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=None, + is_error=True + ) + return + finally: + # The worker incremented it, we'll decrement it. + decrement_ip_count(handler.client_ip, 'processing_ips') + decr_active_workers(handler.selected_model, handler.backend_url) + try: + ws.send(json.dumps({ + 'event': 'stream_end', + 'message_num': message_num + })) + except: + # The client closed the stream. + end_time = time.time() + elapsed_time = end_time - start_time + log_prompt(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url, + response_tokens=None + ) + finally: + try: + # Must close the connection or greenlets will complain. + ws.close() + except: + pass diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 0aff9ac..c5eb12a 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -11,19 +11,23 @@ from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip def worker(): while True: (request_json_body, client_ip, token, parameters, backend_url), event_id, selected_model = priority_queue.get() + if not backend_url: + backend_url = get_a_cluster_backend(selected_model) backend_info = cluster_config.get_backend(backend_url) + # The backend could have died between when the request was + # submitted and now, so let's double check it's still online. if not backend_info['online']: old = backend_url backend_url = get_a_cluster_backend() backend_info = cluster_config.get_backend(backend_url) print(f'Backend {old} offline. Request was redirected to {backend_url}') - del old + del old # gc if not selected_model: selected_model = backend_info['model'] - # This wait time is "invisible", meaning the worker may as + # This wait time will be "invisible", meaning the worker may as # well be still waiting to get an item from the queue. need_to_wait(backend_url) @@ -32,7 +36,8 @@ def worker(): if not request_json_body: # This was a dummy request from the websocket handlers. - # We're going to let the websocket handler decrement processing_ips and active_gen_workers. + # We're going to let the websocket handler decrement + # processing_ips and active_gen_workers. event = DataEvent(event_id) event.set((True, None, None)) continue diff --git a/other/vllm/vllm_api_server.py b/other/vllm/vllm_api_server.py old mode 100644 new mode 100755 diff --git a/requirements.txt b/requirements.txt index bcd1eeb..28e818f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,4 @@ openai~=0.28.0 urllib3~=2.0.4 flask-sock==0.6.0 gunicorn==21.2.0 -redis==5.0.1 -aiohttp==3.8.5 -asyncio==3.4.3 \ No newline at end of file +redis==5.0.1 \ No newline at end of file diff --git a/server.py b/server.py index 1d89ca2..382c7ff 100644 --- a/server.py +++ b/server.py @@ -24,6 +24,7 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio +# TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: is frequency penalty the same as ooba repetition penalty??? # TODO: make sure openai_moderation_enabled works on websockets, completions, and chat completions # TODO: if a backend is at its limit of concurrent requests, choose a different one @@ -93,7 +94,6 @@ create_db() def home(): base_client_api = redis.get('base_client_api', dtype=str) stats = generate_stats() - model_choices, default_backend_info = get_model_choices() if default_backend_info['queued'] == 0 and default_backend_info['queued'] >= opts.concurrent_gens: -- 2.34.1 From aed5db4968d0466754ecdeb81f5112770d910222 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 2 Oct 2023 21:43:36 -0600 Subject: [PATCH 031/163] trying to narrow down error --- llm_server/database/database.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llm_server/database/database.py b/llm_server/database/database.py index fc1aa21..27a059c 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -11,6 +11,9 @@ from llm_server.llm import get_token_count def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): + assert isinstance(prompt, str) + assert isinstance(backend_url, str) + def background_task(): nonlocal ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, backend_url, response_tokens, is_error # Try not to shove JSON into the database. -- 2.34.1 From cd325216e2eaaeccb6df6763ccf4bd7ba5276a32 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 2 Oct 2023 22:45:07 -0600 Subject: [PATCH 032/163] test --- llm_server/llm/openai/oai_to_vllm.py | 1 + llm_server/routes/openai_request_handler.py | 3 +++ 2 files changed, 4 insertions(+) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 9111389..e224418 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -26,6 +26,7 @@ def oai_to_vllm(request_json_body, hashes: bool, mode): def format_oai_err(err_msg): + print('OAI ERROR MESSAGE:', err_msg) return jsonify({ "error": { "message": err_msg, diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 8664695..0dfd558 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -25,6 +25,7 @@ class OpenAIRequestHandler(RequestHandler): self.prompt = None def handle_request(self) -> Tuple[flask.Response, int]: + print('recieved request') assert not self.used if opts.openai_silent_trim: @@ -66,11 +67,13 @@ class OpenAIRequestHandler(RequestHandler): model = self.request_json_body.get('model') if success: + print('sent success response') return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code else: return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): + print('OAI ratelimited:', self.client_ip, self.request.headers) _, default_backend_info = get_model_choices() w = int(default_backend_info['estimated_wait']) if default_backend_info['estimated_wait'] > 0 else 2 response = jsonify({ -- 2.34.1 From 07d6f6d8e9f36895fc19e6b2fd1699eb9ee091a6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 00:03:39 -0600 Subject: [PATCH 033/163] test --- llm_server/routes/openai_request_handler.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 0dfd558..429dccd 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -39,6 +39,7 @@ class OpenAIRequestHandler(RequestHandler): return invalid_response if opts.openai_api_key and is_api_key_moderated(self.token): + print('moderating') try: # Gather the last message from the user and all preceeding system messages msg_l = self.request.json['messages'].copy() @@ -59,8 +60,10 @@ class OpenAIRequestHandler(RequestHandler): print(traceback.format_exc()) # TODO: support Ooba + print('converting to vllm') self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode']) + print('generating') llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) @@ -70,6 +73,7 @@ class OpenAIRequestHandler(RequestHandler): print('sent success response') return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code else: + print(backend_response) return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): -- 2.34.1 From f6acd67738acd46af021355d5b8ee91d8b7fd32b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 00:05:32 -0600 Subject: [PATCH 034/163] t --- llm_server/routes/openai_request_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 429dccd..1bf8b8b 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -39,7 +39,7 @@ class OpenAIRequestHandler(RequestHandler): return invalid_response if opts.openai_api_key and is_api_key_moderated(self.token): - print('moderating') + print('moderating', self.token) try: # Gather the last message from the user and all preceeding system messages msg_l = self.request.json['messages'].copy() -- 2.34.1 From 70126acdf2a7fe974b3fec4d82d9c1fff324ceb6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 00:12:13 -0600 Subject: [PATCH 035/163] test --- llm_server/workers/moderator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm_server/workers/moderator.py b/llm_server/workers/moderator.py index 4457d05..00e0303 100644 --- a/llm_server/workers/moderator.py +++ b/llm_server/workers/moderator.py @@ -24,6 +24,7 @@ def moderation_worker(): result = redis_moderation.blpop('queue:msgs_to_check') try: msg, tag = json.loads(result[1]) + print(tag) _, categories = check_moderation_endpoint(msg) redis_moderation.rpush('queue:flagged_categories', json.dumps((tag, categories))) except: -- 2.34.1 From 0f5e22191c0e737eae33e32b3192d628cdcf8e80 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 00:12:37 -0600 Subject: [PATCH 036/163] test --- llm_server/llm/openai/moderation.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llm_server/llm/openai/moderation.py b/llm_server/llm/openai/moderation.py index 53e234d..1677b37 100644 --- a/llm_server/llm/openai/moderation.py +++ b/llm_server/llm/openai/moderation.py @@ -14,6 +14,8 @@ def check_moderation_endpoint(prompt: str): response.raise_for_status() response = response.json() + print(response.text) + offending_categories = [] for k, v in response['results'][0]['categories'].items(): if v: -- 2.34.1 From 62eb0196cccde7d7880c9204acf1fee18bbe3c6d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 00:13:55 -0600 Subject: [PATCH 037/163] t --- llm_server/llm/openai/moderation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/llm/openai/moderation.py b/llm_server/llm/openai/moderation.py index 1677b37..045b79a 100644 --- a/llm_server/llm/openai/moderation.py +++ b/llm_server/llm/openai/moderation.py @@ -10,11 +10,11 @@ def check_moderation_endpoint(prompt: str): } response = requests.post('https://api.openai.com/v1/moderations', headers=headers, json={"input": prompt}, timeout=10) if response.status_code != 200: - print(response.text) + print(response) response.raise_for_status() response = response.json() - print(response.text) + print(response) offending_categories = [] for k, v in response['results'][0]['categories'].items(): -- 2.34.1 From ca1baa4870e7a85fec3e17aea37bc48388c3f1c0 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 00:15:16 -0600 Subject: [PATCH 038/163] test --- llm_server/workers/moderator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm_server/workers/moderator.py b/llm_server/workers/moderator.py index 00e0303..27ccb28 100644 --- a/llm_server/workers/moderator.py +++ b/llm_server/workers/moderator.py @@ -20,6 +20,7 @@ def start_moderation_workers(num_workers): def moderation_worker(): + print('moderator started') while True: result = redis_moderation.blpop('queue:msgs_to_check') try: -- 2.34.1 From 63c12ea83011850cf08864aaf894de23fd079385 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 01:25:43 -0600 Subject: [PATCH 039/163] fix --- llm_server/routes/openai_request_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 1bf8b8b..780e179 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -38,7 +38,7 @@ class OpenAIRequestHandler(RequestHandler): if not request_valid: return invalid_response - if opts.openai_api_key and is_api_key_moderated(self.token): + if opts.openai_moderation_enabled and opts.openai_api_key and is_api_key_moderated(self.token): print('moderating', self.token) try: # Gather the last message from the user and all preceeding system messages -- 2.34.1 From 32ad97e57cde768151b6eb1ed399c435e38b4fe1 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 13:40:08 -0600 Subject: [PATCH 040/163] do default model rather than default backend, adjust moderation endpoint logic and add timeout, exclude system tokens from recent proompters, calculate number of moderators from endpoint concurrent gens, adjust homepage --- llm_server/cluster/model_choices.py | 43 +++++---------------- llm_server/config/config.py | 9 +++-- llm_server/config/load.py | 11 +----- llm_server/opts.py | 6 +-- llm_server/routes/openai_request_handler.py | 6 +-- llm_server/routes/request_handler.py | 5 ++- llm_server/routes/v1/generate_stats.py | 4 +- llm_server/workers/moderator.py | 43 +++++++++++++-------- llm_server/workers/threader.py | 3 +- server.py | 25 ++++++------ templates/home.html | 10 ++++- 11 files changed, 78 insertions(+), 87 deletions(-) diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index 3df3aea..1aaefca 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -23,12 +23,14 @@ def get_model_choices(regen: bool = False): context_size = [] avg_gen_per_worker = [] + concurrent_gens = 0 for backend_url in b: backend_info = cluster_config.get_backend(backend_url) if backend_info.get('model_config'): context_size.append(backend_info['model_config']['max_position_embeddings']) if backend_info.get('average_generation_elapsed_sec'): avg_gen_per_worker.append(backend_info['average_generation_elapsed_sec']) + concurrent_gens += backend_info['concurrent_gens'] active_gen_workers = get_active_gen_workers_model(model) proompters_in_queue = priority_queue.len(model) @@ -37,16 +39,10 @@ def get_model_choices(regen: bool = False): average_generation_elapsed_sec = np.average(avg_gen_per_worker) else: average_generation_elapsed_sec = 0 - estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, opts.concurrent_gens, active_gen_workers) - - if proompters_in_queue == 0 and active_gen_workers >= opts.concurrent_gens: - # There will be a wait if the queue is empty but prompts are processing, but we don't - # know how long. - estimated_wait_sec = f"less than {estimated_wait_sec} seconds" - else: - estimated_wait_sec = f"{estimated_wait_sec} seconds" + estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, concurrent_gens, active_gen_workers) model_choices[model] = { + 'model': model, 'client_api': f'https://{base_client_api}/{model}', 'ws_client_api': f'wss://{base_client_api}/{model}/v1/stream' if opts.enable_streaming else None, 'openai_client_api': f'https://{base_client_api}/openai/{model}' if opts.enable_openi_compatible_backend else 'disabled', @@ -55,6 +51,7 @@ def get_model_choices(regen: bool = False): 'queued': proompters_in_queue, 'processing': active_gen_workers, 'avg_generation_time': average_generation_elapsed_sec, + 'concurrent_gens': concurrent_gens } if len(context_size): @@ -63,30 +60,8 @@ def get_model_choices(regen: bool = False): # Python wants to sort lowercase vs. uppercase letters differently. model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper())) - default_backend = get_a_cluster_backend() - default_backend_dict = {} - if default_backend: - default_backend_info = cluster_config.get_backend(default_backend) - default_context_size = default_backend_info['model_config']['max_position_embeddings'] - default_average_generation_elapsed_sec = default_backend_info.get('average_generation_elapsed_sec') - default_active_gen_workers = redis.get(f'active_gen_workers:{default_backend}', dtype=int, default=0) - default_proompters_in_queue = priority_queue.len(default_backend_info['model']) - default_estimated_wait_sec = calculate_wait_time(default_average_generation_elapsed_sec, default_proompters_in_queue, default_backend_info['concurrent_gens'], default_active_gen_workers) + default_backend_url = get_a_cluster_backend() + default_model = cluster_config.get_backend(default_backend_url)['model'] - default_backend_dict = { - 'client_api': f'https://{base_client_api}', - 'ws_client_api': f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else None, - 'openai_client_api': f'https://{base_client_api}/openai' if opts.enable_openi_compatible_backend else 'disabled', - 'estimated_wait': default_estimated_wait_sec, - 'queued': default_proompters_in_queue, - 'processing': default_active_gen_workers, - 'context_size': default_context_size, - 'hash': default_backend_info['hash'], - 'model': default_backend_info['model'], - 'avg_generation_time': default_average_generation_elapsed_sec, - 'online': True - } - - redis.setp('model_choices', (model_choices, default_backend_dict)) - - return model_choices, default_backend_dict + redis.setp('model_choices', (model_choices, default_model)) + return model_choices, default_model diff --git a/llm_server/config/config.py b/llm_server/config/config.py index 5308827..b33a9f2 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -1,6 +1,7 @@ import yaml config_default_vars = { + 'frontend_api_mode': 'ooba', 'log_prompts': False, 'database_path': './proxy-server.db', 'auth_required': False, @@ -28,19 +29,19 @@ config_default_vars = { 'openai_force_no_hashes': True, 'include_system_tokens_in_stats': True, 'openai_moderation_scan_last_n': 5, - 'openai_moderation_workers': 10, 'openai_org_name': 'OpenAI', 'openai_silent_trim': False, 'openai_moderation_enabled': True, 'netdata_root': None, 'show_backends': True, 'cluster_workers': 30, - 'background_homepage_cacher': True + 'background_homepage_cacher': True, + 'openai_moderation_timeout': 5 } -config_required_vars = ['cluster', 'mode', 'llm_middleware_name'] +config_required_vars = ['cluster', 'llm_middleware_name'] mode_ui_names = { - 'oobabooga': ('Text Gen WebUI (ooba)', 'Blocking API url', 'Streaming API url'), + 'ooba': ('Text Gen WebUI (ooba)', 'Blocking API url', 'Streaming API url'), 'vllm': ('Text Gen WebUI (ooba)', 'Blocking API url', 'Streaming API url'), } diff --git a/llm_server/config/load.py b/llm_server/config/load.py index edc5991..6f9db8d 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -16,15 +16,9 @@ def load_config(config_path): if not success: return success, config, msg - if config['mode'] not in ['oobabooga', 'vllm']: - print('Unknown mode:', config['mode']) - sys.exit(1) - # TODO: this is atrocious - opts.mode = config['mode'] opts.auth_required = config['auth_required'] opts.log_prompts = config['log_prompts'] - opts.concurrent_gens = config['concurrent_gens'] opts.frontend_api_client = config['frontend_api_client'] opts.show_num_prompts = config['show_num_prompts'] opts.show_uptime = config['show_uptime'] @@ -47,13 +41,14 @@ def load_config(config_path): opts.openai_force_no_hashes = config['openai_force_no_hashes'] opts.include_system_tokens_in_stats = config['include_system_tokens_in_stats'] opts.openai_moderation_scan_last_n = config['openai_moderation_scan_last_n'] - opts.openai_moderation_workers = config['openai_moderation_workers'] opts.openai_org_name = config['openai_org_name'] opts.openai_silent_trim = config['openai_silent_trim'] opts.openai_moderation_enabled = config['openai_moderation_enabled'] opts.show_backends = config['show_backends'] opts.cluster_workers = config['cluster_workers'] opts.background_homepage_cacher = config['background_homepage_cacher'] + opts.openai_moderation_timeout = config['openai_moderation_timeout'] + opts.frontend_api_mode = config['frontend_api_mode'] if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') @@ -75,8 +70,6 @@ def load_config(config_path): if config['load_num_prompts']: redis.set('proompts', get_number_of_rows('prompts')) - redis.set('backend_mode', opts.mode) - return success, config, msg diff --git a/llm_server/opts.py b/llm_server/opts.py index ae07ca4..38542a8 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -2,9 +2,7 @@ # TODO: rewrite the config system so I don't have to add every single config default here -concurrent_gens = 3 -mode = 'oobabooga' -backend_url = None +frontend_api_mode = 'ooba' max_new_tokens = 500 auth_required = False log_prompts = False @@ -31,7 +29,6 @@ openai_expose_our_model = False openai_force_no_hashes = True include_system_tokens_in_stats = True openai_moderation_scan_last_n = 5 -openai_moderation_workers = 10 openai_org_name = 'OpenAI' openai_silent_trim = False openai_moderation_enabled = True @@ -39,3 +36,4 @@ cluster = {} show_backends = True cluster_workers = 30 background_homepage_cacher = True +openai_moderation_timeout = 5 diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 780e179..835b575 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -41,7 +41,7 @@ class OpenAIRequestHandler(RequestHandler): if opts.openai_moderation_enabled and opts.openai_api_key and is_api_key_moderated(self.token): print('moderating', self.token) try: - # Gather the last message from the user and all preceeding system messages + # Gather the last message from the user and all preceding system messages msg_l = self.request.json['messages'].copy() msg_l.reverse() tag = uuid4() @@ -78,8 +78,8 @@ class OpenAIRequestHandler(RequestHandler): def handle_ratelimited(self, do_log: bool = True): print('OAI ratelimited:', self.client_ip, self.request.headers) - _, default_backend_info = get_model_choices() - w = int(default_backend_info['estimated_wait']) if default_backend_info['estimated_wait'] > 0 else 2 + _, default_model = get_model_choices() + w = int(default_model['estimated_wait']) if default_model['estimated_wait'] > 0 else 2 response = jsonify({ "error": { "message": "Rate limit reached on tokens per min. Limit: 10000 / min. Please try again in 6s. Contact us through our help center at help.openai.com if you continue to have issues.", diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 4e8b8e4..53be442 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -44,7 +44,10 @@ class RequestHandler: self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None self.used = False - redis.zadd('recent_prompters', {self.client_ip: time.time()}) + + if not self.token.startswith('SYSTEM__'): + # "recent_prompters" is only used for stats. + redis.zadd('recent_prompters', {self.client_ip: time.time()}) def get_auth_token(self): if self.request_json_body.get('X-API-KEY'): diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index 500f015..c4bc2c0 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -47,8 +47,8 @@ def generate_stats(regen: bool = False): 'timestamp': int(time.time()), 'config': { 'gatekeeper': 'none' if opts.auth_required is False else 'token', - 'concurrent': opts.concurrent_gens, 'simultaneous_requests_per_ip': opts.simultaneous_requests_per_ip, + 'api_mode': opts.frontend_api_mode }, 'keys': { 'openaiKeys': '∞', @@ -76,7 +76,7 @@ def generate_stats(regen: bool = False): else: output['backend_info'] = {} - output['default'] = get_model_choices(regen=True)[1] + output['default_model'] = get_model_choices(regen=True)[1] result = deep_sort(output) diff --git a/llm_server/workers/moderator.py b/llm_server/workers/moderator.py index 27ccb28..86c2da5 100644 --- a/llm_server/workers/moderator.py +++ b/llm_server/workers/moderator.py @@ -1,9 +1,11 @@ import json import threading +import time import traceback import redis as redis_redis +from llm_server import opts from llm_server.llm.openai.moderation import check_moderation_endpoint redis_moderation = redis_redis.Redis() @@ -19,10 +21,32 @@ def start_moderation_workers(num_workers): print(f'Started {i} moderation workers.') +# TODO: don't use UUID tags to identify items. Use native redis + +def get_results(tag, num_tasks): + tag = str(tag) # Cast a UUID4 to a string. + flagged_categories = set() + num_results = 0 + start_time = time.time() + while num_results < num_tasks: + result = redis_moderation.blpop(['queue:flagged_categories'], timeout=opts.openai_moderation_timeout) + if result is None: + break # Timeout occurred, break the loop. + result_tag, categories = json.loads(result[1]) + if result_tag == tag: + if categories: + for item in categories: + flagged_categories.add(item) + num_results += 1 + if time.time() - start_time > opts.openai_moderation_timeout: + print('Timed out waiting for result from moderator.') + break + return list(flagged_categories) + + def moderation_worker(): - print('moderator started') while True: - result = redis_moderation.blpop('queue:msgs_to_check') + result = redis_moderation.blpop(['queue:msgs_to_check']) try: msg, tag = json.loads(result[1]) print(tag) @@ -36,18 +60,3 @@ def moderation_worker(): def add_moderation_task(msg, tag): redis_moderation.rpush('queue:msgs_to_check', json.dumps((msg, str(tag)))) - - -def get_results(tag, num_tasks): - tag = str(tag) # Required for comparison with Redis results. - flagged_categories = set() - num_results = 0 - while num_results < num_tasks: - result = redis_moderation.blpop('queue:flagged_categories') - result_tag, categories = json.loads(result[1]) - if result_tag == tag: - if categories: - for item in categories: - flagged_categories.add(item) - num_results += 1 - return list(flagged_categories) diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index 0c82559..fa6c252 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -2,6 +2,7 @@ import time from threading import Thread from llm_server import opts +from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.stores import redis_running_models from llm_server.cluster.worker import cluster_worker from llm_server.routes.v1.generate_stats import generate_stats @@ -26,7 +27,7 @@ def start_background(): t.start() print('Started the main background thread.') - start_moderation_workers(opts.openai_moderation_workers) + start_moderation_workers(opts.cluster_workers * 3) t = Thread(target=cache_stats) t.daemon = True diff --git a/server.py b/server.py index 382c7ff..0eb0f6c 100644 --- a/server.py +++ b/server.py @@ -24,12 +24,14 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio +# TODO: make sure system tokens are excluded from 5/24 hr proompters # TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: is frequency penalty the same as ooba repetition penalty??? # TODO: make sure openai_moderation_enabled works on websockets, completions, and chat completions # TODO: if a backend is at its limit of concurrent requests, choose a different one # Lower priority +# TODO: fix moderation freezing after a while # TODO: support logit_bias on OpenAI and Ooba endpoints. # TODO: add a way to cancel VLLM gens. Maybe use websockets? # TODO: validate openai_silent_trim works as expected and only when enabled @@ -94,14 +96,15 @@ create_db() def home(): base_client_api = redis.get('base_client_api', dtype=str) stats = generate_stats() - model_choices, default_backend_info = get_model_choices() + model_choices, default_model = get_model_choices() + default_model_info = model_choices[default_model] - if default_backend_info['queued'] == 0 and default_backend_info['queued'] >= opts.concurrent_gens: + if default_model_info['queued'] == 0 and default_model_info['queued'] >= default_model_info['concurrent_gens']: # There will be a wait if the queue is empty but prompts are processing, but we don't # know how long. - default_estimated_wait_sec = f"less than {int(default_backend_info['estimated_wait'])} seconds" + default_estimated_wait_sec = f"less than {int(default_model_info['estimated_wait'])} seconds" else: - default_estimated_wait_sec = f"{int(default_backend_info['estimated_wait'])} seconds" + default_estimated_wait_sec = f"{int(default_model_info['estimated_wait'])} seconds" if len(config['analytics_tracking_code']): analytics_tracking_code = f"" @@ -123,17 +126,17 @@ def home(): llm_middleware_name=opts.llm_middleware_name, analytics_tracking_code=analytics_tracking_code, info_html=info_html, - default_model=default_backend_info['model'], - default_active_gen_workers=default_backend_info['processing'], - default_proompters_in_queue=default_backend_info['queued'], + default_model=default_model_info['model'], + default_active_gen_workers=default_model_info['processing'], + default_proompters_in_queue=default_model_info['queued'], current_model=opts.manual_model_name if opts.manual_model_name else None, # else running_model, client_api=f'https://{base_client_api}', ws_client_api=f'wss://{base_client_api}/v1/stream' if opts.enable_streaming else 'disabled', default_estimated_wait=default_estimated_wait_sec, - mode_name=mode_ui_names[opts.mode][0], - api_input_textbox=mode_ui_names[opts.mode][1], - streaming_input_textbox=mode_ui_names[opts.mode][2], - default_context_size=default_backend_info['context_size'], + mode_name=mode_ui_names[opts.frontend_api_mode][0], + api_input_textbox=mode_ui_names[opts.frontend_api_mode][1], + streaming_input_textbox=mode_ui_names[opts.frontend_api_mode][2], + default_context_size=default_model_info['context_size'], stats_json=json.dumps(stats, indent=4, ensure_ascii=False), extra_info=mode_info, openai_client_api=f'https://{base_client_api}/openai/v1' if opts.enable_openi_compatible_backend else 'disabled', diff --git a/templates/home.html b/templates/home.html index 3a020a4..66340a6 100644 --- a/templates/home.html +++ b/templates/home.html @@ -152,8 +152,16 @@ {% for key, value in model_choices.items() %}

        {{ key }} - {{ value.backend_count }} {% if value.backend_count == 1 %}worker{% else %}workers{% endif %}

        + + {% if value.estimated_wait == 0 and value.estimated_wait >= value.concurrent_gens %} + {# There will be a wait if the queue is empty but prompts are processing, but we don't know how long. #} + {% set estimated_wait_sec = "less than " + value.estimated_wait|int|string + " seconds" %} + {% else %} + {% set estimated_wait_sec = value.estimated_wait|int|string + " seconds" %} + {% endif %} +

        - Estimated Wait Time: {{ value.estimated_wait }}
        + Estimated Wait Time: {{ estimated_wait_sec }}
        Processing: {{ value.processing }}
        Queued: {{ value.queued }}

        -- 2.34.1 From 581a0fec994618284e91dbc3090c98e301bb65b0 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 13:47:18 -0600 Subject: [PATCH 041/163] fix exception --- llm_server/config/config.py | 3 +-- llm_server/database/database.py | 5 +++-- llm_server/llm/__init__.py | 3 ++- llm_server/routes/v1/generate_stream.py | 9 +++++---- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/llm_server/config/config.py b/llm_server/config/config.py index b33a9f2..11092c0 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -1,7 +1,6 @@ import yaml config_default_vars = { - 'frontend_api_mode': 'ooba', 'log_prompts': False, 'database_path': './proxy-server.db', 'auth_required': False, @@ -38,7 +37,7 @@ config_default_vars = { 'background_homepage_cacher': True, 'openai_moderation_timeout': 5 } -config_required_vars = ['cluster', 'llm_middleware_name'] +config_required_vars = ['cluster', 'frontend_api_mode', 'llm_middleware_name'] mode_ui_names = { 'ooba': ('Text Gen WebUI (ooba)', 'Blocking API url', 'Streaming API url'), diff --git a/llm_server/database/database.py b/llm_server/database/database.py index 27a059c..1cd5389 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -50,7 +50,8 @@ def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen if token: increment_token_uses(token) - running_model = cluster_config.get_backend(backend_url).get('model') + backend_info = cluster_config.get_backend(backend_url) + running_model = backend_info.get('model') timestamp = int(time.time()) cursor = database.cursor() try: @@ -59,7 +60,7 @@ def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, - (ip, token, running_model, opts.mode, backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) + (ip, token, running_model, cluster_config['mode'], backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) finally: cursor.close() diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index ba46635..09f1ad7 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -1,3 +1,4 @@ +from llm_server.cluster.cluster_config import cluster_config from llm_server.llm import oobabooga, vllm from llm_server.custom_redis import redis @@ -6,7 +7,7 @@ def get_token_count(prompt: str, backend_url: str): assert isinstance(prompt, str) assert isinstance(backend_url, str) - backend_mode = redis.get('backend_mode', dtype=str) + backend_mode = cluster_config.get_backend(backend_url)['mode'] if backend_mode == 'vllm': return vllm.tokenize(prompt, backend_url) elif backend_mode == 'ooba': diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index ac148dd..55fb6e4 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -73,15 +73,16 @@ def do_stream(ws, model_name): if not request_valid_json or not request_json_body.get('prompt'): return 'Invalid JSON', 400 else: - if opts.mode != 'vllm': - # TODO: implement other backends - raise NotImplementedError - auth_failure = require_api_key(request_json_body) if auth_failure: return auth_failure handler = OobaRequestHandler(incoming_request=request, selected_model=model_name, incoming_json=request_json_body) + + if handler.cluster_backend_info['mode'] != 'vllm': + # TODO: implement other backends + raise NotImplementedError + generated_text = '' input_prompt = request_json_body['prompt'] response_status_code = 0 -- 2.34.1 From e16f4157491a07d7aa2636276325fe6171a759b0 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 13:49:00 -0600 Subject: [PATCH 042/163] fix --- llm_server/database/database.py | 3 ++- llm_server/routes/request_handler.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llm_server/database/database.py b/llm_server/database/database.py index 1cd5389..f4e6c9c 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -52,6 +52,7 @@ def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen backend_info = cluster_config.get_backend(backend_url) running_model = backend_info.get('model') + backend_mode = backend_info['mode'] timestamp = int(time.time()) cursor = database.cursor() try: @@ -60,7 +61,7 @@ def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) """, - (ip, token, running_model, cluster_config['mode'], backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) + (ip, token, running_model, backend_mode, backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) finally: cursor.close() diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 53be442..b712d03 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -45,7 +45,7 @@ class RequestHandler: self.parameters = None self.used = False - if not self.token.startswith('SYSTEM__'): + if self.token and not self.token.startswith('SYSTEM__'): # "recent_prompters" is only used for stats. redis.zadd('recent_prompters', {self.client_ip: time.time()}) -- 2.34.1 From 33b4b8404b324b070b17df753929ccc312f2e531 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 14:10:50 -0600 Subject: [PATCH 043/163] clean up streaming --- llm_server/routes/v1/generate_stream.py | 73 ++++++------------------- server.py | 3 +- 2 files changed, 18 insertions(+), 58 deletions(-) diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 55fb6e4..6cd98c0 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -159,23 +159,10 @@ def do_stream(ws, model_name): })) except: # The has client closed the stream. - if request: + if response: # Cancel the backend? - request.close() - end_time = time.time() - elapsed_time = end_time - start_time - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=None - ) + response.close() + # used to log here return message_num += 1 @@ -184,22 +171,9 @@ def do_stream(ws, model_name): # If there is no more data, break the loop if not chunk: break - - end_time = time.time() - elapsed_time = end_time - start_time - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=None, - is_error=not response - ) + if response: + response.close() + # used to log here except: traceback.print_exc() generated_text = generated_text + '\n\n' + handler.handle_error('Encountered error while streaming.', 'exception')[0].json['results'][0]['text'] @@ -208,33 +182,19 @@ def do_stream(ws, model_name): 'message_num': message_num, 'text': generated_text })) - if request: - request.close() - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=None, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url, - response_tokens=None, - is_error=True - ) - return + # used to log here finally: # The worker incremented it, we'll decrement it. decrement_ip_count(handler.client_ip, 'processing_ips') decr_active_workers(handler.selected_model, handler.backend_url) - try: - ws.send(json.dumps({ - 'event': 'stream_end', - 'message_num': message_num - })) - except: - # The client closed the stream. + try: + ws.send(json.dumps({ + 'event': 'stream_end', + 'message_num': message_num + })) + except: + # The client closed the stream. + pass end_time = time.time() elapsed_time = end_time - start_time log_prompt(ip=handler.client_ip, @@ -246,8 +206,7 @@ def do_stream(ws, model_name): headers=r_headers, backend_response_code=response_status_code, request_url=r_url, - backend_url=handler.backend_url, - response_tokens=None + backend_url=handler.backend_url ) finally: try: diff --git a/server.py b/server.py index 0eb0f6c..040b8d9 100644 --- a/server.py +++ b/server.py @@ -24,7 +24,8 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio -# TODO: make sure system tokens are excluded from 5/24 hr proompters + +# TODO: redis SCAN vs KEYS?? # TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: is frequency penalty the same as ooba repetition penalty??? # TODO: make sure openai_moderation_enabled works on websockets, completions, and chat completions -- 2.34.1 From f88e2362c57c352e7529ebb0e3c04ba1e3d1ef3d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 20:01:28 -0600 Subject: [PATCH 044/163] remove some debug prints --- llm_server/llm/openai/moderation.py | 2 -- llm_server/workers/moderator.py | 3 +-- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/llm_server/llm/openai/moderation.py b/llm_server/llm/openai/moderation.py index 045b79a..827d04b 100644 --- a/llm_server/llm/openai/moderation.py +++ b/llm_server/llm/openai/moderation.py @@ -14,8 +14,6 @@ def check_moderation_endpoint(prompt: str): response.raise_for_status() response = response.json() - print(response) - offending_categories = [] for k, v in response['results'][0]['categories'].items(): if v: diff --git a/llm_server/workers/moderator.py b/llm_server/workers/moderator.py index 86c2da5..64ba527 100644 --- a/llm_server/workers/moderator.py +++ b/llm_server/workers/moderator.py @@ -39,7 +39,7 @@ def get_results(tag, num_tasks): flagged_categories.add(item) num_results += 1 if time.time() - start_time > opts.openai_moderation_timeout: - print('Timed out waiting for result from moderator.') + print('----> Timed out waiting for result from moderator.') break return list(flagged_categories) @@ -49,7 +49,6 @@ def moderation_worker(): result = redis_moderation.blpop(['queue:msgs_to_check']) try: msg, tag = json.loads(result[1]) - print(tag) _, categories = check_moderation_endpoint(msg) redis_moderation.rpush('queue:flagged_categories', json.dumps((tag, categories))) except: -- 2.34.1 From 67f5df9bb95f53a674beb70c06d25edfc0c3f4c6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 20:42:53 -0600 Subject: [PATCH 045/163] fix stats page --- llm_server/cluster/model_choices.py | 5 ++++- llm_server/routes/v1/generate_stats.py | 17 +++++++---------- server.py | 1 - 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index 1aaefca..4333dde 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -61,7 +61,10 @@ def get_model_choices(regen: bool = False): model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper())) default_backend_url = get_a_cluster_backend() - default_model = cluster_config.get_backend(default_backend_url)['model'] + default_backend_info = cluster_config.get_backend(default_backend_url) + if not default_backend_info.get('model'): + return None, None + default_model = default_backend_info['model'] redis.setp('model_choices', (model_choices, default_model)) return model_choices, default_model diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index c4bc2c0..c31f0f6 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -2,7 +2,6 @@ import time from datetime import datetime from llm_server import opts -from llm_server.cluster.backend import get_a_cluster_backend from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.model_choices import get_model_choices from llm_server.custom_redis import redis @@ -17,17 +16,17 @@ def generate_stats(regen: bool = False): if c: return c - default_backend_url = get_a_cluster_backend() - default_backend_info = cluster_config.get_backend(default_backend_url) - if not default_backend_info.get('mode'): - return + model_choices, default_model = get_model_choices(regen=True) + if not model_choices or not default_model: + return 'Please wait for Redis to be populated...' + base_client_api = redis.get('base_client_api', dtype=str) proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf')) output = { - 'default': { - 'model': default_backend_info['model'], - 'backend': default_backend_url, + 'models': { + 'choices': model_choices, + 'default': default_model, }, 'stats': { 'proompters': { @@ -76,8 +75,6 @@ def generate_stats(regen: bool = False): else: output['backend_info'] = {} - output['default_model'] = get_model_choices(regen=True)[1] - result = deep_sort(output) # It may take a bit to get the base client API, so don't cache until then. diff --git a/server.py b/server.py index 040b8d9..478a028 100644 --- a/server.py +++ b/server.py @@ -24,7 +24,6 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio - # TODO: redis SCAN vs KEYS?? # TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: is frequency penalty the same as ooba repetition penalty??? -- 2.34.1 From 1a7f22ec55459ec449fa0cb38929f8c98e8878ed Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 20:47:37 -0600 Subject: [PATCH 046/163] adjust again --- llm_server/routes/v1/generate_stats.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index c31f0f6..fccd3d1 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -53,7 +53,7 @@ def generate_stats(regen: bool = False): 'openaiKeys': '∞', 'anthropicKeys': '∞', }, - 'backend_info': redis.get_dict('backend_info') if opts.show_backend_info else None, + 'backends': {}, } # TODO: have get_model_choices() return all the info so we don't have to loop over the backends ourself @@ -64,7 +64,7 @@ def generate_stats(regen: bool = False): if not backend_info['online']: continue backend_uptime = int((datetime.now() - datetime.fromtimestamp(backend_info['startup_time'])).total_seconds()) if opts.show_uptime else None - output['backend_info'][backend_info['hash']] = { + output['backends'][backend_info['hash']] = { 'uptime': backend_uptime, 'max_tokens': backend_info['model_config']['max_position_embeddings'], 'model': backend_info['model'], @@ -72,8 +72,6 @@ def generate_stats(regen: bool = False): 'nvidia': backend_info['nvidia'], 'priority': backend_info['priority'], } - else: - output['backend_info'] = {} result = deep_sort(output) -- 2.34.1 From 6dc3529190fb62ab7966c4258b049ff56b63eca1 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 3 Oct 2023 23:39:25 -0600 Subject: [PATCH 047/163] show online status on stats page --- llm_server/routes/v1/generate_stats.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index fccd3d1..23902b7 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -54,6 +54,7 @@ def generate_stats(regen: bool = False): 'anthropicKeys': '∞', }, 'backends': {}, + 'online': len(model_choices) > 0 } # TODO: have get_model_choices() return all the info so we don't have to loop over the backends ourself -- 2.34.1 From 5f4e4710c16c6bcf1eb1044970cac676da1051e5 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 10:19:44 -0600 Subject: [PATCH 048/163] option to prioritize by parameter count --- llm_server/cluster/backend.py | 20 +++++++++++++++----- llm_server/config/config.py | 3 ++- llm_server/config/load.py | 1 + llm_server/opts.py | 1 + llm_server/routes/helpers/model.py | 13 +++++++++++++ server.py | 1 + 6 files changed, 33 insertions(+), 6 deletions(-) create mode 100644 llm_server/routes/helpers/model.py diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index 61061bb..cadf86e 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -1,8 +1,10 @@ +from llm_server import opts from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle from llm_server.cluster.stores import redis_running_models from llm_server.llm.generator import generator from llm_server.llm.info import get_info +from llm_server.routes.helpers.model import estimate_model_size def test_backend(backend_url: str, test_prompt: bool = False): @@ -34,11 +36,19 @@ def get_backends(): status = b.get('online', False) priority = b['priority'] result[k] = {'status': status, 'priority': priority} - online_backends = sorted( - ((url, info) for url, info in backends.items() if info['online']), - key=lambda kv: -kv[1]['priority'], - reverse=True - ) + + if not opts.prioritize_by_size: + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: -kv[1]['priority'], + reverse=True + ) + else: + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: estimate_model_size(kv[1]['model_config']), + reverse=True + ) offline_backends = sorted( ((url, info) for url, info in backends.items() if not info['online']), key=lambda kv: -kv[1]['priority'], diff --git a/llm_server/config/config.py b/llm_server/config/config.py index 11092c0..54eb3ec 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -35,7 +35,8 @@ config_default_vars = { 'show_backends': True, 'cluster_workers': 30, 'background_homepage_cacher': True, - 'openai_moderation_timeout': 5 + 'openai_moderation_timeout': 5, + 'prioritize_by_size': False } config_required_vars = ['cluster', 'frontend_api_mode', 'llm_middleware_name'] diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 6f9db8d..9a55a70 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -49,6 +49,7 @@ def load_config(config_path): opts.background_homepage_cacher = config['background_homepage_cacher'] opts.openai_moderation_timeout = config['openai_moderation_timeout'] opts.frontend_api_mode = config['frontend_api_mode'] + opts.prioritize_by_size = config['prioritize_by_size'] if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') diff --git a/llm_server/opts.py b/llm_server/opts.py index 38542a8..5c32f05 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -37,3 +37,4 @@ show_backends = True cluster_workers = 30 background_homepage_cacher = True openai_moderation_timeout = 5 +prioritize_by_size = False \ No newline at end of file diff --git a/llm_server/routes/helpers/model.py b/llm_server/routes/helpers/model.py new file mode 100644 index 0000000..ca35867 --- /dev/null +++ b/llm_server/routes/helpers/model.py @@ -0,0 +1,13 @@ +def estimate_model_size(config: dict): + """ + Estimate the size of a model from its config. No idea if this is correct, + but it allows us to compare models. + :param config: + :return: + """ + vocab_size = config['vocab_size'] + hidden_size = config['hidden_size'] + num_hidden_layers = config['num_hidden_layers'] + intermediate_size = config['intermediate_size'] + total_params = (vocab_size * hidden_size) + (num_hidden_layers * ((hidden_size * intermediate_size * 4) + (hidden_size * hidden_size * 3))) + return int(total_params / 1e9) diff --git a/server.py b/server.py index 478a028..560f15d 100644 --- a/server.py +++ b/server.py @@ -24,6 +24,7 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio +# TODO: what happens when all backends are offline? What about the "online" key in the stats page? # TODO: redis SCAN vs KEYS?? # TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: is frequency penalty the same as ooba repetition penalty??? -- 2.34.1 From b76e77a66a0c8309f5ec4660c50d1b5f19e707b5 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 10:24:28 -0600 Subject: [PATCH 049/163] fix exception --- llm_server/routes/openai_request_handler.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 835b575..468941f 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -78,8 +78,9 @@ class OpenAIRequestHandler(RequestHandler): def handle_ratelimited(self, do_log: bool = True): print('OAI ratelimited:', self.client_ip, self.request.headers) - _, default_model = get_model_choices() - w = int(default_model['estimated_wait']) if default_model['estimated_wait'] > 0 else 2 + model_choices, default_model = get_model_choices() + default_model_info = model_choices[default_model] + w = int(default_model_info['estimated_wait']) if default_model_info['estimated_wait'] > 0 else 2 response = jsonify({ "error": { "message": "Rate limit reached on tokens per min. Limit: 10000 / min. Please try again in 6s. Contact us through our help center at help.openai.com if you continue to have issues.", -- 2.34.1 From 4634e36eeb1c9f06e98454636d8a9d4e61d359c1 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 10:26:39 -0600 Subject: [PATCH 050/163] text --- llm_server/routes/openai_request_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 468941f..975bc59 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -57,7 +57,7 @@ class OpenAIRequestHandler(RequestHandler): self.prompt = transform_messages_to_prompt(self.request.json['messages']) except Exception as e: print(f'OpenAI moderation endpoint failed:', f'{e.__class__.__name__}: {e}') - print(traceback.format_exc()) + traceback.print_exc() # TODO: support Ooba print('converting to vllm') @@ -73,7 +73,7 @@ class OpenAIRequestHandler(RequestHandler): print('sent success response') return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code else: - print(backend_response) + print(backend_response_status_code, backend_response.data) return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): -- 2.34.1 From 7e3af3599d76cf8f317e1525d4bea9df3e327340 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 10:29:58 -0600 Subject: [PATCH 051/163] test --- llm_server/routes/queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index a8a47b1..8b8b85f 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -62,6 +62,8 @@ class RedisPriorityQueue: return self.redis.zcard('queue') def len(self, model_name): + print(self.redis.zrange('queue', 0, -1)) + count = 0 for key in self.redis.zrange('queue', 0, -1): item = json.loads(key) -- 2.34.1 From 4deb32bf1cfcbc7cc0f2ce383a32841c3a1d5219 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 10:32:11 -0600 Subject: [PATCH 052/163] test --- llm_server/llm/openai/moderation.py | 2 +- llm_server/routes/openai_request_handler.py | 1 - server.py | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/llm_server/llm/openai/moderation.py b/llm_server/llm/openai/moderation.py index 827d04b..f62241d 100644 --- a/llm_server/llm/openai/moderation.py +++ b/llm_server/llm/openai/moderation.py @@ -10,7 +10,7 @@ def check_moderation_endpoint(prompt: str): } response = requests.post('https://api.openai.com/v1/moderations', headers=headers, json={"input": prompt}, timeout=10) if response.status_code != 200: - print(response) + print('moderation failed:', response) response.raise_for_status() response = response.json() diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 975bc59..0b768d1 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -73,7 +73,6 @@ class OpenAIRequestHandler(RequestHandler): print('sent success response') return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code else: - print(backend_response_status_code, backend_response.data) return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): diff --git a/server.py b/server.py index 560f15d..a3074e6 100644 --- a/server.py +++ b/server.py @@ -1,3 +1,5 @@ +from llm_server.routes.queue import priority_queue + try: import gevent.monkey @@ -95,6 +97,8 @@ create_db() @app.route('/api/openai') @flask_cache.cached(timeout=10) def home(): + print(len(priority_queue)) + base_client_api = redis.get('base_client_api', dtype=str) stats = generate_stats() model_choices, default_model = get_model_choices() -- 2.34.1 From 1b21cb69c15016ebca8193569f427cf42d1cdd88 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:40:29 -0600 Subject: [PATCH 053/163] test --- llm_server/routes/queue.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 8b8b85f..b9be18d 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -26,7 +26,6 @@ class RedisPriorityQueue: self.pubsub.subscribe('events') def put(self, item, priority, selected_model): - event = DataEvent() # Check if the IP is already in the dictionary and if it has reached the limit ip_count = self.redis.hget('queued_ip_count', item[1]) if ip_count: @@ -36,6 +35,7 @@ class RedisPriorityQueue: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request + event = DataEvent() self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event @@ -46,7 +46,13 @@ class RedisPriorityQueue: if data: item = json.loads(data[0][0]) client_ip = item[0][1] + + b = self.redis.hget('queued_ip_count', item[1]) self.decrement_ip_count(client_ip, 'queued_ip_count') + a = self.redis.hget('queued_ip_count', item[1]) + + print(item[1], a, b) + return item time.sleep(0.1) # wait for something to be added to the queue @@ -56,20 +62,13 @@ class RedisPriorityQueue: def decrement_ip_count(self, client_ip: str, redis_key): new_count = self.redis.hincrby(redis_key, client_ip, -1) if new_count <= 0: - self.redis.hdel(redis_key, client_ip) + self.redis.hdel(redis_key, [client_ip]) def __len__(self): return self.redis.zcard('queue') def len(self, model_name): - print(self.redis.zrange('queue', 0, -1)) - - count = 0 - for key in self.redis.zrange('queue', 0, -1): - item = json.loads(key) - if item[2] == model_name: - count += 1 - return count + self.redis.zrange('queue', 0, -1) def get_queued_ip_count(self, client_ip: str): q = self.redis.hget('queued_ip_count', client_ip) -- 2.34.1 From 95d781725ef4562a1232950de3e73f74155c6ff8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:42:18 -0600 Subject: [PATCH 054/163] t --- llm_server/routes/queue.py | 2 +- llm_server/workers/moderator.py | 1 - server.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index b9be18d..e374311 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -50,7 +50,7 @@ class RedisPriorityQueue: b = self.redis.hget('queued_ip_count', item[1]) self.decrement_ip_count(client_ip, 'queued_ip_count') a = self.redis.hget('queued_ip_count', item[1]) - + print(item[1], a, b) return item diff --git a/llm_server/workers/moderator.py b/llm_server/workers/moderator.py index 64ba527..16ad0a2 100644 --- a/llm_server/workers/moderator.py +++ b/llm_server/workers/moderator.py @@ -52,7 +52,6 @@ def moderation_worker(): _, categories = check_moderation_endpoint(msg) redis_moderation.rpush('queue:flagged_categories', json.dumps((tag, categories))) except: - print(result) traceback.print_exc() continue diff --git a/server.py b/server.py index a3074e6..10c5b8c 100644 --- a/server.py +++ b/server.py @@ -97,8 +97,6 @@ create_db() @app.route('/api/openai') @flask_cache.cached(timeout=10) def home(): - print(len(priority_queue)) - base_client_api = redis.get('base_client_api', dtype=str) stats = generate_stats() model_choices, default_model = get_model_choices() -- 2.34.1 From a15b5465dfc7cc23ee179ae80c8c896866d9a4a0 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:44:09 -0600 Subject: [PATCH 055/163] c --- llm_server/routes/request_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index b712d03..d0990ea 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -39,7 +39,7 @@ class RequestHandler: self.selected_model = self.cluster_backend_info['model'] if not self.cluster_backend_info.get('mode'): - print(selected_model, self.backend_url, self.cluster_backend_info) + print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None -- 2.34.1 From f3a13fcda84ba2092210c99e1b3489a7953f5bde Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:44:33 -0600 Subject: [PATCH 056/163] c --- llm_server/routes/openai_request_handler.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 0b768d1..4b4740c 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -25,7 +25,6 @@ class OpenAIRequestHandler(RequestHandler): self.prompt = None def handle_request(self) -> Tuple[flask.Response, int]: - print('recieved request') assert not self.used if opts.openai_silent_trim: @@ -39,7 +38,6 @@ class OpenAIRequestHandler(RequestHandler): return invalid_response if opts.openai_moderation_enabled and opts.openai_api_key and is_api_key_moderated(self.token): - print('moderating', self.token) try: # Gather the last message from the user and all preceding system messages msg_l = self.request.json['messages'].copy() @@ -60,17 +58,14 @@ class OpenAIRequestHandler(RequestHandler): traceback.print_exc() # TODO: support Ooba - print('converting to vllm') self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode']) - print('generating') llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) model = self.request_json_body.get('model') if success: - print('sent success response') return self.build_openai_response(self.prompt, backend_response.json['results'][0]['text'], model=model), backend_response_status_code else: return backend_response, backend_response_status_code -- 2.34.1 From 6af5365015ccc5bff253f019cce7feface0c5aaf Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:45:20 -0600 Subject: [PATCH 057/163] c --- llm_server/routes/openai_request_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 4b4740c..69029be 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -71,7 +71,7 @@ class OpenAIRequestHandler(RequestHandler): return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): - print('OAI ratelimited:', self.client_ip, self.request.headers) + print('OAI ratelimited:', self.client_ip) model_choices, default_model = get_model_choices() default_model_info = model_choices[default_model] w = int(default_model_info['estimated_wait']) if default_model_info['estimated_wait'] > 0 else 2 -- 2.34.1 From 7cb624c5f513b80cb6e05e27eb55ccae7f2c5424 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:47:59 -0600 Subject: [PATCH 058/163] f --- llm_server/custom_redis.py | 2 +- llm_server/routes/queue.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index d5d278f..485cb58 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -112,7 +112,7 @@ class RedisCustom(Redis): def zcard(self, name: KeyT): return self.redis.zcard(self._key(name)) - def hdel(self, name: str, *keys: List): + def hdel(self, name: str, *keys: str): return self.redis.hdel(self._key(name), *keys) def hget( diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index e374311..89d35e6 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -62,7 +62,7 @@ class RedisPriorityQueue: def decrement_ip_count(self, client_ip: str, redis_key): new_count = self.redis.hincrby(redis_key, client_ip, -1) if new_count <= 0: - self.redis.hdel(redis_key, [client_ip]) + self.redis.hdel(redis_key, client_ip) def __len__(self): return self.redis.zcard('queue') -- 2.34.1 From 364b79526877ab7104e126280237f924f6f943d2 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:57:11 -0600 Subject: [PATCH 059/163] fix --- llm_server/routes/queue.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 89d35e6..a8a47b1 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -26,6 +26,7 @@ class RedisPriorityQueue: self.pubsub.subscribe('events') def put(self, item, priority, selected_model): + event = DataEvent() # Check if the IP is already in the dictionary and if it has reached the limit ip_count = self.redis.hget('queued_ip_count', item[1]) if ip_count: @@ -35,7 +36,6 @@ class RedisPriorityQueue: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request - event = DataEvent() self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event @@ -46,13 +46,7 @@ class RedisPriorityQueue: if data: item = json.loads(data[0][0]) client_ip = item[0][1] - - b = self.redis.hget('queued_ip_count', item[1]) self.decrement_ip_count(client_ip, 'queued_ip_count') - a = self.redis.hget('queued_ip_count', item[1]) - - print(item[1], a, b) - return item time.sleep(0.1) # wait for something to be added to the queue @@ -68,7 +62,12 @@ class RedisPriorityQueue: return self.redis.zcard('queue') def len(self, model_name): - self.redis.zrange('queue', 0, -1) + count = 0 + for key in self.redis.zrange('queue', 0, -1): + item = json.loads(key) + if item[2] == model_name: + count += 1 + return count def get_queued_ip_count(self, client_ip: str): q = self.redis.hget('queued_ip_count', client_ip) -- 2.34.1 From 77db34a6a7669605623a7dcbe09b2c856a8b1448 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 12:59:19 -0600 Subject: [PATCH 060/163] g --- llm_server/routes/request_handler.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index d0990ea..1361fb4 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -36,11 +36,13 @@ class RequestHandler: self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) - self.selected_model = self.cluster_backend_info['model'] if not self.cluster_backend_info.get('mode'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) + if not self.cluster_backend_info.get('model'): + print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) + self.selected_model = self.cluster_backend_info['model'] self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None self.used = False -- 2.34.1 From 6bad5b3fa0249a2c31c2c7aa7e19de50370aa6a8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:02:53 -0600 Subject: [PATCH 061/163] t --- llm_server/llm/openai/oai_to_vllm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index e224418..34dd8aa 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -7,6 +7,8 @@ def oai_to_vllm(request_json_body, hashes: bool, mode): if not request_json_body.get('stop'): request_json_body['stop'] = [] + print(request_json_body['stop']) + if hashes: request_json_body['stop'].extend(['### INSTRUCTION', '### USER', '### ASSISTANT', '### RESPONSE']) if opts.openai_force_no_hashes: -- 2.34.1 From d0eec88dbde9914c560a9d8b5dcb46830666b654 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:03:58 -0600 Subject: [PATCH 062/163] f --- llm_server/llm/openai/oai_to_vllm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 34dd8aa..cf039de 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -6,6 +6,9 @@ from llm_server import opts def oai_to_vllm(request_json_body, hashes: bool, mode): if not request_json_body.get('stop'): request_json_body['stop'] = [] + if not isinstance(request_json_body['stop'], list): + # It is a string, so create a list with the existing element + request_json_body['stop'] = [request_json_body['stop']] print(request_json_body['stop']) -- 2.34.1 From 754a4cbdf3acf2044c3391d1a1ae130af316535e Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:11:43 -0600 Subject: [PATCH 063/163] r --- llm_server/llm/openai/oai_to_vllm.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index cf039de..7d26467 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -7,11 +7,9 @@ def oai_to_vllm(request_json_body, hashes: bool, mode): if not request_json_body.get('stop'): request_json_body['stop'] = [] if not isinstance(request_json_body['stop'], list): - # It is a string, so create a list with the existing element + # It is a string, so create a list with the existing element. request_json_body['stop'] = [request_json_body['stop']] - print(request_json_body['stop']) - if hashes: request_json_body['stop'].extend(['### INSTRUCTION', '### USER', '### ASSISTANT', '### RESPONSE']) if opts.openai_force_no_hashes: -- 2.34.1 From 5e90fa54d4dfcefe9c87e3a4b9a8472182b97e48 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:18:47 -0600 Subject: [PATCH 064/163] handle model offline --- llm_server/routes/ooba_request_handler.py | 4 ++++ llm_server/routes/request_handler.py | 13 +++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 909848e..6944d57 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -15,6 +15,10 @@ class OobaRequestHandler(RequestHandler): def handle_request(self, return_ok: bool = True): assert not self.used + if self.offline: + msg = f'{self.selected_model} is not a valid model choice.' + print(msg) + return jsonify({'results': [{'text': format_sillytavern_err(msg)}]}), 200 request_valid, invalid_response = self.validate_request() if not request_valid: diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 1361fb4..fb02816 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -13,6 +13,7 @@ from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.auth import parse_token +from llm_server.routes.helpers.client import format_sillytavern_err from llm_server.routes.helpers.http import require_api_key, validate_json from llm_server.routes.queue import priority_queue @@ -42,6 +43,11 @@ class RequestHandler: if not self.cluster_backend_info.get('model'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) + if not self.cluster_backend_info.get('mode') or not self.cluster_backend_info.get('model'): + self.offline = True + else: + self.offline = False + self.selected_model = self.cluster_backend_info['model'] self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) self.parameters = None @@ -215,8 +221,11 @@ class RequestHandler: def handle_request(self) -> Tuple[flask.Response, int]: # Must include this in your child. - # if self.used: - # raise Exception('Can only use a RequestHandler object once.') + # assert not self.used + # if self.offline: + # msg = f'{self.selected_model} is not a valid model choice.' + # print(msg) + # return format_sillytavern_err(msg) raise NotImplementedError def handle_ratelimited(self, do_log: bool = True) -> Tuple[flask.Response, int]: -- 2.34.1 From d78ef652fcbafe4239ecf86e72ff13e5b109eea3 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:21:43 -0600 Subject: [PATCH 065/163] c --- llm_server/routes/ooba_request_handler.py | 2 +- llm_server/routes/request_handler.py | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 6944d57..133b370 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -18,7 +18,7 @@ class OobaRequestHandler(RequestHandler): if self.offline: msg = f'{self.selected_model} is not a valid model choice.' print(msg) - return jsonify({'results': [{'text': format_sillytavern_err(msg)}]}), 200 + self.handle_error(msg) request_valid, invalid_response = self.validate_request() if not request_valid: diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index fb02816..b8c2b81 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -13,7 +13,6 @@ from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.auth import parse_token -from llm_server.routes.helpers.client import format_sillytavern_err from llm_server.routes.helpers.http import require_api_key, validate_json from llm_server.routes.queue import priority_queue @@ -47,15 +46,13 @@ class RequestHandler: self.offline = True else: self.offline = False - - self.selected_model = self.cluster_backend_info['model'] - self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) - self.parameters = None - self.used = False - - if self.token and not self.token.startswith('SYSTEM__'): - # "recent_prompters" is only used for stats. - redis.zadd('recent_prompters', {self.client_ip: time.time()}) + self.selected_model = self.cluster_backend_info['model'] + self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) + self.parameters = None + self.used = False + if self.token and not self.token.startswith('SYSTEM__'): + # "recent_prompters" is only used for stats. + redis.zadd('recent_prompters', {self.client_ip: time.time()}) def get_auth_token(self): if self.request_json_body.get('X-API-KEY'): -- 2.34.1 From 7acaa3c885e4f418849138ba53a49c06065c96f8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:32:54 -0600 Subject: [PATCH 066/163] g --- llm_server/llm/vllm/tokenize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index d5a1b71..1e3c2a1 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -28,7 +28,6 @@ def tokenize(prompt: str, backend_url: str) -> int: return j['length'] except Exception as e: print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') - raise Exception return len(tokenizer.encode(chunk)) + 10 # Use a ThreadPoolExecutor to send all chunks to the server at once -- 2.34.1 From 62d5d43da49ea91c387d32e811a7ca4c0fa46289 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:34:59 -0600 Subject: [PATCH 067/163] handle backend offline in tokenizer --- llm_server/llm/vllm/tokenize.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 1e3c2a1..2df7789 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -4,6 +4,8 @@ import requests import tiktoken from llm_server import opts +from llm_server.cluster.backend import get_a_cluster_backend +from llm_server.cluster.cluster_config import cluster_config def tokenize(prompt: str, backend_url: str) -> int: @@ -11,6 +13,16 @@ def tokenize(prompt: str, backend_url: str) -> int: assert isinstance(prompt, str) assert isinstance(backend_url, str) + # TODO: put this in a shared function + # The backend could have died between when the request was + # submitted and now, so let's double check it's still online. + backend_info = cluster_config.get_backend(backend_url) + if not backend_info['online']: + old = backend_url + backend_url = get_a_cluster_backend() + print(f'Backend {old} offline. Request was redirected to {backend_url}') + del old # gc + if not prompt: # The tokenizers have issues when the prompt is None. return 0 -- 2.34.1 From 09fa69e031a857671d15638c5bd36e164aad6675 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 13:37:39 -0600 Subject: [PATCH 068/163] fix --- llm_server/routes/request_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index b8c2b81..d5d7175 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -36,6 +36,8 @@ class RequestHandler: self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + self.parameters = None + self.used = False if not self.cluster_backend_info.get('mode'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) @@ -48,8 +50,6 @@ class RequestHandler: self.offline = False self.selected_model = self.cluster_backend_info['model'] self.backend = get_backend_handler(self.cluster_backend_info['mode'], self.backend_url) - self.parameters = None - self.used = False if self.token and not self.token.startswith('SYSTEM__'): # "recent_prompters" is only used for stats. redis.zadd('recent_prompters', {self.client_ip: time.time()}) -- 2.34.1 From 6723dd79dc4c7254355219874824dd4bc740347d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 16:04:03 -0600 Subject: [PATCH 069/163] fix exceptoin --- llm_server/routes/ooba_request_handler.py | 2 +- server.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 133b370..350621f 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -16,7 +16,7 @@ class OobaRequestHandler(RequestHandler): def handle_request(self, return_ok: bool = True): assert not self.used if self.offline: - msg = f'{self.selected_model} is not a valid model choice.' + msg = 'The model you requested is not a valid choice. Please retry your query.' print(msg) self.handle_error(msg) diff --git a/server.py b/server.py index 10c5b8c..92d768d 100644 --- a/server.py +++ b/server.py @@ -1,5 +1,3 @@ -from llm_server.routes.queue import priority_queue - try: import gevent.monkey @@ -26,6 +24,7 @@ from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.sock import init_socketio +# TODO: return an `error: True`, error code, and error message rather than just a formatted message # TODO: what happens when all backends are offline? What about the "online" key in the stats page? # TODO: redis SCAN vs KEYS?? # TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) -- 2.34.1 From 167059490897fed585db72e956de215cf3377d9e Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 16:29:19 -0600 Subject: [PATCH 070/163] fix import error --- llm_server/cluster/backend.py | 150 ++++++++++---------- llm_server/cluster/cluster_config.py | 57 ++++++++ llm_server/cluster/model_choices.py | 69 --------- llm_server/cluster/redis_config_cache.py | 15 +- llm_server/cluster/worker.py | 2 +- llm_server/llm/vllm/tokenize.py | 9 +- llm_server/routes/openai/models.py | 3 +- llm_server/routes/openai_request_handler.py | 2 +- llm_server/routes/request_handler.py | 3 +- llm_server/routes/v1/generate_stats.py | 2 +- llm_server/routes/v1/info.py | 4 +- llm_server/routes/v1/proxy.py | 3 +- llm_server/workers/inferencer.py | 3 +- llm_server/workers/mainer.py | 3 +- server.py | 2 +- 15 files changed, 160 insertions(+), 167 deletions(-) diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index cadf86e..c301f93 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -1,10 +1,33 @@ +import numpy as np + from llm_server import opts -from llm_server.cluster.cluster_config import cluster_config -from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle +from llm_server.cluster.cluster_config import cluster_config, get_a_cluster_backend from llm_server.cluster.stores import redis_running_models +from llm_server.custom_redis import redis from llm_server.llm.generator import generator from llm_server.llm.info import get_info -from llm_server.routes.helpers.model import estimate_model_size +from llm_server.routes.queue import priority_queue +from llm_server.routes.stats import get_active_gen_workers_model, calculate_wait_time + + +def get_backends_from_model(model_name: str): + return [x.decode('utf-8') for x in redis_running_models.smembers(model_name)] + + +def get_running_models(): + return redis_running_models.keys() + + +def purge_backend_from_running_models(backend_url: str): + keys = redis_running_models.keys() + pipeline = redis_running_models.pipeline() + for model in keys: + pipeline.srem(model, backend_url) + pipeline.execute() + + +def is_valid_model(model_name: str): + return redis_running_models.exists(model_name) def test_backend(backend_url: str, test_prompt: bool = False): @@ -28,81 +51,64 @@ def test_backend(backend_url: str, test_prompt: bool = False): return True, i -def get_backends(): - backends = cluster_config.all() - result = {} - for k, v in backends.items(): - b = cluster_config.get_backend(k) - status = b.get('online', False) - priority = b['priority'] - result[k] = {'status': status, 'priority': priority} - - if not opts.prioritize_by_size: - online_backends = sorted( - ((url, info) for url, info in backends.items() if info['online']), - key=lambda kv: -kv[1]['priority'], - reverse=True - ) - else: - online_backends = sorted( - ((url, info) for url, info in backends.items() if info['online']), - key=lambda kv: estimate_model_size(kv[1]['model_config']), - reverse=True - ) - offline_backends = sorted( - ((url, info) for url, info in backends.items() if not info['online']), - key=lambda kv: -kv[1]['priority'], - reverse=True - ) - return [url for url, info in online_backends], [url for url, info in offline_backends] -def get_a_cluster_backend(model=None): - """ - Get a backend from Redis. If there are no online backends, return None. - If `model` is not supplied, we will pick one ourself. - """ - if model: - # First, determine if there are multiple backends hosting the same model. - backends_hosting_model = [i.decode('utf-8') for i in redis_running_models.smembers(model)] +def get_model_choices(regen: bool = False): + if not regen: + c = redis.getp('model_choices') + if c: + return c - # If so, create an iterator for those backends - if len(backends_hosting_model): - add_backend_cycler(model, backends_hosting_model) - cycled = redis_cycle(model) - if len(cycled): - return cycled[0] + base_client_api = redis.get('base_client_api', dtype=str) + running_models = get_running_models() + model_choices = {} + for model in running_models: + b = get_backends_from_model(model) + + context_size = [] + avg_gen_per_worker = [] + concurrent_gens = 0 + for backend_url in b: + backend_info = cluster_config.get_backend(backend_url) + if backend_info.get('model_config'): + context_size.append(backend_info['model_config']['max_position_embeddings']) + if backend_info.get('average_generation_elapsed_sec'): + avg_gen_per_worker.append(backend_info['average_generation_elapsed_sec']) + concurrent_gens += backend_info['concurrent_gens'] + + active_gen_workers = get_active_gen_workers_model(model) + proompters_in_queue = priority_queue.len(model) + + if len(avg_gen_per_worker): + average_generation_elapsed_sec = np.average(avg_gen_per_worker) else: - # No backend hosting that model - return None - else: - online, _ = get_backends() - if len(online): - return online[0] + average_generation_elapsed_sec = 0 + estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, concurrent_gens, active_gen_workers) + model_choices[model] = { + 'model': model, + 'client_api': f'https://{base_client_api}/{model}', + 'ws_client_api': f'wss://{base_client_api}/{model}/v1/stream' if opts.enable_streaming else None, + 'openai_client_api': f'https://{base_client_api}/openai/{model}' if opts.enable_openi_compatible_backend else 'disabled', + 'backend_count': len(b), + 'estimated_wait': estimated_wait_sec, + 'queued': proompters_in_queue, + 'processing': active_gen_workers, + 'avg_generation_time': average_generation_elapsed_sec, + 'concurrent_gens': concurrent_gens + } -def get_backends_from_model(model_name: str): - return [x.decode('utf-8') for x in redis_running_models.smembers(model_name)] + if len(context_size): + model_choices[model]['context_size'] = min(context_size) + # Python wants to sort lowercase vs. uppercase letters differently. + model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper())) -# def verify_context_size(model_name:str): -# b = get_backends_from_model(model_name) -# for backend_url in b: -# backend_info = cluster_config.get_backend(backend_url) -# backend_info.get() + default_backend_url = get_a_cluster_backend() + default_backend_info = cluster_config.get_backend(default_backend_url) + if not default_backend_info.get('model'): + return None, None + default_model = default_backend_info['model'] - -def get_running_models(): - return redis_running_models.keys() - - -def purge_backend_from_running_models(backend_url: str): - keys = redis_running_models.keys() - pipeline = redis_running_models.pipeline() - for model in keys: - pipeline.srem(model, backend_url) - pipeline.execute() - - -def is_valid_model(model_name: str): - return redis_running_models.exists(model_name) + redis.setp('model_choices', (model_choices, default_model)) + return model_choices, default_model diff --git a/llm_server/cluster/cluster_config.py b/llm_server/cluster/cluster_config.py index 14a6cb0..17e9b05 100644 --- a/llm_server/cluster/cluster_config.py +++ b/llm_server/cluster/cluster_config.py @@ -1,3 +1,60 @@ +from llm_server import opts from llm_server.cluster.redis_config_cache import RedisClusterStore +from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle +from llm_server.cluster.stores import redis_running_models +from llm_server.routes.helpers.model import estimate_model_size cluster_config = RedisClusterStore('cluster_config') + + +def get_backends(): + backends = cluster_config.all() + result = {} + for k, v in backends.items(): + b = cluster_config.get_backend(k) + status = b.get('online', False) + priority = b['priority'] + result[k] = {'status': status, 'priority': priority} + + if not opts.prioritize_by_size: + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: -kv[1]['priority'], + reverse=True + ) + else: + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: estimate_model_size(kv[1]['model_config']), + reverse=True + ) + offline_backends = sorted( + ((url, info) for url, info in backends.items() if not info['online']), + key=lambda kv: -kv[1]['priority'], + reverse=True + ) + return [url for url, info in online_backends], [url for url, info in offline_backends] + + +def get_a_cluster_backend(model=None): + """ + Get a backend from Redis. If there are no online backends, return None. + If `model` is not supplied, we will pick one ourself. + """ + if model: + # First, determine if there are multiple backends hosting the same model. + backends_hosting_model = [i.decode('utf-8') for i in redis_running_models.smembers(model)] + + # If so, create an iterator for those backends + if len(backends_hosting_model): + add_backend_cycler(model, backends_hosting_model) + cycled = redis_cycle(model) + if len(cycled): + return cycled[0] + else: + # No backend hosting that model + return None + else: + online, _ = get_backends() + if len(online): + return online[0] diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py index 4333dde..ef93bba 100644 --- a/llm_server/cluster/model_choices.py +++ b/llm_server/cluster/model_choices.py @@ -1,70 +1 @@ -import numpy as np - -from llm_server import opts -from llm_server.cluster.backend import get_a_cluster_backend, get_backends_from_model, get_running_models -from llm_server.cluster.cluster_config import cluster_config -from llm_server.custom_redis import redis -from llm_server.routes.queue import priority_queue -from llm_server.routes.stats import calculate_wait_time, get_active_gen_workers_model - - # TODO: give this a better name! -def get_model_choices(regen: bool = False): - if not regen: - c = redis.getp('model_choices') - if c: - return c - - base_client_api = redis.get('base_client_api', dtype=str) - running_models = get_running_models() - model_choices = {} - for model in running_models: - b = get_backends_from_model(model) - - context_size = [] - avg_gen_per_worker = [] - concurrent_gens = 0 - for backend_url in b: - backend_info = cluster_config.get_backend(backend_url) - if backend_info.get('model_config'): - context_size.append(backend_info['model_config']['max_position_embeddings']) - if backend_info.get('average_generation_elapsed_sec'): - avg_gen_per_worker.append(backend_info['average_generation_elapsed_sec']) - concurrent_gens += backend_info['concurrent_gens'] - - active_gen_workers = get_active_gen_workers_model(model) - proompters_in_queue = priority_queue.len(model) - - if len(avg_gen_per_worker): - average_generation_elapsed_sec = np.average(avg_gen_per_worker) - else: - average_generation_elapsed_sec = 0 - estimated_wait_sec = calculate_wait_time(average_generation_elapsed_sec, proompters_in_queue, concurrent_gens, active_gen_workers) - - model_choices[model] = { - 'model': model, - 'client_api': f'https://{base_client_api}/{model}', - 'ws_client_api': f'wss://{base_client_api}/{model}/v1/stream' if opts.enable_streaming else None, - 'openai_client_api': f'https://{base_client_api}/openai/{model}' if opts.enable_openi_compatible_backend else 'disabled', - 'backend_count': len(b), - 'estimated_wait': estimated_wait_sec, - 'queued': proompters_in_queue, - 'processing': active_gen_workers, - 'avg_generation_time': average_generation_elapsed_sec, - 'concurrent_gens': concurrent_gens - } - - if len(context_size): - model_choices[model]['context_size'] = min(context_size) - - # Python wants to sort lowercase vs. uppercase letters differently. - model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper())) - - default_backend_url = get_a_cluster_backend() - default_backend_info = cluster_config.get_backend(default_backend_url) - if not default_backend_info.get('model'): - return None, None - default_model = default_backend_info['model'] - - redis.setp('model_choices', (model_choices, default_model)) - return model_choices, default_model diff --git a/llm_server/cluster/redis_config_cache.py b/llm_server/cluster/redis_config_cache.py index 3bab915..da0b581 100644 --- a/llm_server/cluster/redis_config_cache.py +++ b/llm_server/cluster/redis_config_cache.py @@ -45,5 +45,16 @@ class RedisClusterStore: else: return {} - # def get(self, name: str): - # return self.all().get(name) + def validate_backend(self, backend_url: str): + """ + Returns the backend URL that was given, or a new one if that was offline. + :param backend_url: + :return: + """ + backend_info = self.get_backend(backend_url) + if not backend_info['online']: + old = backend_url + backend_url = get_a_cluster_backend() + print(f'Backend {old} offline. Request was redirected to {backend_url}') + return backend_url + diff --git a/llm_server/cluster/worker.py b/llm_server/cluster/worker.py index 7956198..951f18d 100644 --- a/llm_server/cluster/worker.py +++ b/llm_server/cluster/worker.py @@ -1,8 +1,8 @@ import time from threading import Thread -from llm_server.cluster.backend import test_backend from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.backend import test_backend from llm_server.cluster.stores import redis_running_models diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 2df7789..db2c49d 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -4,7 +4,6 @@ import requests import tiktoken from llm_server import opts -from llm_server.cluster.backend import get_a_cluster_backend from llm_server.cluster.cluster_config import cluster_config @@ -13,15 +12,9 @@ def tokenize(prompt: str, backend_url: str) -> int: assert isinstance(prompt, str) assert isinstance(backend_url, str) - # TODO: put this in a shared function # The backend could have died between when the request was # submitted and now, so let's double check it's still online. - backend_info = cluster_config.get_backend(backend_url) - if not backend_info['online']: - old = backend_url - backend_url = get_a_cluster_backend() - print(f'Backend {old} offline. Request was redirected to {backend_url}') - del old # gc + backend_url = cluster_config.validate_backend(backend_url) if not prompt: # The tokenizers have issues when the prompt is None. diff --git a/llm_server/routes/openai/models.py b/llm_server/routes/openai/models.py index 9f2845d..2ff0629 100644 --- a/llm_server/routes/openai/models.py +++ b/llm_server/routes/openai/models.py @@ -7,8 +7,7 @@ from llm_server.custom_redis import ONE_MONTH_SECONDS, flask_cache, redis from . import openai_bp from ..stats import server_start_time from ... import opts -from ...cluster.backend import get_a_cluster_backend -from ...cluster.cluster_config import cluster_config +from ...cluster.cluster_config import cluster_config, get_a_cluster_backend from ...helpers import jsonify_pretty from ...llm.openai.transform import generate_oai_string diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 69029be..84b2c76 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -9,7 +9,7 @@ import flask from flask import Response, jsonify, make_response from llm_server import opts -from llm_server.cluster.model_choices import get_model_choices +from llm_server.cluster.backend import get_model_choices from llm_server.custom_redis import redis from llm_server.database.database import is_api_key_moderated, log_prompt from llm_server.llm import get_token_count diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index d5d7175..d981be8 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -5,8 +5,7 @@ import flask from flask import Response, request from llm_server import opts -from llm_server.cluster.backend import get_a_cluster_backend -from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.cluster_config import cluster_config, get_a_cluster_backend from llm_server.custom_redis import redis from llm_server.database.database import get_token_ratelimit, log_prompt from llm_server.helpers import auto_set_base_client_api diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index 23902b7..3bfbca6 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -3,7 +3,7 @@ from datetime import datetime from llm_server import opts from llm_server.cluster.cluster_config import cluster_config -from llm_server.cluster.model_choices import get_model_choices +from llm_server.cluster.backend import get_model_choices from llm_server.custom_redis import redis from llm_server.database.database import get_distinct_ips_24h, sum_column from llm_server.helpers import deep_sort diff --git a/llm_server/routes/v1/info.py b/llm_server/routes/v1/info.py index 6e37720..342921e 100644 --- a/llm_server/routes/v1/info.py +++ b/llm_server/routes/v1/info.py @@ -5,8 +5,8 @@ from flask import jsonify, request from llm_server.custom_redis import flask_cache from . import bp from ... import opts -from ...cluster.backend import get_a_cluster_backend, get_backends_from_model, is_valid_model -from ...cluster.cluster_config import cluster_config +from ...cluster.backend import get_backends_from_model, is_valid_model +from ...cluster.cluster_config import cluster_config, get_a_cluster_backend @bp.route('/v1/model', methods=['GET']) diff --git a/llm_server/routes/v1/proxy.py b/llm_server/routes/v1/proxy.py index e5ff5d3..6e3708e 100644 --- a/llm_server/routes/v1/proxy.py +++ b/llm_server/routes/v1/proxy.py @@ -4,8 +4,7 @@ from llm_server.custom_redis import flask_cache from . import bp from .generate_stats import generate_stats from ..auth import requires_auth -from ...cluster.backend import get_backends -from ...cluster.cluster_config import cluster_config +from ...cluster.cluster_config import cluster_config, get_backends from ...helpers import jsonify_pretty diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index c5eb12a..84faceb 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,8 +1,7 @@ import threading import time -from llm_server.cluster.backend import get_a_cluster_backend -from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.cluster_config import cluster_config, get_a_cluster_backend from llm_server.custom_redis import redis from llm_server.llm.generator import generator from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, priority_queue diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py index 580060d..37c1178 100644 --- a/llm_server/workers/mainer.py +++ b/llm_server/workers/mainer.py @@ -3,8 +3,7 @@ import time import requests from llm_server import opts -from llm_server.cluster.backend import get_backends -from llm_server.cluster.cluster_config import cluster_config +from llm_server.cluster.cluster_config import cluster_config, get_backends from llm_server.custom_redis import redis from llm_server.database.database import weighted_average_column_for_model from llm_server.llm.info import get_info diff --git a/server.py b/server.py index 92d768d..8d723ab 100644 --- a/server.py +++ b/server.py @@ -13,7 +13,7 @@ import simplejson as json from flask import Flask, jsonify, render_template, request from llm_server.cluster.cluster_config import cluster_config -from llm_server.cluster.model_choices import get_model_choices +from llm_server.cluster.backend import get_model_choices from llm_server.config.config import mode_ui_names from llm_server.config.load import load_config from llm_server.database.conn import database -- 2.34.1 From acf409abfc8fdf566603fd98b554a10c04060834 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 4 Oct 2023 19:24:47 -0600 Subject: [PATCH 071/163] fix background logger, add gradio chat example --- llm_server/database/database.py | 94 +++++++++----------- llm_server/database/log_to_db.py | 27 ++++++ llm_server/llm/oobabooga/ooba_backend.py | 8 +- llm_server/llm/openai/oai_to_vllm.py | 15 +++- llm_server/llm/openai/transform.py | 2 +- llm_server/llm/vllm/generate.py | 8 -- llm_server/llm/vllm/vllm_backend.py | 11 +-- llm_server/routes/ooba_request_handler.py | 5 +- llm_server/routes/openai/chat_completions.py | 23 +++-- llm_server/routes/openai/completions.py | 9 +- llm_server/routes/openai_request_handler.py | 9 +- llm_server/routes/request_handler.py | 37 ++++---- llm_server/routes/v1/generate_stream.py | 67 +++++++------- llm_server/workers/logger.py | 28 ++++++ llm_server/workers/threader.py | 7 +- other/gradio_chat.py | 33 +++++++ 16 files changed, 242 insertions(+), 141 deletions(-) create mode 100644 llm_server/database/log_to_db.py create mode 100644 llm_server/workers/logger.py create mode 100644 other/gradio_chat.py diff --git a/llm_server/database/database.py b/llm_server/database/database.py index f4e6c9c..fc800a2 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -1,7 +1,6 @@ import json import time import traceback -from threading import Thread from typing import Union from llm_server import opts @@ -10,65 +9,60 @@ from llm_server.database.conn import database from llm_server.llm import get_token_count -def log_prompt(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): +def do_db_log(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): assert isinstance(prompt, str) assert isinstance(backend_url, str) - def background_task(): - nonlocal ip, token, prompt, response, gen_time, parameters, headers, backend_response_code, request_url, backend_url, response_tokens, is_error - # Try not to shove JSON into the database. - if isinstance(response, dict) and response.get('results'): - response = response['results'][0]['text'] - try: - j = json.loads(response) - if j.get('results'): - response = j['results'][0]['text'] - except: - pass + # Try not to shove JSON into the database. + if isinstance(response, dict) and response.get('results'): + response = response['results'][0]['text'] + try: + j = json.loads(response) + if j.get('results'): + response = j['results'][0]['text'] + except: + pass - prompt_tokens = get_token_count(prompt, backend_url) - if not is_error: - if not response_tokens: - response_tokens = get_token_count(response, backend_url) - else: - response_tokens = None + prompt_tokens = get_token_count(prompt, backend_url) + print('starting') - # Sometimes we may want to insert null into the DB, but - # usually we want to insert a float. - if gen_time: - gen_time = round(gen_time, 3) - if is_error: - gen_time = None + if not is_error: + if not response_tokens: + response_tokens = get_token_count(response, backend_url) + else: + response_tokens = None - if not opts.log_prompts: - prompt = None + # Sometimes we may want to insert null into the DB, but + # usually we want to insert a float. + if gen_time: + gen_time = round(gen_time, 3) + if is_error: + gen_time = None - if not opts.log_prompts and not is_error: - # TODO: test and verify this works as expected - response = None + if not opts.log_prompts: + prompt = None - if token: - increment_token_uses(token) + if not opts.log_prompts and not is_error: + # TODO: test and verify this works as expected + response = None - backend_info = cluster_config.get_backend(backend_url) - running_model = backend_info.get('model') - backend_mode = backend_info['mode'] - timestamp = int(time.time()) - cursor = database.cursor() - try: - cursor.execute(""" - INSERT INTO prompts - (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) - """, - (ip, token, running_model, backend_mode, backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) - finally: - cursor.close() + if token: + increment_token_uses(token) - # TODO: use async/await instead of threads - thread = Thread(target=background_task) - thread.start() - thread.join() + backend_info = cluster_config.get_backend(backend_url) + running_model = backend_info.get('model') + backend_mode = backend_info['mode'] + timestamp = int(time.time()) + cursor = database.cursor() + try: + cursor.execute(""" + INSERT INTO prompts + (ip, token, model, backend_mode, backend_url, request_url, generation_time, prompt, prompt_tokens, response, response_tokens, response_status, parameters, headers, timestamp) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + """, + (ip, token, running_model, backend_mode, backend_url, request_url, gen_time, prompt, prompt_tokens, response, response_tokens, backend_response_code, json.dumps(parameters), json.dumps(headers), timestamp)) + finally: + cursor.close() def is_valid_api_key(api_key): diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py new file mode 100644 index 0000000..fa97ad7 --- /dev/null +++ b/llm_server/database/log_to_db.py @@ -0,0 +1,27 @@ +import pickle +from typing import Union + +from redis import Redis + + +def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): + r = Redis(host='localhost', port=6379, db=3) + data = { + 'function': 'log_prompt', + 'args': [], + 'kwargs': { + 'ip': ip, + 'token': token, + 'prompt': prompt, + 'response': response, + 'gen_time': gen_time, + 'parameters': parameters, + 'headers': headers, + 'backend_response_code': backend_response_code, + 'request_url': request_url, + 'backend_url': backend_url, + 'response_tokens': response_tokens, + 'is_error': is_error + } + } + r.publish('database-logger', pickle.dumps(data)) diff --git a/llm_server/llm/oobabooga/ooba_backend.py b/llm_server/llm/oobabooga/ooba_backend.py index fe450bf..0e2b2d8 100644 --- a/llm_server/llm/oobabooga/ooba_backend.py +++ b/llm_server/llm/oobabooga/ooba_backend.py @@ -2,7 +2,7 @@ from flask import jsonify from llm_server.custom_redis import redis from ..llm_backend import LLMBackend -from ...database.database import log_prompt +from ...database.database import do_db_log from ...helpers import safe_list_get from ...routes.helpers.client import format_sillytavern_err from ...routes.helpers.http import validate_json @@ -34,7 +34,7 @@ class OobaboogaBackend(LLMBackend): else: error_msg = error_msg.strip('.') + '.' backend_response = format_sillytavern_err(error_msg, error_type='error', backend_url=self.backend_url) - log_prompt(client_ip, token, prompt, backend_response, None, parameters, headers, response_status_code, request.url, is_error=True) + log_to_db(client_ip, token, prompt, backend_response, None, parameters, headers, response_status_code, request.url, is_error=True) return jsonify({ 'code': 500, 'msg': error_msg, @@ -57,13 +57,13 @@ class OobaboogaBackend(LLMBackend): if not backend_err: redis.incr('proompts') - log_prompt(client_ip, token, prompt, backend_response, elapsed_time if not backend_err else None, parameters, headers, response_status_code, request.url, response_tokens=response_json_body.get('details', {}).get('generated_tokens'), is_error=backend_err) + log_to_db(client_ip, token, prompt, backend_response, elapsed_time if not backend_err else None, parameters, headers, response_status_code, request.url, response_tokens=response_json_body.get('details', {}).get('generated_tokens'), is_error=backend_err) return jsonify({ **response_json_body }), 200 else: backend_response = format_sillytavern_err(f'The backend did not return valid JSON.', error_type='error', backend_url=self.backend_url) - log_prompt(client_ip, token, prompt, backend_response, elapsed_time, parameters, headers, response.status_code, request.url, is_error=True) + log_to_db(client_ip, token, prompt, backend_response, elapsed_time, parameters, headers, response.status_code, request.url, is_error=True) return jsonify({ 'code': 500, 'msg': 'the backend did not return valid JSON', diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 7d26467..35c9f30 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -3,17 +3,19 @@ from flask import jsonify from llm_server import opts -def oai_to_vllm(request_json_body, hashes: bool, mode): +def oai_to_vllm(request_json_body, stop_hashes: bool, mode): if not request_json_body.get('stop'): request_json_body['stop'] = [] if not isinstance(request_json_body['stop'], list): # It is a string, so create a list with the existing element. request_json_body['stop'] = [request_json_body['stop']] - if hashes: - request_json_body['stop'].extend(['### INSTRUCTION', '### USER', '### ASSISTANT', '### RESPONSE']) + if stop_hashes: if opts.openai_force_no_hashes: - request_json_body['stop'].append('### ') + request_json_body['stop'].append('###') + else: + # TODO: make stopping strings a configurable + request_json_body['stop'].extend(['### INSTRUCTION', '### USER', '### ASSISTANT']) else: request_json_body['stop'].extend(['user:', 'assistant:']) @@ -41,6 +43,11 @@ def format_oai_err(err_msg): def validate_oai(parameters): + if parameters.get('messages'): + for m in parameters['messages']: + if m['role'].lower() not in ['assistant', 'user', 'system']: + return format_oai_err('messages role must be assistant, user, or system') + if parameters.get('temperature', 0) > 2: return format_oai_err(f"{parameters['temperature']} is greater than the maximum of 2 - 'temperature'") if parameters.get('temperature', 0) < 0: diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index 39f942a..0c2946b 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -96,7 +96,7 @@ def transform_messages_to_prompt(oai_messages): elif msg['role'] == 'assistant': prompt += f'### ASSISTANT: {msg["content"]}\n\n' else: - return False + raise Exception(f'Unknown role: {msg["role"]}') except Exception as e: # TODO: use logging traceback.print_exc() diff --git a/llm_server/llm/vllm/generate.py b/llm_server/llm/vllm/generate.py index 72b0243..31cd511 100644 --- a/llm_server/llm/vllm/generate.py +++ b/llm_server/llm/vllm/generate.py @@ -1,24 +1,16 @@ """ This file is used by the worker that processes requests. """ -import json -import time -from uuid import uuid4 import requests -import llm_server from llm_server import opts -from llm_server.custom_redis import redis # TODO: make the VLMM backend return TPS and time elapsed # https://github.com/vllm-project/vllm/blob/main/vllm/entrypoints/openai/api_server.py def prepare_json(json_data: dict): - # logit_bias is not currently supported - # del json_data['logit_bias'] - # Convert back to VLLM. json_data['max_tokens'] = json_data.pop('max_new_tokens') return json_data diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index abc1cbb..a9ec821 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -3,7 +3,7 @@ from typing import Tuple from flask import jsonify from vllm import SamplingParams -from llm_server.database.database import log_prompt +from llm_server.database.log_to_db import log_to_db from llm_server.llm.llm_backend import LLMBackend @@ -18,8 +18,8 @@ class VLLMBackend(LLMBackend): # Failsafe backend_response = '' - log_prompt(ip=client_ip, token=token, prompt=prompt, response=backend_response, gen_time=elapsed_time, parameters=parameters, headers=headers, backend_response_code=response_status_code, request_url=request.url, - response_tokens=response_json_body.get('details', {}).get('generated_tokens'), backend_url=self.backend_url) + log_to_db(ip=client_ip, token=token, prompt=prompt, response=backend_response, gen_time=elapsed_time, parameters=parameters, headers=headers, backend_response_code=response_status_code, request_url=request.url, + response_tokens=response_json_body.get('details', {}).get('generated_tokens'), backend_url=self.backend_url) return jsonify({'results': [{'text': backend_response}]}), 200 @@ -29,14 +29,15 @@ class VLLMBackend(LLMBackend): top_k = parameters.get('top_k', self._default_params['top_k']) if top_k <= 0: top_k = -1 + sampling_params = SamplingParams( temperature=parameters.get('temperature', self._default_params['temperature']), top_p=parameters.get('top_p', self._default_params['top_p']), top_k=top_k, use_beam_search=True if parameters.get('num_beams', 0) > 1 else False, - stop=list(set(parameters.get('stopping_strings', self._default_params['stop']) or parameters.get('stop', self._default_params['stop']))), + stop=list(set(parameters.get('stopping_strings') or parameters.get('stop', self._default_params['stop']))), ignore_eos=parameters.get('ban_eos_token', False), - max_tokens=parameters.get('max_new_tokens', self._default_params['max_tokens']) or parameters.get('max_tokens', self._default_params['max_tokens']), + max_tokens=parameters.get('max_new_tokens') or parameters.get('max_tokens', self._default_params['max_tokens']), presence_penalty=parameters.get('presence_penalty', self._default_params['presence_penalty']), frequency_penalty=parameters.get('frequency_penalty', self._default_params['frequency_penalty']) ) diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 350621f..c01bfed 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -4,7 +4,8 @@ import flask from flask import jsonify, request from llm_server import opts -from llm_server.database.database import log_prompt +from llm_server.database.database import do_db_log +from llm_server.database.log_to_db import log_to_db from llm_server.routes.helpers.client import format_sillytavern_err from llm_server.routes.request_handler import RequestHandler @@ -40,7 +41,7 @@ class OobaRequestHandler(RequestHandler): msg = f'Ratelimited: you are only allowed to have {opts.simultaneous_requests_per_ip} simultaneous requests at a time. Please complete your other requests before sending another.' backend_response = self.handle_error(msg) if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) + log_to_db(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) return backend_response[0], 429 def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index c46e89f..d10bdf6 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -3,6 +3,7 @@ import time import traceback from flask import Response, jsonify, request +from redis import Redis from llm_server.custom_redis import redis from . import openai_bp @@ -10,7 +11,7 @@ from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts -from ...database.database import log_prompt +from ...database.log_to_db import log_to_db from ...llm.generator import generator from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit @@ -18,6 +19,7 @@ from ...llm.openai.transform import generate_oai_string, transform_messages_to_p # TODO: add rate-limit headers? + @openai_bp.route('/chat/completions', methods=['POST']) def openai_chat_completions(): request_valid_json, request_json_body = validate_json(request) @@ -36,12 +38,20 @@ def openai_chat_completions(): return 'Internal server error', 500 else: if not opts.enable_streaming: - return 'DISABLED', 401 + return + + handler.parameters, _ = handler.get_parameters() + handler.request_json_body = { + 'messages': handler.request_json_body['messages'], + 'model': handler.request_json_body['model'], + **handler.parameters + } invalid_oai_err_msg = validate_oai(handler.request_json_body) if invalid_oai_err_msg: return invalid_oai_err_msg - handler.request_json_body = oai_to_vllm(handler.request_json_body, hashes=False, mode=handler.cluster_backend_info['mode']) + + handler.request_json_body = oai_to_vllm(handler.request_json_body, stop_hashes=True, mode=handler.cluster_backend_info['mode']) if opts.openai_silent_trim: handler.prompt = transform_messages_to_prompt(trim_messages_to_fit(handler.request.json['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url)) @@ -64,7 +74,7 @@ def openai_chat_completions(): # Add a dummy event to the queue and wait for it to reach a worker event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: - log_prompt( + log_to_db( handler.client_ip, handler.token, handler.prompt, @@ -82,7 +92,6 @@ def openai_chat_completions(): _, _, _ = event.wait() try: - response = generator(msg_to_backend, handler.backend_url) r_headers = dict(request.headers) r_url = request.url model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') @@ -90,6 +99,7 @@ def openai_chat_completions(): def generate(): try: + response = generator(msg_to_backend, handler.backend_url) generated_text = '' partial_response = b'' for chunk in response.iter_content(chunk_size=1): @@ -125,8 +135,7 @@ def openai_chat_completions(): yield 'data: [DONE]\n\n' end_time = time.time() elapsed_time = end_time - start_time - - log_prompt( + log_to_db( handler.client_ip, handler.token, handler.prompt, diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 6904348..1843226 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -10,7 +10,8 @@ from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts -from ...database.database import log_prompt +from ...database.database import do_db_log +from ...database.log_to_db import log_to_db from ...llm import get_token_count from ...llm.generator import generator from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai @@ -34,7 +35,7 @@ def openai_completions(): invalid_oai_err_msg = validate_oai(handler.request_json_body) if invalid_oai_err_msg: return invalid_oai_err_msg - handler.request_json_body = oai_to_vllm(handler.request_json_body, hashes=False, mode=handler.cluster_backend_info['mode']) + handler.request_json_body = oai_to_vllm(handler.request_json_body, stop_hashes=False, mode=handler.cluster_backend_info['mode']) if opts.openai_silent_trim: handler.request_json_body['prompt'] = trim_string_to_fit(request_json_body['prompt'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) @@ -102,7 +103,7 @@ def openai_completions(): # Add a dummy event to the queue and wait for it to reach a worker event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: - log_prompt( + log_to_db( handler.client_ip, handler.token, handler.prompt, @@ -164,7 +165,7 @@ def openai_completions(): end_time = time.time() elapsed_time = end_time - start_time - log_prompt( + log_to_db( handler.client_ip, handler.token, handler.prompt, diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 84b2c76..037de27 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -11,7 +11,8 @@ from flask import Response, jsonify, make_response from llm_server import opts from llm_server.cluster.backend import get_model_choices from llm_server.custom_redis import redis -from llm_server.database.database import is_api_key_moderated, log_prompt +from llm_server.database.database import is_api_key_moderated, do_db_log +from llm_server.database.log_to_db import log_to_db from llm_server.llm import get_token_count from llm_server.llm.openai.oai_to_vllm import oai_to_vllm, validate_oai from llm_server.llm.openai.transform import ANTI_CONTINUATION_RE, ANTI_RESPONSE_RE, generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit @@ -58,7 +59,7 @@ class OpenAIRequestHandler(RequestHandler): traceback.print_exc() # TODO: support Ooba - self.parameters = oai_to_vllm(self.parameters, hashes=True, mode=self.cluster_backend_info['mode']) + self.parameters = oai_to_vllm(self.parameters, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) @@ -88,7 +89,7 @@ class OpenAIRequestHandler(RequestHandler): response.headers['x-ratelimit-reset-requests'] = f"{w}s" if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), response.data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) + log_to_db(self.client_ip, self.token, self.request_json_body.get('prompt', ''), response.data.decode('utf-8'), None, self.parameters, dict(self.request.headers), 429, self.request.url, self.backend_url, is_error=True) return response, 429 @@ -146,6 +147,6 @@ class OpenAIRequestHandler(RequestHandler): invalid_oai_err_msg = validate_oai(self.request_json_body) if invalid_oai_err_msg: return False, invalid_oai_err_msg - self.request_json_body = oai_to_vllm(self.request_json_body, hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) + self.request_json_body = oai_to_vllm(self.request_json_body, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) # If the parameters were invalid, let the superclass deal with it. return super().validate_request(prompt, do_log) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index d981be8..90da0b1 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -7,7 +7,8 @@ from flask import Response, request from llm_server import opts from llm_server.cluster.cluster_config import cluster_config, get_a_cluster_backend from llm_server.custom_redis import redis -from llm_server.database.database import get_token_ratelimit, log_prompt +from llm_server.database.database import get_token_ratelimit, do_db_log +from llm_server.database.log_to_db import log_to_db from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend from llm_server.llm.vllm.vllm_backend import VLLMBackend @@ -41,9 +42,11 @@ class RequestHandler: if not self.cluster_backend_info.get('mode'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) if not self.cluster_backend_info.get('model'): - print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) + print('keyerror: model -', selected_model, self.backend_url, self.cluster_backend_info) + if not self.cluster_backend_info.get('model_config'): + print('keyerror: model_config -', selected_model, self.backend_url, self.cluster_backend_info) - if not self.cluster_backend_info.get('mode') or not self.cluster_backend_info.get('model'): + if not self.cluster_backend_info.get('mode') or not self.cluster_backend_info.get('model') or not self.cluster_backend_info.get('model_config'): self.offline = True else: self.offline = False @@ -74,8 +77,6 @@ class RequestHandler: return self.request.remote_addr def get_parameters(self): - if self.request_json_body.get('max_tokens'): - self.request_json_body['max_new_tokens'] = self.request_json_body.pop('max_tokens') parameters, parameters_invalid_msg = self.backend.get_parameters(self.request_json_body) return parameters, parameters_invalid_msg @@ -117,7 +118,7 @@ class RequestHandler: backend_response = self.handle_error(combined_error_message, 'Validation Error') if do_log: - log_prompt(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), 0, self.parameters, dict(self.request.headers), 0, self.request.url, self.backend_url, is_error=True) + log_to_db(self.client_ip, self.token, self.request_json_body.get('prompt', ''), backend_response[0].data.decode('utf-8'), 0, self.parameters, dict(self.request.headers), 0, self.request.url, self.backend_url, is_error=True) return False, backend_response return True, (None, 0) @@ -160,17 +161,17 @@ class RequestHandler: else: error_msg = error_msg.strip('.') + '.' backend_response = self.handle_error(error_msg) - log_prompt(ip=self.client_ip, - token=self.token, - prompt=prompt, - response=backend_response[0].data.decode('utf-8'), - gen_time=None, - parameters=self.parameters, - headers=dict(self.request.headers), - backend_response_code=response_status_code, - request_url=self.request.url, - backend_url=self.backend_url, - is_error=True) + log_to_db(ip=self.client_ip, + token=self.token, + prompt=prompt, + response=backend_response[0].data.decode('utf-8'), + gen_time=None, + parameters=self.parameters, + headers=dict(self.request.headers), + backend_response_code=response_status_code, + request_url=self.request.url, + backend_url=self.backend_url, + is_error=True) return (False, None, None, 0), backend_response # =============================================== @@ -190,7 +191,7 @@ class RequestHandler: if return_json_err: error_msg = 'The backend did not return valid JSON.' backend_response = self.handle_error(error_msg) - log_prompt(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), elapsed_time, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.backend_url, is_error=True) + log_to_db(self.client_ip, self.token, prompt, backend_response[0].data.decode('utf-8'), elapsed_time, self.parameters, dict(self.request.headers), response_status_code, self.request.url, self.backend_url, is_error=True) return (False, None, None, 0), backend_response # =============================================== diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 6cd98c0..9962ff8 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -9,7 +9,8 @@ from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts -from ...database.database import log_prompt +from ...database.database import do_db_log +from ...database.log_to_db import log_to_db from ...llm.generator import generator from ...sock import sock @@ -34,38 +35,38 @@ def stream_with_model(ws, model_name=None): def do_stream(ws, model_name): - def send_err_and_quit(quitting_err_msg): - ws.send(json.dumps({ - 'event': 'text_stream', - 'message_num': 0, - 'text': quitting_err_msg - })) - ws.send(json.dumps({ - 'event': 'stream_end', - 'message_num': 1 - })) - log_prompt(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=quitting_err_msg, - gen_time=None, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.cluster_backend_info, - response_tokens=None, - is_error=True - ) - - if not opts.enable_streaming: - return 'Streaming is disabled', 500 - - r_headers = dict(request.headers) - r_url = request.url - message_num = 0 - try: + def send_err_and_quit(quitting_err_msg): + ws.send(json.dumps({ + 'event': 'text_stream', + 'message_num': 0, + 'text': quitting_err_msg + })) + ws.send(json.dumps({ + 'event': 'stream_end', + 'message_num': 1 + })) + log_to_db(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=quitting_err_msg, + gen_time=None, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.cluster_backend_info, + response_tokens=None, + is_error=True + ) + + if not opts.enable_streaming: + return 'Streaming is disabled', 500 + + r_headers = dict(request.headers) + r_url = request.url + message_num = 0 + while ws.connected: message = ws.receive() request_valid_json, request_json_body = validate_json(message) @@ -197,7 +198,7 @@ def do_stream(ws, model_name): pass end_time = time.time() elapsed_time = end_time - start_time - log_prompt(ip=handler.client_ip, + log_to_db(ip=handler.client_ip, token=handler.token, prompt=input_prompt, response=generated_text, diff --git a/llm_server/workers/logger.py b/llm_server/workers/logger.py new file mode 100644 index 0000000..2707615 --- /dev/null +++ b/llm_server/workers/logger.py @@ -0,0 +1,28 @@ +import pickle + +import redis + +from llm_server.database.database import do_db_log + + +def db_logger(): + """ + We don't want the logging operation to be blocking, so we will use a background worker + to do the logging. + :return: + """ + + r = redis.Redis(host='localhost', port=6379, db=3) + p = r.pubsub() + p.subscribe('database-logger') + + for message in p.listen(): + if message['type'] == 'message': + data = pickle.loads(message['data']) + function_name = data['function'] + args = data['args'] + kwargs = data['kwargs'] + + if function_name == 'log_prompt': + do_db_log(*args, **kwargs) + print('finished log') diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index fa6c252..bf14d60 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -2,11 +2,11 @@ import time from threading import Thread from llm_server import opts -from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.stores import redis_running_models from llm_server.cluster.worker import cluster_worker from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.inferencer import start_workers +from llm_server.workers.logger import db_logger from llm_server.workers.mainer import main_background_thread from llm_server.workers.moderator import start_moderation_workers from llm_server.workers.printer import console_printer @@ -49,3 +49,8 @@ def start_background(): t.daemon = True t.start() print('Started the cluster worker.') + + t = Thread(target=db_logger) + t.daemon = True + t.start() + print('Started background logger') diff --git a/other/gradio_chat.py b/other/gradio_chat.py new file mode 100644 index 0000000..eb10d26 --- /dev/null +++ b/other/gradio_chat.py @@ -0,0 +1,33 @@ +import warnings + +import gradio as gr +import openai + +warnings.filterwarnings("ignore") + +openai.api_key = 'null' +openai.api_base = 'http://localhost:5000/api/openai/v1' + + +def stream_response(prompt, history): + messages = [] + for x in history: + messages.append({'role': 'user', 'content': x[0]}) + messages.append({'role': 'assistant', 'content': x[1]}) + messages.append({'role': 'user', 'content': prompt}) + + response = openai.ChatCompletion.create( + model='0', + messages=messages, + temperature=0, + max_tokens=300, + stream=True + ) + + message = '' + for chunk in response: + message += chunk['choices'][0]['delta']['content'] + yield message + + +gr.ChatInterface(stream_response, examples=["hello", "hola", "merhaba"], title="Chatbot Demo", analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}').queue().launch() -- 2.34.1 From 08df52a4fd16f3cff19c8c63cc83b0feed3b5931 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 12:28:00 -0600 Subject: [PATCH 072/163] fix exception when not valid model --- llm_server/database/database.py | 1 - llm_server/routes/request_handler.py | 5 ++- llm_server/routes/v1/generate_stream.py | 56 +++++++++++++++---------- llm_server/workers/logger.py | 1 - 4 files changed, 36 insertions(+), 27 deletions(-) diff --git a/llm_server/database/database.py b/llm_server/database/database.py index fc800a2..d6bd6b2 100644 --- a/llm_server/database/database.py +++ b/llm_server/database/database.py @@ -24,7 +24,6 @@ def do_db_log(ip: str, token: str, prompt: str, response: Union[str, None], gen_ pass prompt_tokens = get_token_count(prompt, backend_url) - print('starting') if not is_error: if not response_tokens: diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 90da0b1..ef5aa34 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -34,11 +34,12 @@ class RequestHandler: self.client_ip = self.get_client_ip() self.token = self.get_auth_token() self.token_priority, self.token_simultaneous_ip = get_token_ratelimit(self.token) - self.backend_url = get_a_cluster_backend(selected_model) - self.cluster_backend_info = cluster_config.get_backend(self.backend_url) self.parameters = None self.used = False + self.backend_url = get_a_cluster_backend(selected_model) + self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + if not self.cluster_backend_info.get('mode'): print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) if not self.cluster_backend_info.get('model'): diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 9962ff8..55fceb9 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -9,7 +9,6 @@ from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import decr_active_workers, decrement_ip_count, priority_queue from ... import opts -from ...database.database import do_db_log from ...database.log_to_db import log_to_db from ...llm.generator import generator from ...sock import sock @@ -47,18 +46,18 @@ def do_stream(ws, model_name): 'message_num': 1 })) log_to_db(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=quitting_err_msg, - gen_time=None, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.cluster_backend_info, - response_tokens=None, - is_error=True - ) + token=handler.token, + prompt=input_prompt, + response=quitting_err_msg, + gen_time=None, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.cluster_backend_info, + response_tokens=None, + is_error=True + ) if not opts.enable_streaming: return 'Streaming is disabled', 500 @@ -79,6 +78,17 @@ def do_stream(ws, model_name): return auth_failure handler = OobaRequestHandler(incoming_request=request, selected_model=model_name, incoming_json=request_json_body) + if handler.offline: + msg = f'{handler.selected_model} is not a valid model choice.' + print(msg) + ws.send(json.dumps({ + 'event': 'text_stream', + 'message_num': 0, + 'text': msg + })) + return + + assert not handler.offline if handler.cluster_backend_info['mode'] != 'vllm': # TODO: implement other backends @@ -199,16 +209,16 @@ def do_stream(ws, model_name): end_time = time.time() elapsed_time = end_time - start_time log_to_db(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url - ) + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url + ) finally: try: # Must close the connection or greenlets will complain. diff --git a/llm_server/workers/logger.py b/llm_server/workers/logger.py index 2707615..cb8dc01 100644 --- a/llm_server/workers/logger.py +++ b/llm_server/workers/logger.py @@ -25,4 +25,3 @@ def db_logger(): if function_name == 'log_prompt': do_db_log(*args, **kwargs) - print('finished log') -- 2.34.1 From 27e461c76b575c02b288291c8558b4acf2bedbe9 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 17:00:35 -0600 Subject: [PATCH 073/163] test --- llm_server/routes/queue.py | 5 +++++ llm_server/workers/printer.py | 2 ++ llm_server/workers/threader.py | 2 +- 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index a8a47b1..b86e664 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -50,6 +50,11 @@ class RedisPriorityQueue: return item time.sleep(0.1) # wait for something to be added to the queue + def print_all_items(self): + items = self.redis.zrange('queue', 0, -1) + for item in items: + print(item.decode('utf-8')) + def increment_ip_count(self, client_ip: str, redis_key): self.redis.hincrby(redis_key, client_ip, 1) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index ed6ff65..dcd4e4b 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -25,4 +25,6 @@ def console_printer(): processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') + priority_queue.print_all_items() + print('============================') time.sleep(10) diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index bf14d60..dbdc8e0 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -53,4 +53,4 @@ def start_background(): t = Thread(target=db_logger) t.daemon = True t.start() - print('Started background logger') + print('Started background logger.') -- 2.34.1 From 19e62be3e85068dfbf40287a4ce4754697ad9467 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 17:41:01 -0600 Subject: [PATCH 074/163] t --- llm_server/routes/queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index b86e664..e4c49c5 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -52,6 +52,7 @@ class RedisPriorityQueue: def print_all_items(self): items = self.redis.zrange('queue', 0, -1) + print(items) for item in items: print(item.decode('utf-8')) -- 2.34.1 From 979a9454664a61d890280ba4af745521ee267ba5 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 17:55:44 -0600 Subject: [PATCH 075/163] t --- llm_server/database/log_to_db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index fa97ad7..e971747 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -24,4 +24,5 @@ def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_ 'is_error': is_error } } + print(data) r.publish('database-logger', pickle.dumps(data)) -- 2.34.1 From 84c1ed87375d96bbf6988edebf92669028c2497d Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 17:58:23 -0600 Subject: [PATCH 076/163] t --- llm_server/database/log_to_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index e971747..76712ff 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -24,5 +24,6 @@ def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_ 'is_error': is_error } } - print(data) + for k, v in data['kwargs'].items(): + print(k, type(v)) r.publish('database-logger', pickle.dumps(data)) -- 2.34.1 From a53790ee378ff1617749bdf926ead5bbfe1f3ea6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:01:08 -0600 Subject: [PATCH 077/163] fix??? --- llm_server/database/log_to_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index 76712ff..8e27055 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -16,7 +16,7 @@ def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_ 'response': response, 'gen_time': gen_time, 'parameters': parameters, - 'headers': headers, + 'headers': dict(headers) if headers else headers, 'backend_response_code': backend_response_code, 'request_url': request_url, 'backend_url': backend_url, -- 2.34.1 From a229b4d6c554ca9e5aa4e75876af2154edeba160 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:04:29 -0600 Subject: [PATCH 078/163] c --- llm_server/database/log_to_db.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index 8e27055..364e1d3 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -5,6 +5,9 @@ from redis import Redis def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): + assert isinstance(prompt, str) + assert isinstance(backend_url, str) + r = Redis(host='localhost', port=6379, db=3) data = { 'function': 'log_prompt', -- 2.34.1 From 01fb619b9b77c760e03c1841978017e1e75e5795 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:05:31 -0600 Subject: [PATCH 079/163] f --- server.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/server.py b/server.py index 8d723ab..f604763 100644 --- a/server.py +++ b/server.py @@ -99,6 +99,10 @@ def home(): base_client_api = redis.get('base_client_api', dtype=str) stats = generate_stats() model_choices, default_model = get_model_choices() + + if not model_choices.get(default_model): + return 'The server is still starting up. Please wait...' + default_model_info = model_choices[default_model] if default_model_info['queued'] == 0 and default_model_info['queued'] >= default_model_info['concurrent_gens']: -- 2.34.1 From 3d0a5cf0a2fb25e12cabd7201b4a87c695fdd09a Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:06:36 -0600 Subject: [PATCH 080/163] t --- llm_server/llm/__init__.py | 3 +++ server.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 09f1ad7..7736684 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -4,6 +4,9 @@ from llm_server.custom_redis import redis def get_token_count(prompt: str, backend_url: str): + if not isinstance(prompt, str): + print(prompt) + assert isinstance(prompt, str) assert isinstance(backend_url, str) diff --git a/server.py b/server.py index f604763..5966bfb 100644 --- a/server.py +++ b/server.py @@ -12,8 +12,8 @@ from pathlib import Path import simplejson as json from flask import Flask, jsonify, render_template, request -from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.backend import get_model_choices +from llm_server.cluster.cluster_config import cluster_config from llm_server.config.config import mode_ui_names from llm_server.config.load import load_config from llm_server.database.conn import database -- 2.34.1 From 5a61bdccd4843fa7e166b9067c9e4a5df4cfaa5b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:07:59 -0600 Subject: [PATCH 081/163] f --- llm_server/llm/__init__.py | 10 +++++----- llm_server/llm/vllm/tokenize.py | 9 +++++---- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 7736684..5f87c61 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -1,15 +1,15 @@ from llm_server.cluster.cluster_config import cluster_config from llm_server.llm import oobabooga, vllm -from llm_server.custom_redis import redis def get_token_count(prompt: str, backend_url: str): - if not isinstance(prompt, str): - print(prompt) - - assert isinstance(prompt, str) assert isinstance(backend_url, str) + if not prompt: + # The tokenizers have issues when the prompt is None. + return 0 + assert isinstance(prompt, str) + backend_mode = cluster_config.get_backend(backend_url)['mode'] if backend_mode == 'vllm': return vllm.tokenize(prompt, backend_url) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index db2c49d..8b18073 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -9,16 +9,17 @@ from llm_server.cluster.cluster_config import cluster_config def tokenize(prompt: str, backend_url: str) -> int: assert backend_url - assert isinstance(prompt, str) assert isinstance(backend_url, str) + if not prompt: + # The tokenizers have issues when the prompt is None. + return 0 + assert isinstance(prompt, str) + # The backend could have died between when the request was # submitted and now, so let's double check it's still online. backend_url = cluster_config.validate_backend(backend_url) - if not prompt: - # The tokenizers have issues when the prompt is None. - return 0 tokenizer = tiktoken.get_encoding("cl100k_base") # Split the prompt into 300 character chunks -- 2.34.1 From 64d7a9edbb4e108579fae9da74c1e395b4b1919f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:09:24 -0600 Subject: [PATCH 082/163] fix --- server.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/server.py b/server.py index 5966bfb..37c254b 100644 --- a/server.py +++ b/server.py @@ -12,16 +12,22 @@ from pathlib import Path import simplejson as json from flask import Flask, jsonify, render_template, request +import config +from llm_server import opts from llm_server.cluster.backend import get_model_choices from llm_server.cluster.cluster_config import cluster_config from llm_server.config.config import mode_ui_names from llm_server.config.load import load_config +from llm_server.custom_redis import flask_cache, redis from llm_server.database.conn import database from llm_server.database.create import create_db +from llm_server.helpers import auto_set_base_client_api +from llm_server.llm.vllm.info import vllm_info from llm_server.pre_fork import server_startup from llm_server.routes.openai import openai_bp from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp +from llm_server.routes.v1.generate_stats import generate_stats from llm_server.sock import init_socketio # TODO: return an `error: True`, error code, and error message rather than just a formatted message @@ -60,14 +66,6 @@ except ModuleNotFoundError as e: print('Please see README.md for install instructions.') sys.exit(1) -import config -from llm_server import opts -from llm_server.helpers import auto_set_base_client_api -from llm_server.llm.vllm.info import vllm_info -from llm_server.custom_redis import flask_cache -from llm_server.llm import redis -from llm_server.routes.v1.generate_stats import generate_stats - app = Flask(__name__) app.register_blueprint(bp, url_prefix='/api/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') -- 2.34.1 From 10eb6269b7ff31b04ac03d05258aa0c974be3b1b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:15:02 -0600 Subject: [PATCH 083/163] t --- llm_server/database/log_to_db.py | 2 -- llm_server/workers/inferencer.py | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index 364e1d3..75bcaab 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -27,6 +27,4 @@ def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_ 'is_error': is_error } } - for k, v in data['kwargs'].items(): - print(k, type(v)) r.publish('database-logger', pickle.dumps(data)) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 84faceb..f1a1a4f 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -66,6 +66,7 @@ def need_to_wait(backend_url: str): concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) s = time.time() while active_workers >= concurrent_gens: + print('worker waiting') time.sleep(0.01) e = time.time() if e - s > 0.5: -- 2.34.1 From 6be1e9acd36eb65b193facebaef2ecc4323aa9b1 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:16:17 -0600 Subject: [PATCH 084/163] t --- llm_server/workers/inferencer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index f1a1a4f..4ec85b5 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -65,8 +65,8 @@ def need_to_wait(backend_url: str): active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) s = time.time() + print(active_workers >= concurrent_gens, active_workers, concurrent_gens) while active_workers >= concurrent_gens: - print('worker waiting') time.sleep(0.01) e = time.time() if e - s > 0.5: -- 2.34.1 From fb8bc05b4c86da77540f5f83933855588fb5e21e Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:30:54 -0600 Subject: [PATCH 085/163] t --- llm_server/routes/openai/chat_completions.py | 1 + llm_server/workers/inferencer.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index d10bdf6..2c13131 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -151,6 +151,7 @@ def openai_chat_completions(): # The worker incremented it, we'll decrement it. decrement_ip_count(handler.client_ip, 'processing_ips') decr_active_workers(handler.selected_model, handler.backend_url) + print('cleaned up') return Response(generate(), mimetype='text/event-stream') except Exception: diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 4ec85b5..e023c86 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -69,5 +69,5 @@ def need_to_wait(backend_url: str): while active_workers >= concurrent_gens: time.sleep(0.01) e = time.time() - if e - s > 0.5: + if e - s > 0.1: print(f'Worker was delayed {e - s} seconds.') -- 2.34.1 From 0718f10eb964777fcf0cded1e0249175022d69f0 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:36:02 -0600 Subject: [PATCH 086/163] t --- llm_server/routes/openai/chat_completions.py | 105 +++++++++---------- 1 file changed, 51 insertions(+), 54 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 2c13131..f45756a 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -3,7 +3,6 @@ import time import traceback from flask import Response, jsonify, request -from redis import Redis from llm_server.custom_redis import redis from . import openai_bp @@ -98,62 +97,60 @@ def openai_chat_completions(): oai_string = generate_oai_string(30) def generate(): - try: - response = generator(msg_to_backend, handler.backend_url) - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + response = generator(msg_to_backend, handler.backend_url) + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"chatcmpl-{oai_string}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) - finally: - # The worker incremented it, we'll decrement it. - decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers(handler.selected_model, handler.backend_url) - print('cleaned up') + data = { + "id": f"chatcmpl-{oai_string}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'INTERNAL SERVER', 500 + finally: + # The worker incremented it, we'll decrement it. + decrement_ip_count(handler.client_ip, 'processing_ips') + decr_active_workers(handler.selected_model, handler.backend_url) -- 2.34.1 From e07e31df0ac8e103c468e081432416c01cba28a8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:37:50 -0600 Subject: [PATCH 087/163] fix --- llm_server/cluster/redis_config_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/cluster/redis_config_cache.py b/llm_server/cluster/redis_config_cache.py index da0b581..fd5d4c1 100644 --- a/llm_server/cluster/redis_config_cache.py +++ b/llm_server/cluster/redis_config_cache.py @@ -1,6 +1,7 @@ import hashlib import pickle +from llm_server.cluster.cluster_config import get_a_cluster_backend from llm_server.custom_redis import RedisCustom @@ -57,4 +58,3 @@ class RedisClusterStore: backend_url = get_a_cluster_backend() print(f'Backend {old} offline. Request was redirected to {backend_url}') return backend_url - -- 2.34.1 From 9b819573e878856813b0ac4a35348d448a0840d8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:39:31 -0600 Subject: [PATCH 088/163] fix import error --- llm_server/cluster/cluster_config.py | 61 +++++++++++++++++++++++- llm_server/cluster/redis_config_cache.py | 60 ----------------------- 2 files changed, 60 insertions(+), 61 deletions(-) delete mode 100644 llm_server/cluster/redis_config_cache.py diff --git a/llm_server/cluster/cluster_config.py b/llm_server/cluster/cluster_config.py index 17e9b05..453fa33 100644 --- a/llm_server/cluster/cluster_config.py +++ b/llm_server/cluster/cluster_config.py @@ -1,9 +1,68 @@ +import hashlib +import pickle + from llm_server import opts -from llm_server.cluster.redis_config_cache import RedisClusterStore from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle from llm_server.cluster.stores import redis_running_models +from llm_server.custom_redis import RedisCustom from llm_server.routes.helpers.model import estimate_model_size + +class RedisClusterStore: + def __init__(self, name: str, **kwargs): + self.name = name + self.config_redis = RedisCustom(name, **kwargs) + + def clear(self): + self.config_redis.flush() + + def load(self, config: dict): + for k, v in config.items(): + self.add_backend(k, v) + + def add_backend(self, name: str, values: dict): + self.config_redis.hset(name, mapping={k: pickle.dumps(v) for k, v in values.items()}) + self.set_backend_value(name, 'online', False) + h = hashlib.sha256(name.encode('utf-8')).hexdigest() + self.set_backend_value(name, 'hash', f'{h[:8]}-{h[-8:]}') + + def set_backend_value(self, backend: str, key: str, value): + # By storing the value as a pickle we don't have to cast anything when getting the value from Redis. + self.config_redis.hset(backend, key, pickle.dumps(value)) + + def get_backend(self, name: str): + r = self.config_redis.hgetall(name) + output = {} + for k, v in r.items(): + output[k.decode('utf8')] = pickle.loads(v) + return output + + def all(self): + keys = self.config_redis.keys('*') + if keys: + result = {} + for key in keys: + if key != f'{self.name}:____': + v = self.get_backend(key) + result[key] = v + return result + else: + return {} + + def validate_backend(self, backend_url: str): + """ + Returns the backend URL that was given, or a new one if that was offline. + :param backend_url: + :return: + """ + backend_info = self.get_backend(backend_url) + if not backend_info['online']: + old = backend_url + backend_url = get_a_cluster_backend() + print(f'Backend {old} offline. Request was redirected to {backend_url}') + return backend_url + + cluster_config = RedisClusterStore('cluster_config') diff --git a/llm_server/cluster/redis_config_cache.py b/llm_server/cluster/redis_config_cache.py deleted file mode 100644 index fd5d4c1..0000000 --- a/llm_server/cluster/redis_config_cache.py +++ /dev/null @@ -1,60 +0,0 @@ -import hashlib -import pickle - -from llm_server.cluster.cluster_config import get_a_cluster_backend -from llm_server.custom_redis import RedisCustom - - -class RedisClusterStore: - def __init__(self, name: str, **kwargs): - self.name = name - self.config_redis = RedisCustom(name, **kwargs) - - def clear(self): - self.config_redis.flush() - - def load(self, config: dict): - for k, v in config.items(): - self.add_backend(k, v) - - def add_backend(self, name: str, values: dict): - self.config_redis.hset(name, mapping={k: pickle.dumps(v) for k, v in values.items()}) - self.set_backend_value(name, 'online', False) - h = hashlib.sha256(name.encode('utf-8')).hexdigest() - self.set_backend_value(name, 'hash', f'{h[:8]}-{h[-8:]}') - - def set_backend_value(self, backend: str, key: str, value): - # By storing the value as a pickle we don't have to cast anything when getting the value from Redis. - self.config_redis.hset(backend, key, pickle.dumps(value)) - - def get_backend(self, name: str): - r = self.config_redis.hgetall(name) - output = {} - for k, v in r.items(): - output[k.decode('utf8')] = pickle.loads(v) - return output - - def all(self): - keys = self.config_redis.keys('*') - if keys: - result = {} - for key in keys: - if key != f'{self.name}:____': - v = self.get_backend(key) - result[key] = v - return result - else: - return {} - - def validate_backend(self, backend_url: str): - """ - Returns the backend URL that was given, or a new one if that was offline. - :param backend_url: - :return: - """ - backend_info = self.get_backend(backend_url) - if not backend_info['online']: - old = backend_url - backend_url = get_a_cluster_backend() - print(f'Backend {old} offline. Request was redirected to {backend_url}') - return backend_url -- 2.34.1 From 817c454c89de13c82fee2c0bb8d34dd1e6b4d988 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:48:07 -0600 Subject: [PATCH 089/163] t --- llm_server/workers/printer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index dcd4e4b..bcc90f4 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -27,4 +27,4 @@ def console_printer(): logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') priority_queue.print_all_items() print('============================') - time.sleep(10) + time.sleep(3) -- 2.34.1 From 46d44f95acfd37c8ccc893d0dbdd6a02e53da808 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:50:10 -0600 Subject: [PATCH 090/163] t --- llm_server/routes/openai/chat_completions.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index f45756a..1b9b2d7 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -57,6 +57,7 @@ def openai_chat_completions(): else: handler.prompt = transform_messages_to_prompt(handler.request.json['messages']) + generated_text = '' response_status_code = 0 start_time = time.time() @@ -154,3 +155,4 @@ def openai_chat_completions(): # The worker incremented it, we'll decrement it. decrement_ip_count(handler.client_ip, 'processing_ips') decr_active_workers(handler.selected_model, handler.backend_url) + print(len(generated_text)) -- 2.34.1 From a37b12a2210a8460dee47afa289c820b6074496b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:51:56 -0600 Subject: [PATCH 091/163] t --- llm_server/routes/openai/chat_completions.py | 106 +++++++++---------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 1b9b2d7..0c62df9 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -57,7 +57,6 @@ def openai_chat_completions(): else: handler.prompt = transform_messages_to_prompt(handler.request.json['messages']) - generated_text = '' response_status_code = 0 start_time = time.time() @@ -98,61 +97,62 @@ def openai_chat_completions(): oai_string = generate_oai_string(30) def generate(): - response = generator(msg_to_backend, handler.backend_url) - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + try: + response = generator(msg_to_backend, handler.backend_url) + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"chatcmpl-{oai_string}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) + data = { + "id": f"chatcmpl-{oai_string}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) + finally: + # The worker incremented it, we'll decrement it. + decrement_ip_count(handler.client_ip, 'processing_ips') + decr_active_workers(handler.selected_model, handler.backend_url) + print(len(generated_text)) return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'INTERNAL SERVER', 500 - finally: - # The worker incremented it, we'll decrement it. - decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers(handler.selected_model, handler.backend_url) - print(len(generated_text)) -- 2.34.1 From 96dd62478f2f1248bf7c77556920c8e78b037caa Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:53:19 -0600 Subject: [PATCH 092/163] fix --- llm_server/database/log_to_db.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index 75bcaab..be6946f 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -5,9 +5,6 @@ from redis import Redis def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): - assert isinstance(prompt, str) - assert isinstance(backend_url, str) - r = Redis(host='localhost', port=6379, db=3) data = { 'function': 'log_prompt', -- 2.34.1 From 50992116f52006e6c9de743e4830e498069d18e9 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 18:59:19 -0600 Subject: [PATCH 093/163] fix --- llm_server/llm/__init__.py | 8 +------- llm_server/routes/openai/chat_completions.py | 1 - 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 5f87c61..73e439f 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -3,13 +3,7 @@ from llm_server.llm import oobabooga, vllm def get_token_count(prompt: str, backend_url: str): - assert isinstance(backend_url, str) - - if not prompt: - # The tokenizers have issues when the prompt is None. - return 0 - assert isinstance(prompt, str) - + backend_url = cluster_config.validate_backend(backend_url) backend_mode = cluster_config.get_backend(backend_url)['mode'] if backend_mode == 'vllm': return vllm.tokenize(prompt, backend_url) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 0c62df9..e470a7b 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -150,7 +150,6 @@ def openai_chat_completions(): # The worker incremented it, we'll decrement it. decrement_ip_count(handler.client_ip, 'processing_ips') decr_active_workers(handler.selected_model, handler.backend_url) - print(len(generated_text)) return Response(generate(), mimetype='text/event-stream') except Exception: -- 2.34.1 From 9befda5acb32bc2f93767abf96f8877e605750c2 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 19:05:41 -0600 Subject: [PATCH 094/163] c --- llm_server/workers/printer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index bcc90f4..7be02d7 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -27,4 +27,4 @@ def console_printer(): logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') priority_queue.print_all_items() print('============================') - time.sleep(3) + time.sleep(1) -- 2.34.1 From 5540112607c368fcd8bdb81ab8297e4adda2e381 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 19:09:25 -0600 Subject: [PATCH 095/163] t --- llm_server/routes/queue.py | 2 ++ llm_server/workers/inferencer.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index e4c49c5..c8cf2e6 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -36,6 +36,8 @@ class RedisPriorityQueue: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request + print('--->', event.event_id) + self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index e023c86..5e1232f 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -33,6 +33,8 @@ def worker(): increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) + print('<---', event_id.event_id) + if not request_json_body: # This was a dummy request from the websocket handlers. # We're going to let the websocket handler decrement -- 2.34.1 From 0bef14ea55cf1780d1bfa049888cedd0cd6836d9 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 19:11:59 -0600 Subject: [PATCH 096/163] t --- llm_server/workers/inferencer.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 5e1232f..fc36474 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -33,7 +33,10 @@ def worker(): increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - print('<---', event_id.event_id) + try: + print('<---', event_id.event_id) + except: + print(event_id) if not request_json_body: # This was a dummy request from the websocket handlers. -- 2.34.1 From c4cc7bbaa00870c1bd17784e6ddfa7c9e69eb0c4 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 19:13:43 -0600 Subject: [PATCH 097/163] f --- llm_server/workers/inferencer.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index fc36474..26028ab 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -33,10 +33,7 @@ def worker(): increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - try: - print('<---', event_id.event_id) - except: - print(event_id) + print('<---', event_id) if not request_json_body: # This was a dummy request from the websocket handlers. -- 2.34.1 From 8df667bc0a05d63c75d1bdd0cd5cff802b295432 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 19:25:08 -0600 Subject: [PATCH 098/163] t --- llm_server/workers/inferencer.py | 13 +++---------- server.py | 1 + 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 26028ab..178bfd6 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -12,17 +12,10 @@ def worker(): (request_json_body, client_ip, token, parameters, backend_url), event_id, selected_model = priority_queue.get() if not backend_url: backend_url = get_a_cluster_backend(selected_model) + else: + backend_url = cluster_config.validate_backend(backend_url) backend_info = cluster_config.get_backend(backend_url) - # The backend could have died between when the request was - # submitted and now, so let's double check it's still online. - if not backend_info['online']: - old = backend_url - backend_url = get_a_cluster_backend() - backend_info = cluster_config.get_backend(backend_url) - print(f'Backend {old} offline. Request was redirected to {backend_url}') - del old # gc - if not selected_model: selected_model = backend_info['model'] @@ -67,7 +60,7 @@ def need_to_wait(backend_url: str): active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) s = time.time() - print(active_workers >= concurrent_gens, active_workers, concurrent_gens) + print(active_workers) while active_workers >= concurrent_gens: time.sleep(0.01) e = time.time() diff --git a/server.py b/server.py index 37c254b..c3ed4a2 100644 --- a/server.py +++ b/server.py @@ -30,6 +30,7 @@ from llm_server.routes.v1 import bp from llm_server.routes.v1.generate_stats import generate_stats from llm_server.sock import init_socketio +# TODO: queue item timeout # TODO: return an `error: True`, error code, and error message rather than just a formatted message # TODO: what happens when all backends are offline? What about the "online" key in the stats page? # TODO: redis SCAN vs KEYS?? -- 2.34.1 From 67173f30ddfa6be1add9e8fe065fc9a05cb9bff6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 19:35:12 -0600 Subject: [PATCH 099/163] t --- llm_server/routes/queue.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index c8cf2e6..f6e7993 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -59,10 +59,12 @@ class RedisPriorityQueue: print(item.decode('utf-8')) def increment_ip_count(self, client_ip: str, redis_key): - self.redis.hincrby(redis_key, client_ip, 1) + new_count = self.redis.hincrby(redis_key, client_ip, 1) + print(client_ip, new_count) def decrement_ip_count(self, client_ip: str, redis_key): new_count = self.redis.hincrby(redis_key, client_ip, -1) + print(client_ip, new_count) if new_count <= 0: self.redis.hdel(redis_key, client_ip) -- 2.34.1 From e9f6fdf65e130c7f9e9fbff0346c56a5d136ba0f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 20:14:28 -0600 Subject: [PATCH 100/163] fix streaming? --- llm_server/config/config.py | 1 - llm_server/config/load.py | 5 +- llm_server/opts.py | 4 +- llm_server/routes/openai/chat_completions.py | 126 ++++++++++--------- llm_server/routes/openai/completions.py | 123 +++++++++--------- llm_server/routes/queue.py | 7 -- llm_server/routes/v1/generate_stream.py | 45 +++++-- llm_server/workers/inferencer.py | 50 +++----- 8 files changed, 193 insertions(+), 168 deletions(-) diff --git a/llm_server/config/config.py b/llm_server/config/config.py index 54eb3ec..2c08544 100644 --- a/llm_server/config/config.py +++ b/llm_server/config/config.py @@ -33,7 +33,6 @@ config_default_vars = { 'openai_moderation_enabled': True, 'netdata_root': None, 'show_backends': True, - 'cluster_workers': 30, 'background_homepage_cacher': True, 'openai_moderation_timeout': 5, 'prioritize_by_size': False diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 9a55a70..2847265 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -45,12 +45,15 @@ def load_config(config_path): opts.openai_silent_trim = config['openai_silent_trim'] opts.openai_moderation_enabled = config['openai_moderation_enabled'] opts.show_backends = config['show_backends'] - opts.cluster_workers = config['cluster_workers'] opts.background_homepage_cacher = config['background_homepage_cacher'] opts.openai_moderation_timeout = config['openai_moderation_timeout'] opts.frontend_api_mode = config['frontend_api_mode'] opts.prioritize_by_size = config['prioritize_by_size'] + # Scale the number of workers. + for item in config['cluster']: + opts.cluster_workers += item['concurrent_gens'] + if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') sys.exit(1) diff --git a/llm_server/opts.py b/llm_server/opts.py index 5c32f05..69b25eb 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -34,7 +34,7 @@ openai_silent_trim = False openai_moderation_enabled = True cluster = {} show_backends = True -cluster_workers = 30 background_homepage_cacher = True openai_moderation_timeout = 5 -prioritize_by_size = False \ No newline at end of file +prioritize_by_size = False +cluster_workers = 0 \ No newline at end of file diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index e470a7b..6e1fdf5 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -8,7 +8,7 @@ from llm_server.custom_redis import redis from . import openai_bp from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler -from ..queue import decr_active_workers, decrement_ip_count, priority_queue +from ..queue import priority_queue from ... import opts from ...database.log_to_db import log_to_db from ...llm.generator import generator @@ -57,6 +57,7 @@ def openai_chat_completions(): else: handler.prompt = transform_messages_to_prompt(handler.request.json['messages']) + event_id = None response_status_code = 0 start_time = time.time() @@ -70,8 +71,10 @@ def openai_chat_completions(): 'stream': True, } - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + event = None + if not handler.is_client_ratelimited(): + # Add a dummy event to the queue and wait for it to reach a worker + event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: log_to_db( handler.client_ip, @@ -87,8 +90,15 @@ def openai_chat_completions(): ) return handler.handle_ratelimited() - # Wait for a worker to get our request and discard it. - _, _, _ = event.wait() + # Once the worker receives our streaming request, it will tell us we are ready + # to begin inference. + event_id = event.event_id + pubsub = redis.pubsub() + pubsub.subscribe(event_id) + for item in pubsub.listen(): + if item['type'] == 'message' and item['data'].decode('utf-8') == 'begin': + break + time.sleep(0.1) try: r_headers = dict(request.headers) @@ -97,61 +107,63 @@ def openai_chat_completions(): oai_string = generate_oai_string(30) def generate(): - try: - response = generator(msg_to_backend, handler.backend_url) - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + response = generator(msg_to_backend, handler.backend_url) + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"chatcmpl-{oai_string}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) - finally: - # The worker incremented it, we'll decrement it. - decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers(handler.selected_model, handler.backend_url) + data = { + "id": f"chatcmpl-{oai_string}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'INTERNAL SERVER', 500 + finally: + # After completing inference, we need to tell the worker we + # are finished. + if event_id: # may be None if ratelimited. + redis.publish(event_id, 'finished') + else: + print('event_id was None!') diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 1843226..9c42cf6 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -8,9 +8,8 @@ from llm_server.custom_redis import redis from . import openai_bp from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler -from ..queue import decr_active_workers, decrement_ip_count, priority_queue +from ..queue import priority_queue from ... import opts -from ...database.database import do_db_log from ...database.log_to_db import log_to_db from ...llm import get_token_count from ...llm.generator import generator @@ -53,7 +52,6 @@ def openai_completions(): return handler.handle_ratelimited() output = response.json['results'][0]['text'] - # TODO: async/await prompt_tokens = get_token_count(request_json_body['prompt'], handler.backend_url) response_tokens = get_token_count(output, handler.backend_url) running_model = redis.get('running_model', 'ERROR', dtype=str) @@ -86,6 +84,7 @@ def openai_completions(): if not opts.enable_streaming: return 'DISABLED', 401 + event_id = None response_status_code = 0 start_time = time.time() @@ -100,8 +99,10 @@ def openai_completions(): 'stream': True, } - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + event = None + if not handler.is_client_ratelimited(): + # Add a dummy event to the queue and wait for it to reach a worker + event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: log_to_db( handler.client_ip, @@ -117,8 +118,14 @@ def openai_completions(): ) return handler.handle_ratelimited() - # Wait for a worker to get our request and discard it. - _, _, _ = event.wait() + # Wait for permission to begin. + event_id = event.event_id + pubsub = redis.pubsub() + pubsub.subscribe(event_id) + for item in pubsub.listen(): + if item['type'] == 'message' and item['data'].decode('utf-8') == 'begin': + break + time.sleep(0.1) try: response = generator(msg_to_backend, handler.backend_url) @@ -128,61 +135,61 @@ def openai_completions(): oai_string = generate_oai_string(30) def generate(): - try: - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"cmpl-{oai_string}", - "object": "text_completion", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time + data = { + "id": f"cmpl-{oai_string}", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) - finally: - # The worker incremented it, we'll decrement it. - decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers(handler.selected_model, handler.backend_url) + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'INTERNAL SERVER', 500 + finally: + if event_id: + redis.publish(event_id, 'finished') + else: + print('event_id was None!') diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index f6e7993..b075ead 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -22,8 +22,6 @@ def decrement_ip_count(client_ip: str, redis_key): class RedisPriorityQueue: def __init__(self, name: str = 'priority_queue', db: int = 12): self.redis = RedisCustom(name, db=db) - self.pubsub = self.redis.pubsub() - self.pubsub.subscribe('events') def put(self, item, priority, selected_model): event = DataEvent() @@ -36,8 +34,6 @@ class RedisPriorityQueue: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request - print('--->', event.event_id) - self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event @@ -54,17 +50,14 @@ class RedisPriorityQueue: def print_all_items(self): items = self.redis.zrange('queue', 0, -1) - print(items) for item in items: print(item.decode('utf-8')) def increment_ip_count(self, client_ip: str, redis_key): new_count = self.redis.hincrby(redis_key, client_ip, 1) - print(client_ip, new_count) def decrement_ip_count(self, client_ip: str, redis_key): new_count = self.redis.hincrby(redis_key, client_ip, -1) - print(client_ip, new_count) if new_count <= 0: self.redis.hdel(redis_key, client_ip) diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 55fceb9..e3818c2 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -7,8 +7,9 @@ from flask import request from . import bp from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler -from ..queue import decr_active_workers, decrement_ip_count, priority_queue +from ..queue import priority_queue from ... import opts +from ...custom_redis import redis from ...database.log_to_db import log_to_db from ...llm.generator import generator from ...sock import sock @@ -94,6 +95,7 @@ def do_stream(ws, model_name): # TODO: implement other backends raise NotImplementedError + event_id = None generated_text = '' input_prompt = request_json_body['prompt'] response_status_code = 0 @@ -117,16 +119,33 @@ def do_stream(ws, model_name): 'stream': True, } - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + event = None + if not handler.is_client_ratelimited(): + # Add a dummy event to the queue and wait for it to reach a worker + event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) if not event: - r, _ = handler.handle_ratelimited() - err_msg = r.json['results'][0]['text'] - send_err_and_quit(err_msg) - return + log_to_db( + handler.client_ip, + handler.token, + handler.request_json_body.get('prompt'), + None, + None, + handler.parameters, + request.headers, + response_status_code, + request.url, + handler.backend_url, + ) + return handler.handle_ratelimited() - # Wait for a worker to get our request and discard it. - _, _, _ = event.wait() + # Wait for permission to begin. + event_id = event.event_id + pubsub = redis.pubsub() + pubsub.subscribe(event_id) + for item in pubsub.listen(): + if item['type'] == 'message' and item['data'].decode('utf-8') == 'begin': + break + time.sleep(0.1) try: response = generator(llm_request, handler.backend_url) @@ -195,9 +214,11 @@ def do_stream(ws, model_name): })) # used to log here finally: - # The worker incremented it, we'll decrement it. - decrement_ip_count(handler.client_ip, 'processing_ips') - decr_active_workers(handler.selected_model, handler.backend_url) + if event_id: + redis.publish(event_id, 'finished') + else: + print('event_id was None!') + try: ws.send(json.dumps({ 'event': 'stream_end', diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 178bfd6..a545ae6 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -19,27 +19,30 @@ def worker(): if not selected_model: selected_model = backend_info['model'] - # This wait time will be "invisible", meaning the worker may as - # well be still waiting to get an item from the queue. - need_to_wait(backend_url) - increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - print('<---', event_id) - - if not request_json_body: - # This was a dummy request from the websocket handlers. - # We're going to let the websocket handler decrement - # processing_ips and active_gen_workers. - event = DataEvent(event_id) - event.set((True, None, None)) - continue - try: - success, response, error_msg = generator(request_json_body, backend_url) - event = DataEvent(event_id) - event.set((success, response, error_msg)) + if not request_json_body: + # This was a dummy request from the streaming handlers. + # The worker will let the handler do the streaming instead + # of the worker. The worker will block until the handler + # is finished. Since a lot of ratelimiting and stats are + # based off the number of active workers, we must keep + # the generation based off the workers. + pubsub = redis.pubsub() + pubsub.subscribe(event_id) + redis.publish(event_id, 'begin') + for item in pubsub.listen(): + if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': + # Once the handler is complete, move on. + break + time.sleep(0.1) + else: + # Normal inference (not streaming). + success, response, error_msg = generator(request_json_body, backend_url) + event = DataEvent(event_id) + event.set((success, response, error_msg)) finally: decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) @@ -53,16 +56,3 @@ def start_workers(num_workers: int): t.start() i += 1 print(f'Started {i} inference workers.') - - -def need_to_wait(backend_url: str): - # We need to check the number of active workers since the streaming endpoint may be doing something. - active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) - concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) - s = time.time() - print(active_workers) - while active_workers >= concurrent_gens: - time.sleep(0.01) - e = time.time() - if e - s > 0.1: - print(f'Worker was delayed {e - s} seconds.') -- 2.34.1 From da20d1807b1f419304d6604b5fdf60cfd0154281 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 20:17:42 -0600 Subject: [PATCH 101/163] actually wait again --- llm_server/workers/inferencer.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index a545ae6..05a2981 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -19,6 +19,8 @@ def worker(): if not selected_model: selected_model = backend_info['model'] + need_to_wait(backend_url) + increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) @@ -56,3 +58,16 @@ def start_workers(num_workers: int): t.start() i += 1 print(f'Started {i} inference workers.') + + +def need_to_wait(backend_url: str): + # We need to check the number of active workers since the streaming endpoint may be doing something. + active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) + concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) + s = time.time() + print(active_workers) + while active_workers >= concurrent_gens: + time.sleep(0.01) + e = time.time() + if e - s > 0.1: + print(f'Worker was delayed {e - s} seconds.') -- 2.34.1 From ea61766838582d19547b50cad3a63d000da6e527 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 20:19:57 -0600 Subject: [PATCH 102/163] fix --- llm_server/workers/inferencer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 05a2981..4a1e61f 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -19,11 +19,11 @@ def worker(): if not selected_model: selected_model = backend_info['model'] - need_to_wait(backend_url) - increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) + need_to_wait(backend_url) + try: if not request_json_body: # This was a dummy request from the streaming handlers. @@ -65,7 +65,6 @@ def need_to_wait(backend_url: str): active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) s = time.time() - print(active_workers) while active_workers >= concurrent_gens: time.sleep(0.01) e = time.time() -- 2.34.1 From e8964fcfd29675c2d3dde5b70395f6f48cbe9309 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 21:37:18 -0600 Subject: [PATCH 103/163] fix the queue?? --- daemon.py | 13 +-- llm_server/config/load.py | 4 + llm_server/routes/openai/chat_completions.py | 111 ++++++++++--------- llm_server/routes/openai/completions.py | 105 +++++++++--------- llm_server/routes/queue.py | 64 ++++++++++- llm_server/routes/request_handler.py | 4 +- llm_server/routes/v1/generate_stream.py | 2 +- llm_server/workers/inferencer.py | 45 +++----- llm_server/workers/printer.py | 2 - llm_server/workers/threader.py | 2 +- 10 files changed, 195 insertions(+), 157 deletions(-) diff --git a/daemon.py b/daemon.py index 0fa3601..35c1d59 100644 --- a/daemon.py +++ b/daemon.py @@ -3,13 +3,12 @@ import sys import time from pathlib import Path +from redis import Redis + from llm_server.cluster.cluster_config import cluster_config -from llm_server.cluster.redis_cycle import redis_cycler_db -from llm_server.cluster.stores import redis_running_models from llm_server.config.load import load_config, parse_backends from llm_server.custom_redis import redis from llm_server.database.create import create_db -from llm_server.routes.queue import priority_queue from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.threader import start_background @@ -21,11 +20,8 @@ else: config_path = Path(script_path, 'config', 'config.yml') if __name__ == "__main__": - flushed_keys = redis.flush() - print('Flushed', len(flushed_keys), 'keys from Redis.') - - redis_cycler_db.flushall() - redis_running_models.flush() + Redis().flushall() + print('Flushed Redis.') success, config, msg = load_config(config_path) if not success: @@ -34,7 +30,6 @@ if __name__ == "__main__": create_db() - priority_queue.flush() cluster_config.clear() cluster_config.load(parse_backends(config)) diff --git a/llm_server/config/load.py b/llm_server/config/load.py index 2847265..cc3250c 100644 --- a/llm_server/config/load.py +++ b/llm_server/config/load.py @@ -3,11 +3,13 @@ import sys import openai +import llm_server from llm_server import opts from llm_server.config.config import ConfigLoader, config_default_vars, config_required_vars from llm_server.custom_redis import redis from llm_server.database.conn import database from llm_server.database.database import get_number_of_rows +from llm_server.routes.queue import PriorityQueue def load_config(config_path): @@ -54,6 +56,8 @@ def load_config(config_path): for item in config['cluster']: opts.cluster_workers += item['concurrent_gens'] + llm_server.routes.queue.priority_queue = PriorityQueue([x['backend_url'] for x in config['cluster']]) + if opts.openai_expose_our_model and not opts.openai_api_key: print('If you set openai_epose_our_model to false, you must set your OpenAI key in openai_api_key.') sys.exit(1) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 6e1fdf5..bcbd24c 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -74,7 +74,7 @@ def openai_chat_completions(): event = None if not handler.is_client_ratelimited(): # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) if not event: log_to_db( handler.client_ip, @@ -107,63 +107,64 @@ def openai_chat_completions(): oai_string = generate_oai_string(30) def generate(): - response = generator(msg_to_backend, handler.backend_url) - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + try: + response = generator(msg_to_backend, handler.backend_url) + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"chatcmpl-{oai_string}", - "object": "chat.completion.chunk", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) + data = { + "id": f"chatcmpl-{oai_string}", + "object": "chat.completion.chunk", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) + finally: + # After completing inference, we need to tell the worker we + # are finished. + if event_id: # may be None if ratelimited. + redis.publish(event_id, 'finished') + else: + print('event_id was None!') return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'INTERNAL SERVER', 500 - finally: - # After completing inference, we need to tell the worker we - # are finished. - if event_id: # may be None if ratelimited. - redis.publish(event_id, 'finished') - else: - print('event_id was None!') diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 9c42cf6..8b5d987 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -102,7 +102,7 @@ def openai_completions(): event = None if not handler.is_client_ratelimited(): # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) if not event: log_to_db( handler.client_ip, @@ -135,61 +135,62 @@ def openai_completions(): oai_string = generate_oai_string(30) def generate(): - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue + try: + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(handler.prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue - data = { - "id": f"cmpl-{oai_string}", - "object": "text_completion", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time + data = { + "id": f"cmpl-{oai_string}", + "object": "text_completion", + "created": int(time.time()), + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": new + }, + "finish_reason": None + } + ] + } + yield f'data: {json.dumps(data)}\n\n' + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - response_status_code, - r_url, - handler.backend_url, - ) + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + response_status_code, + r_url, + handler.backend_url, + ) + finally: + if event_id: + redis.publish(event_id, 'finished') + else: + print('event_id was None!') return Response(generate(), mimetype='text/event-stream') except Exception: traceback.print_exc() return 'INTERNAL SERVER', 500 - finally: - if event_id: - redis.publish(event_id, 'finished') - else: - print('event_id was None!') diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index b075ead..d88ed45 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -1,10 +1,12 @@ import json import pickle import time +from typing import Tuple from uuid import uuid4 from redis import Redis +from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import RedisCustom, redis from llm_server.database.database import get_token_ratelimit @@ -20,7 +22,7 @@ def decrement_ip_count(client_ip: str, redis_key): class RedisPriorityQueue: - def __init__(self, name: str = 'priority_queue', db: int = 12): + def __init__(self, name, db: int = 12): self.redis = RedisCustom(name, db=db) def put(self, item, priority, selected_model): @@ -98,9 +100,6 @@ class DataEvent: return pickle.loads(item['data']) -priority_queue = RedisPriorityQueue() - - def update_active_workers(key: str, operation: str): if operation == 'incr': redis.incr(f'active_gen_workers:{key}') @@ -118,3 +117,60 @@ def incr_active_workers(selected_model: str, backend_url: str): def decr_active_workers(selected_model: str, backend_url: str): update_active_workers(selected_model, 'decr') update_active_workers(backend_url, 'decr') + + +class PriorityQueue: + def __init__(self, backends: list = None): + """ + Only have to load the backends once. + :param backends: + """ + self.redis = Redis(host='localhost', port=6379, db=9) + if backends: + for item in backends: + self.redis.lpush('backends', item) + + def get_backends(self): + return [x.decode('utf-8') for x in self.redis.lrange('backends', 0, -1)] + + def get_queued_ip_count(self, client_ip: str): + count = 0 + for backend_url in self.get_backends(): + queue = RedisPriorityQueue(backend_url) + count += queue.get_queued_ip_count(client_ip) + return count + + def put(self, backend_url, item: Tuple[dict, str, str, dict], priority: int, selected_model: str): + queue = RedisPriorityQueue(backend_url) + return queue.put(item, priority, selected_model) + + def len(self, model_name): + count = 0 + backends_with_models = [] + for k in self.get_backends(): + info = cluster_config.get_backend(k) + if info.get('model') == model_name: + backends_with_models.append(k) + for backend_url in backends_with_models: + queue = RedisPriorityQueue(backend_url) + count += queue.len(model_name) + return count + + def __len__(self): + count = 0 + for backend_url in self.get_backends(): + queue = RedisPriorityQueue(backend_url) + count += len(queue) + return count + + def flush(self): + for k in self.redis.keys(): + q = json.loads(self.redis.get(k)) + q.flush() + self.redis.set(k, json.dumps(q)) + + def flush_db(self): + self.redis.flushdb() + + +priority_queue = PriorityQueue() diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index ef5aa34..a048df7 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -7,7 +7,7 @@ from flask import Response, request from llm_server import opts from llm_server.cluster.cluster_config import cluster_config, get_a_cluster_backend from llm_server.custom_redis import redis -from llm_server.database.database import get_token_ratelimit, do_db_log +from llm_server.database.database import get_token_ratelimit from llm_server.database.log_to_db import log_to_db from llm_server.helpers import auto_set_base_client_api from llm_server.llm.oobabooga.ooba_backend import OobaboogaBackend @@ -131,7 +131,7 @@ class RequestHandler: request_valid, invalid_response = self.validate_request(prompt, do_log=True) if not request_valid: return (False, None, None, 0), invalid_response - event = priority_queue.put((llm_request, self.client_ip, self.token, self.parameters, self.backend_url), self.token_priority, self.selected_model) + event = priority_queue.put(self.backend_url, (llm_request, self.client_ip, self.token, self.parameters), self.token_priority, self.selected_model) else: event = None diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index e3818c2..c55e36f 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -122,7 +122,7 @@ def do_stream(ws, model_name): event = None if not handler.is_client_ratelimited(): # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put((None, handler.client_ip, handler.token, None, handler.backend_url), handler.token_priority, handler.selected_model) + event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) if not event: log_to_db( handler.client_ip, diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 4a1e61f..324c13a 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,29 +1,24 @@ import threading import time +from uuid import uuid4 -from llm_server.cluster.cluster_config import cluster_config, get_a_cluster_backend -from llm_server.custom_redis import redis +from llm_server.cluster.cluster_config import cluster_config +from llm_server.custom_redis import redis, RedisCustom from llm_server.llm.generator import generator -from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, priority_queue +from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, RedisPriorityQueue, PriorityQueue, priority_queue -def worker(): +def worker(backend_url): + queue = RedisPriorityQueue(backend_url) while True: - (request_json_body, client_ip, token, parameters, backend_url), event_id, selected_model = priority_queue.get() - if not backend_url: - backend_url = get_a_cluster_backend(selected_model) - else: - backend_url = cluster_config.validate_backend(backend_url) + (request_json_body, client_ip, token, parameters), event_id, selected_model = queue.get() backend_info = cluster_config.get_backend(backend_url) - if not selected_model: selected_model = backend_info['model'] increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - need_to_wait(backend_url) - try: if not request_json_body: # This was a dummy request from the streaming handlers. @@ -37,7 +32,6 @@ def worker(): redis.publish(event_id, 'begin') for item in pubsub.listen(): if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': - # Once the handler is complete, move on. break time.sleep(0.1) else: @@ -50,23 +44,12 @@ def worker(): decr_active_workers(selected_model, backend_url) -def start_workers(num_workers: int): +def start_workers(cluster: dict): i = 0 - for _ in range(num_workers): - t = threading.Thread(target=worker) - t.daemon = True - t.start() - i += 1 + for item in cluster: + for _ in range(item['concurrent_gens']): + t = threading.Thread(target=worker, args=(item['backend_url'],)) + t.daemon = True + t.start() + i += 1 print(f'Started {i} inference workers.') - - -def need_to_wait(backend_url: str): - # We need to check the number of active workers since the streaming endpoint may be doing something. - active_workers = redis.get(f'active_gen_workers:{backend_url}', 0, dtype=int) - concurrent_gens = cluster_config.get_backend(backend_url).get('concurrent_gens', 1) - s = time.time() - while active_workers >= concurrent_gens: - time.sleep(0.01) - e = time.time() - if e - s > 0.1: - print(f'Worker was delayed {e - s} seconds.') diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index 7be02d7..cf691c1 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -25,6 +25,4 @@ def console_printer(): processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - priority_queue.print_all_items() - print('============================') time.sleep(1) diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index dbdc8e0..1f5266f 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -20,7 +20,7 @@ def cache_stats(): def start_background(): - start_workers(opts.cluster_workers) + start_workers(opts.cluster) t = Thread(target=main_background_thread) t.daemon = True -- 2.34.1 From 3e5feb9c9734dd7a1f16be486adb98e33cb9ef3f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 5 Oct 2023 21:43:49 -0600 Subject: [PATCH 104/163] fix stat --- llm_server/routes/queue.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index d88ed45..24bc019 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -66,14 +66,6 @@ class RedisPriorityQueue: def __len__(self): return self.redis.zcard('queue') - def len(self, model_name): - count = 0 - for key in self.redis.zrange('queue', 0, -1): - item = json.loads(key) - if item[2] == model_name: - count += 1 - return count - def get_queued_ip_count(self, client_ip: str): q = self.redis.hget('queued_ip_count', client_ip) if not q: @@ -152,8 +144,7 @@ class PriorityQueue: if info.get('model') == model_name: backends_with_models.append(k) for backend_url in backends_with_models: - queue = RedisPriorityQueue(backend_url) - count += queue.len(model_name) + count += len(RedisPriorityQueue(backend_url)) return count def __len__(self): -- 2.34.1 From 467e1893ea2abfbc8995913297af9077b904e38b Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 8 Oct 2023 19:36:12 -0600 Subject: [PATCH 105/163] fix issue with null data on openai --- llm_server/llm/openai/transform.py | 3 ++- llm_server/routes/openai/chat_completions.py | 4 ++++ llm_server/routes/openai_request_handler.py | 4 ++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/llm_server/llm/openai/transform.py b/llm_server/llm/openai/transform.py index 0c2946b..daec3dc 100644 --- a/llm_server/llm/openai/transform.py +++ b/llm_server/llm/openai/transform.py @@ -87,8 +87,9 @@ def transform_messages_to_prompt(oai_messages): try: prompt = f'### INSTRUCTION: {opts.openai_system_prompt}' for msg in oai_messages: - if not msg.get('content') or not msg.get('role'): + if 'content' not in msg.keys() or 'role' not in msg.keys(): return False + msg['content'] = str(msg['content']) # Prevent any weird issues. if msg['role'] == 'system': prompt += f'### INSTRUCTION: {msg["content"]}\n\n' elif msg['role'] == 'user': diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index bcbd24c..b088a18 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -57,6 +57,10 @@ def openai_chat_completions(): else: handler.prompt = transform_messages_to_prompt(handler.request.json['messages']) + if not handler.prompt: + # Prevent issues on the backend. + return 'Invalid prompt', 400 + event_id = None response_status_code = 0 start_time = time.time() diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 037de27..9716eb9 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -61,6 +61,10 @@ class OpenAIRequestHandler(RequestHandler): # TODO: support Ooba self.parameters = oai_to_vllm(self.parameters, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) + if not self.prompt: + # TODO: format this as an openai error message + return 'Invalid prompt', 400 + llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) -- 2.34.1 From ae4d4e5ca92b59901e57d59bbf3d8fc548c627d2 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 9 Oct 2023 10:31:35 -0600 Subject: [PATCH 106/163] fix exception --- llm_server/llm/openai/oai_to_vllm.py | 2 ++ llm_server/routes/request_handler.py | 1 + 2 files changed, 3 insertions(+) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 35c9f30..1490933 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -27,6 +27,8 @@ def oai_to_vllm(request_json_body, stop_hashes: bool, mode): if mode == 'vllm' and request_json_body.get('top_p') == 0: request_json_body['top_p'] = 0.01 + request_json_body['max_tokens'] = min(max(request_json_body.get('max_new_tokens', 0), request_json_body.get('max_tokens', 0)), opts.max_new_tokens) + return request_json_body diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index a048df7..dd8326b 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -37,6 +37,7 @@ class RequestHandler: self.parameters = None self.used = False + self.selected_model = selected_model self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) -- 2.34.1 From 5f7bf4faca6aeadfa52fde42252498a2a2e07062 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 9 Oct 2023 18:12:12 -0600 Subject: [PATCH 107/163] misc changes --- llm_server/routes/openai_request_handler.py | 2 +- other/gradio/gradio_chat.py | 62 +++++++++++++++++++++ other/gradio_chat.py | 33 ----------- 3 files changed, 63 insertions(+), 34 deletions(-) create mode 100644 other/gradio/gradio_chat.py delete mode 100644 other/gradio_chat.py diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 9716eb9..bc5c6f5 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -67,7 +67,6 @@ class OpenAIRequestHandler(RequestHandler): llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) - model = self.request_json_body.get('model') if success: @@ -98,6 +97,7 @@ class OpenAIRequestHandler(RequestHandler): return response, 429 def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: + print(error_msg) return jsonify({ "error": { "message": "Invalid request, check your parameters and try again.", diff --git a/other/gradio/gradio_chat.py b/other/gradio/gradio_chat.py new file mode 100644 index 0000000..e8e54b2 --- /dev/null +++ b/other/gradio/gradio_chat.py @@ -0,0 +1,62 @@ +import os +import sys +import warnings + +import gradio as gr +import openai + +warnings.filterwarnings("ignore") + +API_BASE = os.getenv('API_BASE') +if not API_BASE: + print('Must set the secret variable API_BASE to your https://your-site/api/openai/v1') + sys.exit(1) + +# A system prompt can be injected into the very first spot in the context. +# If the user sends a message that contains the CONTEXT_TRIGGER_PHRASE, +# the content in CONTEXT_TRIGGER_INJECTION will be injected. +# Setting CONTEXT_TRIGGER_PHRASE will also add it to the selectable examples. +CONTEXT_TRIGGER_PHRASE = os.getenv('CONTEXT_TRIGGER_PHRASE') +CONTEXT_TRIGGER_INJECTION = os.getenv('CONTEXT_TRIGGER_INJECTION') + +openai.api_key = 'null' +openai.api_base = API_BASE + + +def stream_response(prompt, history): + messages = [] + do_injection = False + for human, assistant in history: + messages.append({'role': 'user', 'content': str(human)}) + messages.append({'role': 'assistant', 'content': str(assistant)}) + + if CONTEXT_TRIGGER_INJECTION and CONTEXT_TRIGGER_PHRASE in human: + do_injection = True + messages.append({'role': 'user', 'content': prompt}) + + if do_injection or (CONTEXT_TRIGGER_INJECTION and CONTEXT_TRIGGER_PHRASE in prompt): + messages.insert(0, {'role': 'system', 'content': CONTEXT_TRIGGER_INJECTION}) + + try: + response = openai.ChatCompletion.create( + model='0', + messages=messages, + temperature=0, + max_tokens=300, + stream=True + ) + except Exception: + raise gr.Error("Failed to reach inference endpoint.") + + message = '' + for chunk in response: + if len(chunk['choices'][0]['delta']) != 0: + message += chunk['choices'][0]['delta']['content'] + yield message + + +examples = ["hello", "hola", "merhaba"] +if CONTEXT_TRIGGER_PHRASE: + examples.insert(0, CONTEXT_TRIGGER_PHRASE) + +gr.ChatInterface(stream_response, examples=examples, title="Chatbot Demo", analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}').queue(concurrency_count=3).launch() diff --git a/other/gradio_chat.py b/other/gradio_chat.py deleted file mode 100644 index eb10d26..0000000 --- a/other/gradio_chat.py +++ /dev/null @@ -1,33 +0,0 @@ -import warnings - -import gradio as gr -import openai - -warnings.filterwarnings("ignore") - -openai.api_key = 'null' -openai.api_base = 'http://localhost:5000/api/openai/v1' - - -def stream_response(prompt, history): - messages = [] - for x in history: - messages.append({'role': 'user', 'content': x[0]}) - messages.append({'role': 'assistant', 'content': x[1]}) - messages.append({'role': 'user', 'content': prompt}) - - response = openai.ChatCompletion.create( - model='0', - messages=messages, - temperature=0, - max_tokens=300, - stream=True - ) - - message = '' - for chunk in response: - message += chunk['choices'][0]['delta']['content'] - yield message - - -gr.ChatInterface(stream_response, examples=["hello", "hola", "merhaba"], title="Chatbot Demo", analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}').queue().launch() -- 2.34.1 From 18e37a72ae9ee0251bf2fca61091c4b187930cd8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 9 Oct 2023 23:51:26 -0600 Subject: [PATCH 108/163] add model selection to openai endpoint --- llm_server/cluster/backend.py | 6 ++---- llm_server/cluster/model_choices.py | 1 - llm_server/routes/openai/__init__.py | 3 +++ llm_server/routes/openai/chat_completions.py | 7 ++++--- llm_server/routes/openai/completions.py | 7 ++++--- llm_server/routes/request_handler.py | 2 +- llm_server/routes/v1/generate.py | 2 +- server.py | 3 ++- 8 files changed, 17 insertions(+), 14 deletions(-) delete mode 100644 llm_server/cluster/model_choices.py diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index c301f93..2a7edc3 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -7,7 +7,7 @@ from llm_server.custom_redis import redis from llm_server.llm.generator import generator from llm_server.llm.info import get_info from llm_server.routes.queue import priority_queue -from llm_server.routes.stats import get_active_gen_workers_model, calculate_wait_time +from llm_server.routes.stats import calculate_wait_time, get_active_gen_workers_model def get_backends_from_model(model_name: str): @@ -51,8 +51,6 @@ def test_backend(backend_url: str, test_prompt: bool = False): return True, i - - def get_model_choices(regen: bool = False): if not regen: c = redis.getp('model_choices') @@ -89,7 +87,7 @@ def get_model_choices(regen: bool = False): 'model': model, 'client_api': f'https://{base_client_api}/{model}', 'ws_client_api': f'wss://{base_client_api}/{model}/v1/stream' if opts.enable_streaming else None, - 'openai_client_api': f'https://{base_client_api}/openai/{model}' if opts.enable_openi_compatible_backend else 'disabled', + 'openai_client_api': f'https://{base_client_api}/openai/{model}/v1' if opts.enable_openi_compatible_backend else 'disabled', 'backend_count': len(b), 'estimated_wait': estimated_wait_sec, 'queued': proompters_in_queue, diff --git a/llm_server/cluster/model_choices.py b/llm_server/cluster/model_choices.py deleted file mode 100644 index ef93bba..0000000 --- a/llm_server/cluster/model_choices.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: give this a better name! diff --git a/llm_server/routes/openai/__init__.py b/llm_server/routes/openai/__init__.py index 67febc9..3a69aa7 100644 --- a/llm_server/routes/openai/__init__.py +++ b/llm_server/routes/openai/__init__.py @@ -5,9 +5,11 @@ from ..server_error import handle_server_error from ... import opts openai_bp = Blueprint('openai/v1/', __name__) +openai_model_bp = Blueprint('openai/', __name__) @openai_bp.before_request +@openai_model_bp.before_request def before_oai_request(): if not opts.enable_openi_compatible_backend: return 'The OpenAI-compatible backend is disabled.', 401 @@ -15,6 +17,7 @@ def before_oai_request(): @openai_bp.errorhandler(500) +@openai_model_bp.errorhandler(500) def handle_error(e): return handle_server_error(e) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index b088a18..76f1b6c 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -5,7 +5,7 @@ import traceback from flask import Response, jsonify, request from llm_server.custom_redis import redis -from . import openai_bp +from . import openai_bp, openai_model_bp from ..helpers.http import validate_json from ..openai_request_handler import OpenAIRequestHandler from ..queue import priority_queue @@ -20,12 +20,13 @@ from ...llm.openai.transform import generate_oai_string, transform_messages_to_p @openai_bp.route('/chat/completions', methods=['POST']) -def openai_chat_completions(): +@openai_model_bp.route('//v1/chat/completions', methods=['POST']) +def openai_chat_completions(model_name=None): request_valid_json, request_json_body = validate_json(request) if not request_valid_json or not request_json_body.get('messages') or not request_json_body.get('model'): return jsonify({'code': 400, 'msg': 'invalid JSON'}), 400 else: - handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body) + handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body, selected_model=model_name) if not request_json_body.get('stream'): try: invalid_oai_err_msg = validate_oai(request_json_body) diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 8b5d987..c8e7f19 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -5,7 +5,7 @@ import simplejson as json from flask import Response, jsonify, request from llm_server.custom_redis import redis -from . import openai_bp +from . import openai_bp, openai_model_bp from ..helpers.http import validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import priority_queue @@ -20,12 +20,13 @@ from ...llm.openai.transform import generate_oai_string, trim_string_to_fit # TODO: add rate-limit headers? @openai_bp.route('/completions', methods=['POST']) -def openai_completions(): +@openai_model_bp.route('//v1/completions', methods=['POST']) +def openai_completions(model_name=None): request_valid_json, request_json_body = validate_json(request) if not request_valid_json or not request_json_body.get('prompt'): return jsonify({'code': 400, 'msg': 'Invalid JSON'}), 400 else: - handler = OobaRequestHandler(incoming_request=request) + handler = OobaRequestHandler(incoming_request=request, incoming_json=request_json_body, selected_model=model_name) if handler.cluster_backend_info['mode'] != 'vllm': # TODO: implement other backends diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index dd8326b..6a6ad4a 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -20,7 +20,7 @@ from llm_server.routes.queue import priority_queue class RequestHandler: def __init__(self, incoming_request: flask.Request, selected_model: str = None, incoming_json: Union[dict, str] = None): self.request = incoming_request - self.enable_backend_blind_rrd = request.headers.get('LLM-Blind-RRD', False) == 'true' + # self.enable_backend_blind_rrd = request.headers.get('LLM-Blind-RRD', False) == 'true' # Routes need to validate it, here we just load it if incoming_json: diff --git a/llm_server/routes/v1/generate.py b/llm_server/routes/v1/generate.py index 1a63db9..fcdc298 100644 --- a/llm_server/routes/v1/generate.py +++ b/llm_server/routes/v1/generate.py @@ -14,7 +14,7 @@ def generate(model_name=None): if not request_valid_json or not request_json_body.get('prompt'): return jsonify({'code': 400, 'msg': 'Invalid JSON'}), 400 else: - handler = OobaRequestHandler(request, model_name) + handler = OobaRequestHandler(request, selected_model=model_name) try: return handler.handle_request() except Exception: diff --git a/server.py b/server.py index c3ed4a2..89c71aa 100644 --- a/server.py +++ b/server.py @@ -24,7 +24,7 @@ from llm_server.database.create import create_db from llm_server.helpers import auto_set_base_client_api from llm_server.llm.vllm.info import vllm_info from llm_server.pre_fork import server_startup -from llm_server.routes.openai import openai_bp +from llm_server.routes.openai import openai_bp, openai_model_bp from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.routes.v1.generate_stats import generate_stats @@ -70,6 +70,7 @@ except ModuleNotFoundError as e: app = Flask(__name__) app.register_blueprint(bp, url_prefix='/api/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') +app.register_blueprint(openai_model_bp, url_prefix='/api/openai/') init_socketio(app) flask_cache.init_app(app) flask_cache.clear() -- 2.34.1 From f4e5b5275dd41bce0ef595746047dc9faff90dad Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 09:09:41 -0600 Subject: [PATCH 109/163] test --- daemon.py | 4 ++-- llm_server/workers/inferencer.py | 2 ++ llm_server/workers/threader.py | 1 - other/gradio/gradio_chat.py | 36 +++++++++++++++++++++----------- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/daemon.py b/daemon.py index 35c1d59..9e0d5a9 100644 --- a/daemon.py +++ b/daemon.py @@ -20,8 +20,8 @@ else: config_path = Path(script_path, 'config', 'config.yml') if __name__ == "__main__": - Redis().flushall() - print('Flushed Redis.') + # Redis().flushall() + # print('Flushed Redis.') success, config, msg = load_config(config_path) if not success: diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 324c13a..f06ff8c 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -19,6 +19,8 @@ def worker(backend_url): increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) + print('Worker starting processing for', client_ip) + try: if not request_json_body: # This was a dummy request from the streaming handlers. diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index 1f5266f..f19ce1c 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -44,7 +44,6 @@ def start_background(): t.start() print('Started the console printer.') - redis_running_models.flush() t = Thread(target=cluster_worker) t.daemon = True t.start() diff --git a/other/gradio/gradio_chat.py b/other/gradio/gradio_chat.py index e8e54b2..fa1b892 100644 --- a/other/gradio/gradio_chat.py +++ b/other/gradio/gradio_chat.py @@ -12,6 +12,12 @@ if not API_BASE: print('Must set the secret variable API_BASE to your https://your-site/api/openai/v1') sys.exit(1) +BACKUP_API_BASE = os.getenv('BACKUP_API_BASE') +if BACKUP_API_BASE: + print('Using BACKUP_API_BASE:', BACKUP_API_BASE) + +APP_TITLE = os.getenv('APP_TITLE') + # A system prompt can be injected into the very first spot in the context. # If the user sends a message that contains the CONTEXT_TRIGGER_PHRASE, # the content in CONTEXT_TRIGGER_INJECTION will be injected. @@ -37,16 +43,22 @@ def stream_response(prompt, history): if do_injection or (CONTEXT_TRIGGER_INJECTION and CONTEXT_TRIGGER_PHRASE in prompt): messages.insert(0, {'role': 'system', 'content': CONTEXT_TRIGGER_INJECTION}) - try: - response = openai.ChatCompletion.create( - model='0', - messages=messages, - temperature=0, - max_tokens=300, - stream=True - ) - except Exception: - raise gr.Error("Failed to reach inference endpoint.") + for _ in range(2): + try: + response = openai.ChatCompletion.create( + model='0', + messages=messages, + temperature=0, + max_tokens=300, + stream=True + ) + break + except Exception: + openai.api_base = BACKUP_API_BASE + raise gr.Error("Failed to reach inference endpoint.") + + # Go back to the default endpoint + openai.api_base = API_BASE message = '' for chunk in response: @@ -55,8 +67,8 @@ def stream_response(prompt, history): yield message -examples = ["hello", "hola", "merhaba"] +examples = ["hello"] if CONTEXT_TRIGGER_PHRASE: examples.insert(0, CONTEXT_TRIGGER_PHRASE) -gr.ChatInterface(stream_response, examples=examples, title="Chatbot Demo", analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}').queue(concurrency_count=3).launch() +gr.ChatInterface(stream_response, examples=examples, title=APP_TITLE, analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}').queue(concurrency_count=1, api_open=False).launch(show_api=False) -- 2.34.1 From 7286e38cb049f0ce7b9aa143417511f3aab131f3 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 09:10:32 -0600 Subject: [PATCH 110/163] t --- daemon.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/daemon.py b/daemon.py index 9e0d5a9..35c1d59 100644 --- a/daemon.py +++ b/daemon.py @@ -20,8 +20,8 @@ else: config_path = Path(script_path, 'config', 'config.yml') if __name__ == "__main__": - # Redis().flushall() - # print('Flushed Redis.') + Redis().flushall() + print('Flushed Redis.') success, config, msg = load_config(config_path) if not success: -- 2.34.1 From 78114771b0876f553f4464286b43d14381d9cc02 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 09:20:00 -0600 Subject: [PATCH 111/163] fix oai exception --- llm_server/llm/openai/oai_to_vllm.py | 15 +++++++++++++++ llm_server/routes/openai/chat_completions.py | 7 ++++++- llm_server/routes/openai/completions.py | 6 +++++- llm_server/routes/openai_request_handler.py | 10 +++++++--- llm_server/workers/inferencer.py | 3 +++ 5 files changed, 36 insertions(+), 5 deletions(-) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index 1490933..cde5180 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -77,3 +77,18 @@ def validate_oai(parameters): if parameters.get('max_tokens', 2) < 1: return format_oai_err(f"{parameters['max_tokens']} is less than the minimum of 1 - 'max_tokens'") + + +def return_invalid_model_err(requested_model: str): + if requested_model: + msg = f"The model `{requested_model}` does not exist" + else: + msg = "The requested model does not exist" + return jsonify({ + "error": { + "message": msg, + "type": "invalid_request_error", + "param": None, + "code": "model_not_found" + } + }), 404 diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 76f1b6c..d054703 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -12,7 +12,7 @@ from ..queue import priority_queue from ... import opts from ...database.log_to_db import log_to_db from ...llm.generator import generator -from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai +from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai, return_invalid_model_err from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit @@ -27,6 +27,11 @@ def openai_chat_completions(model_name=None): return jsonify({'code': 400, 'msg': 'invalid JSON'}), 400 else: handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body, selected_model=model_name) + if handler.offline: + msg = return_invalid_model_err(model_name) + print(msg) + return handler.handle_error(msg) + if not request_json_body.get('stream'): try: invalid_oai_err_msg = validate_oai(request_json_body) diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index c8e7f19..3dcde2e 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -13,7 +13,7 @@ from ... import opts from ...database.log_to_db import log_to_db from ...llm import get_token_count from ...llm.generator import generator -from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai +from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai, return_invalid_model_err from ...llm.openai.transform import generate_oai_string, trim_string_to_fit @@ -27,6 +27,10 @@ def openai_completions(model_name=None): return jsonify({'code': 400, 'msg': 'Invalid JSON'}), 400 else: handler = OobaRequestHandler(incoming_request=request, incoming_json=request_json_body, selected_model=model_name) + if handler.offline: + msg = return_invalid_model_err(model_name) + print(msg) + return handler.handle_error(msg) if handler.cluster_backend_info['mode'] != 'vllm': # TODO: implement other backends diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index bc5c6f5..3c2a5b1 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -11,10 +11,10 @@ from flask import Response, jsonify, make_response from llm_server import opts from llm_server.cluster.backend import get_model_choices from llm_server.custom_redis import redis -from llm_server.database.database import is_api_key_moderated, do_db_log +from llm_server.database.database import is_api_key_moderated from llm_server.database.log_to_db import log_to_db from llm_server.llm import get_token_count -from llm_server.llm.openai.oai_to_vllm import oai_to_vllm, validate_oai +from llm_server.llm.openai.oai_to_vllm import oai_to_vllm, validate_oai, return_invalid_model_err from llm_server.llm.openai.transform import ANTI_CONTINUATION_RE, ANTI_RESPONSE_RE, generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit from llm_server.routes.request_handler import RequestHandler from llm_server.workers.moderator import add_moderation_task, get_results @@ -27,6 +27,10 @@ class OpenAIRequestHandler(RequestHandler): def handle_request(self) -> Tuple[flask.Response, int]: assert not self.used + if self.offline: + msg = return_invalid_model_err(self.selected_model) + print(msg) + return self.handle_error(msg) if opts.openai_silent_trim: oai_messages = trim_messages_to_fit(self.request.json['messages'], self.cluster_backend_info['model_config']['max_position_embeddings'], self.backend_url) @@ -63,7 +67,7 @@ class OpenAIRequestHandler(RequestHandler): if not self.prompt: # TODO: format this as an openai error message - return 'Invalid prompt', 400 + return Response('Invalid prompt'), 400 llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index f06ff8c..79eed01 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,5 +1,6 @@ import threading import time +import traceback from uuid import uuid4 from llm_server.cluster.cluster_config import cluster_config @@ -41,6 +42,8 @@ def worker(backend_url): success, response, error_msg = generator(request_json_body, backend_url) event = DataEvent(event_id) event.set((success, response, error_msg)) + except: + traceback.print_exc() finally: decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) -- 2.34.1 From 1d1c45dc1a55081b945914a06f39de5f7e9716f9 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 12:22:50 -0600 Subject: [PATCH 112/163] add length penalty param to vllm --- llm_server/helpers.py | 3 ++- llm_server/llm/vllm/vllm_backend.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/llm_server/helpers.py b/llm_server/helpers.py index 9fc7274..91f3b15 100644 --- a/llm_server/helpers.py +++ b/llm_server/helpers.py @@ -54,7 +54,8 @@ def jsonify_pretty(json_dict: Union[list, dict], status=200, indent=4, sort_keys def round_up_base(n, base): if base == 0: - print('round_up_base DIVIDE BY ZERO ERROR????', n, base) + # TODO: I don't think passing (0, 0) to this function is a sign of any underlying issues. + # print('round_up_base DIVIDE BY ZERO ERROR????', n, base) return 0 return math.ceil(n / base) * base diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index a9ec821..835d2ce 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -39,7 +39,8 @@ class VLLMBackend(LLMBackend): ignore_eos=parameters.get('ban_eos_token', False), max_tokens=parameters.get('max_new_tokens') or parameters.get('max_tokens', self._default_params['max_tokens']), presence_penalty=parameters.get('presence_penalty', self._default_params['presence_penalty']), - frequency_penalty=parameters.get('frequency_penalty', self._default_params['frequency_penalty']) + frequency_penalty=parameters.get('frequency_penalty', self._default_params['frequency_penalty']), + length_penalty=parameters.get('length_penalty', self._default_params['length_penalty']) ) except ValueError as e: return None, str(e).strip('.') -- 2.34.1 From 69b8c1e35c81f5c1361d81cee2de720b7c137dd6 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 12:50:20 -0600 Subject: [PATCH 113/163] fix openai confusion --- llm_server/llm/openai/oai_to_vllm.py | 3 +++ llm_server/llm/vllm/vllm_backend.py | 3 ++- llm_server/routes/openai/chat_completions.py | 3 --- llm_server/routes/openai_request_handler.py | 28 +++++++++++++------- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/llm_server/llm/openai/oai_to_vllm.py b/llm_server/llm/openai/oai_to_vllm.py index cde5180..ef07a08 100644 --- a/llm_server/llm/openai/oai_to_vllm.py +++ b/llm_server/llm/openai/oai_to_vllm.py @@ -28,6 +28,9 @@ def oai_to_vllm(request_json_body, stop_hashes: bool, mode): request_json_body['top_p'] = 0.01 request_json_body['max_tokens'] = min(max(request_json_body.get('max_new_tokens', 0), request_json_body.get('max_tokens', 0)), opts.max_new_tokens) + if request_json_body['max_tokens'] == 0: + # We don't want to set any defaults here. + del request_json_body['max_tokens'] return request_json_body diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index 835d2ce..9665547 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -40,7 +40,8 @@ class VLLMBackend(LLMBackend): max_tokens=parameters.get('max_new_tokens') or parameters.get('max_tokens', self._default_params['max_tokens']), presence_penalty=parameters.get('presence_penalty', self._default_params['presence_penalty']), frequency_penalty=parameters.get('frequency_penalty', self._default_params['frequency_penalty']), - length_penalty=parameters.get('length_penalty', self._default_params['length_penalty']) + length_penalty=parameters.get('length_penalty', self._default_params['length_penalty']), + early_stopping=parameters.get('early_stopping', self._default_params['early_stopping']) ) except ValueError as e: return None, str(e).strip('.') diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index d054703..afa6fd1 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -34,9 +34,6 @@ def openai_chat_completions(model_name=None): if not request_json_body.get('stream'): try: - invalid_oai_err_msg = validate_oai(request_json_body) - if invalid_oai_err_msg: - return invalid_oai_err_msg return handler.handle_request() except Exception: traceback.print_exc() diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 3c2a5b1..246c3b6 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -38,10 +38,23 @@ class OpenAIRequestHandler(RequestHandler): oai_messages = self.request.json['messages'] self.prompt = transform_messages_to_prompt(oai_messages) + self.request_json_body = oai_to_vllm(self.request_json_body, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) + request_valid, invalid_response = self.validate_request() if not request_valid: return invalid_response + if not self.prompt: + # TODO: format this as an openai error message + return Response('Invalid prompt'), 400 + + # TODO: support Ooba backend + self.parameters = oai_to_vllm(self.parameters, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) + + invalid_oai_err_msg = validate_oai(self.request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg + if opts.openai_moderation_enabled and opts.openai_api_key and is_api_key_moderated(self.token): try: # Gather the last message from the user and all preceding system messages @@ -62,13 +75,6 @@ class OpenAIRequestHandler(RequestHandler): print(f'OpenAI moderation endpoint failed:', f'{e.__class__.__name__}: {e}') traceback.print_exc() - # TODO: support Ooba - self.parameters = oai_to_vllm(self.parameters, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) - - if not self.prompt: - # TODO: format this as an openai error message - return Response('Invalid prompt'), 400 - llm_request = {**self.parameters, 'prompt': self.prompt} (success, _, _, _), (backend_response, backend_response_status_code) = self.generate_response(llm_request) model = self.request_json_body.get('model') @@ -152,9 +158,13 @@ class OpenAIRequestHandler(RequestHandler): return response def validate_request(self, prompt: str = None, do_log: bool = False) -> Tuple[bool, Tuple[Response | None, int]]: - invalid_oai_err_msg = validate_oai(self.request_json_body) + self.parameters, parameters_invalid_msg = self.get_parameters() + if not self.parameters: + print('OAI BACKEND VALIDATION ERROR:', parameters_invalid_msg) + return False, (Response('Invalid request, check your parameters and try again.'), 400) + invalid_oai_err_msg = validate_oai(self.parameters) if invalid_oai_err_msg: return False, invalid_oai_err_msg - self.request_json_body = oai_to_vllm(self.request_json_body, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) + # self.request_json_body = oai_to_vllm(self.request_json_body, stop_hashes=('instruct' not in self.request_json_body['model'].lower()), mode=self.cluster_backend_info['mode']) # If the parameters were invalid, let the superclass deal with it. return super().validate_request(prompt, do_log) -- 2.34.1 From 169e216a38693ecafbd41ccb379c843183c78201 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 12:56:04 -0600 Subject: [PATCH 114/163] add background thread to gradio --- other/gradio/gradio_chat.py | 64 +++++++++++++++++++++++------------ other/gradio/requirements.txt | 3 ++ 2 files changed, 45 insertions(+), 22 deletions(-) create mode 100644 other/gradio/requirements.txt diff --git a/other/gradio/gradio_chat.py b/other/gradio/gradio_chat.py index fa1b892..af4aeeb 100644 --- a/other/gradio/gradio_chat.py +++ b/other/gradio/gradio_chat.py @@ -1,22 +1,48 @@ import os import sys +import time +import traceback import warnings +from threading import Thread import gradio as gr import openai +import requests warnings.filterwarnings("ignore") API_BASE = os.getenv('API_BASE') if not API_BASE: - print('Must set the secret variable API_BASE to your https://your-site/api/openai/v1') + print('Must set the secret variable API_BASE to your https://your-site/api') sys.exit(1) - -BACKUP_API_BASE = os.getenv('BACKUP_API_BASE') -if BACKUP_API_BASE: - print('Using BACKUP_API_BASE:', BACKUP_API_BASE) +API_BASE = API_BASE.strip('/') APP_TITLE = os.getenv('APP_TITLE') +PRIMARY_MODEL_CHOICE = os.getenv('PRIMARY_MODEL_CHOICE') + + +def background(): + while True: + previous = openai.api_base + try: + r = requests.get(API_BASE + '/stats').json() + if PRIMARY_MODEL_CHOICE in r['models']['choices'].keys(): + openai.api_base = API_BASE + '/openai/' + PRIMARY_MODEL_CHOICE + '/v1' + else: + openai.api_base = API_BASE + '/openai/v1' + except: + traceback.print_exc() + openai.api_base = API_BASE + '/openai/v1' + if openai.api_base != previous: + print('Set primary model to', openai.api_base) + time.sleep(10) + + +if PRIMARY_MODEL_CHOICE: + t = Thread(target=background) + t.daemon = True + t.start() + print('Started the background thread.') # A system prompt can be injected into the very first spot in the context. # If the user sends a message that contains the CONTEXT_TRIGGER_PHRASE, @@ -26,7 +52,7 @@ CONTEXT_TRIGGER_PHRASE = os.getenv('CONTEXT_TRIGGER_PHRASE') CONTEXT_TRIGGER_INJECTION = os.getenv('CONTEXT_TRIGGER_INJECTION') openai.api_key = 'null' -openai.api_base = API_BASE +openai.api_base = API_BASE + '/openai/v1' def stream_response(prompt, history): @@ -43,22 +69,16 @@ def stream_response(prompt, history): if do_injection or (CONTEXT_TRIGGER_INJECTION and CONTEXT_TRIGGER_PHRASE in prompt): messages.insert(0, {'role': 'system', 'content': CONTEXT_TRIGGER_INJECTION}) - for _ in range(2): - try: - response = openai.ChatCompletion.create( - model='0', - messages=messages, - temperature=0, - max_tokens=300, - stream=True - ) - break - except Exception: - openai.api_base = BACKUP_API_BASE - raise gr.Error("Failed to reach inference endpoint.") - - # Go back to the default endpoint - openai.api_base = API_BASE + try: + response = openai.ChatCompletion.create( + model='0', + messages=messages, + temperature=0, + max_tokens=300, + stream=True + ) + except Exception: + raise gr.Error("Failed to reach inference endpoint.") message = '' for chunk in response: diff --git a/other/gradio/requirements.txt b/other/gradio/requirements.txt new file mode 100644 index 0000000..eb4baac --- /dev/null +++ b/other/gradio/requirements.txt @@ -0,0 +1,3 @@ +gradio +openai +requests \ No newline at end of file -- 2.34.1 From 74cf8f309b7ad640db4b1db7c072f519d93f4fcc Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 18:04:15 -0600 Subject: [PATCH 115/163] clean up --- llm_server/database/conn.py | 8 ++++---- llm_server/routes/ooba_request_handler.py | 1 - llm_server/workers/inferencer.py | 2 ++ llm_server/workers/logger.py | 18 +++++++++++------- llm_server/workers/printer.py | 2 +- other/gradio/gradio_chat.py | 3 ++- 6 files changed, 20 insertions(+), 14 deletions(-) diff --git a/llm_server/database/conn.py b/llm_server/database/conn.py index 25f3326..f63f555 100644 --- a/llm_server/database/conn.py +++ b/llm_server/database/conn.py @@ -5,20 +5,20 @@ class DatabaseConnection: host: str = None username: str = None password: str = None - database: str = None + database_name: str = None - def init_db(self, host, username, password, database): + def init_db(self, host, username, password, database_name): self.host = host self.username = username self.password = password - self.database = database + self.database_name = database_name def cursor(self): db = pymysql.connect( host=self.host, user=self.username, password=self.password, - database=self.database, + database=self.database_name, charset='utf8mb4', autocommit=True, ) diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index c01bfed..ab20fbd 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -4,7 +4,6 @@ import flask from flask import jsonify, request from llm_server import opts -from llm_server.database.database import do_db_log from llm_server.database.log_to_db import log_to_db from llm_server.routes.helpers.client import format_sillytavern_err from llm_server.routes.request_handler import RequestHandler diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 79eed01..0357eab 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -35,6 +35,7 @@ def worker(backend_url): redis.publish(event_id, 'begin') for item in pubsub.listen(): if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': + # The streaming endpoint has said that it has finished break time.sleep(0.1) else: @@ -47,6 +48,7 @@ def worker(backend_url): finally: decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) + print('Worker finished processing for', client_ip) def start_workers(cluster: dict): diff --git a/llm_server/workers/logger.py b/llm_server/workers/logger.py index cb8dc01..eada969 100644 --- a/llm_server/workers/logger.py +++ b/llm_server/workers/logger.py @@ -1,4 +1,5 @@ import pickle +import traceback import redis @@ -17,11 +18,14 @@ def db_logger(): p.subscribe('database-logger') for message in p.listen(): - if message['type'] == 'message': - data = pickle.loads(message['data']) - function_name = data['function'] - args = data['args'] - kwargs = data['kwargs'] + try: + if message['type'] == 'message': + data = pickle.loads(message['data']) + function_name = data['function'] + args = data['args'] + kwargs = data['kwargs'] - if function_name == 'log_prompt': - do_db_log(*args, **kwargs) + if function_name == 'log_prompt': + do_db_log(*args, **kwargs) + except: + traceback.print_exc() diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index cf691c1..ed6ff65 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -25,4 +25,4 @@ def console_printer(): processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - time.sleep(1) + time.sleep(10) diff --git a/other/gradio/gradio_chat.py b/other/gradio/gradio_chat.py index af4aeeb..ee3d3ab 100644 --- a/other/gradio/gradio_chat.py +++ b/other/gradio/gradio_chat.py @@ -75,7 +75,8 @@ def stream_response(prompt, history): messages=messages, temperature=0, max_tokens=300, - stream=True + stream=True, + headers={'LLM-Source': 'huggingface-demo'} ) except Exception: raise gr.Error("Failed to reach inference endpoint.") -- 2.34.1 From 4e3985e156ad0c62090dcf9349ef0ae2043e6e12 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 18:17:02 -0600 Subject: [PATCH 116/163] fix wrong status code on openai streaming --- llm_server/routes/openai/chat_completions.py | 5 ++--- llm_server/routes/openai/completions.py | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index afa6fd1..99b7488 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -65,7 +65,6 @@ def openai_chat_completions(model_name=None): return 'Invalid prompt', 400 event_id = None - response_status_code = 0 start_time = time.time() request_valid, invalid_response = handler.validate_request() @@ -91,7 +90,7 @@ def openai_chat_completions(model_name=None): None, handler.parameters, request.headers, - response_status_code, + 429, request.url, handler.backend_url, ) @@ -159,7 +158,7 @@ def openai_chat_completions(model_name=None): elapsed_time, handler.parameters, r_headers, - response_status_code, + 200, r_url, handler.backend_url, ) diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 3dcde2e..8851fcc 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -90,7 +90,6 @@ def openai_completions(model_name=None): return 'DISABLED', 401 event_id = None - response_status_code = 0 start_time = time.time() request_valid, invalid_response = handler.validate_request() @@ -117,7 +116,7 @@ def openai_completions(model_name=None): None, handler.parameters, request.headers, - response_status_code, + 429, request.url, handler.backend_url, ) @@ -185,7 +184,7 @@ def openai_completions(model_name=None): elapsed_time, handler.parameters, r_headers, - response_status_code, + 200, r_url, handler.backend_url, ) -- 2.34.1 From ca7044bc90689cac1b577025c687e18b3428df57 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 11 Oct 2023 18:49:31 -0600 Subject: [PATCH 117/163] update gradio chat --- other/gradio/gradio_chat.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/other/gradio/gradio_chat.py b/other/gradio/gradio_chat.py index ee3d3ab..179e748 100644 --- a/other/gradio/gradio_chat.py +++ b/other/gradio/gradio_chat.py @@ -19,6 +19,7 @@ API_BASE = API_BASE.strip('/') APP_TITLE = os.getenv('APP_TITLE') PRIMARY_MODEL_CHOICE = os.getenv('PRIMARY_MODEL_CHOICE') +TRACKING_CODE = os.getenv('TRACKING_CODE') def background(): @@ -92,4 +93,11 @@ examples = ["hello"] if CONTEXT_TRIGGER_PHRASE: examples.insert(0, CONTEXT_TRIGGER_PHRASE) -gr.ChatInterface(stream_response, examples=examples, title=APP_TITLE, analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}').queue(concurrency_count=1, api_open=False).launch(show_api=False) +with gr.Blocks(analytics_enabled=False) as demo: + gr.ChatInterface(stream_response, examples=examples, title=APP_TITLE, analytics_enabled=False, cache_examples=False, css='#component-0{height:100%!important}') + + if TRACKING_CODE: + print('Inserting tracking code') + gr.HTML(TRACKING_CODE) + +demo.queue(concurrency_count=1, api_open=False).launch(show_api=False) -- 2.34.1 From 83f3ba8919a5e7cdccc9d3620f599a803943ce07 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 15 Oct 2023 15:11:37 -0600 Subject: [PATCH 118/163] trying to fix workers still processing after backend goes offline --- llm_server/llm/llm_backend.py | 1 - llm_server/messages.py | 1 + llm_server/routes/ooba_request_handler.py | 7 +++---- llm_server/routes/openai/chat_completions.py | 6 +++++- llm_server/routes/openai/completions.py | 6 +++++- llm_server/routes/request_handler.py | 4 ++++ llm_server/routes/v1/generate_stream.py | 7 +++++-- llm_server/workers/inferencer.py | 15 +++++++++------ server.py | 2 +- 9 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 llm_server/messages.py diff --git a/llm_server/llm/llm_backend.py b/llm_server/llm/llm_backend.py index 2ac2beb..f864b18 100644 --- a/llm_server/llm/llm_backend.py +++ b/llm_server/llm/llm_backend.py @@ -3,7 +3,6 @@ from typing import Tuple, Union import flask from llm_server.cluster.cluster_config import cluster_config -from llm_server.custom_redis import redis from llm_server.llm import get_token_count diff --git a/llm_server/messages.py b/llm_server/messages.py new file mode 100644 index 0000000..c7e3eb7 --- /dev/null +++ b/llm_server/messages.py @@ -0,0 +1 @@ +BACKEND_OFFLINE = 'The model you requested is not a valid choice. Please retry your query.' diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index ab20fbd..6966e32 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -3,7 +3,7 @@ from typing import Tuple import flask from flask import jsonify, request -from llm_server import opts +from llm_server import messages, opts from llm_server.database.log_to_db import log_to_db from llm_server.routes.helpers.client import format_sillytavern_err from llm_server.routes.request_handler import RequestHandler @@ -16,9 +16,8 @@ class OobaRequestHandler(RequestHandler): def handle_request(self, return_ok: bool = True): assert not self.used if self.offline: - msg = 'The model you requested is not a valid choice. Please retry your query.' - print(msg) - self.handle_error(msg) + print(messages.BACKEND_OFFLINE) + self.handle_error(messages.BACKEND_OFFLINE) request_valid, invalid_response = self.validate_request() if not request_valid: diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 99b7488..4018fee 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -12,7 +12,7 @@ from ..queue import priority_queue from ... import opts from ...database.log_to_db import log_to_db from ...llm.generator import generator -from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai, return_invalid_model_err +from ...llm.openai.oai_to_vllm import oai_to_vllm, return_invalid_model_err, validate_oai from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit @@ -106,6 +106,10 @@ def openai_chat_completions(model_name=None): break time.sleep(0.1) + # Double check the model is still online + if not handler.check_online(): + return return_invalid_model_err(handler.request_json_body['model']) + try: r_headers = dict(request.headers) r_url = request.url diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 8851fcc..4dda2f2 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -13,7 +13,7 @@ from ... import opts from ...database.log_to_db import log_to_db from ...llm import get_token_count from ...llm.generator import generator -from ...llm.openai.oai_to_vllm import oai_to_vllm, validate_oai, return_invalid_model_err +from ...llm.openai.oai_to_vllm import oai_to_vllm, return_invalid_model_err, validate_oai from ...llm.openai.transform import generate_oai_string, trim_string_to_fit @@ -131,6 +131,10 @@ def openai_completions(model_name=None): break time.sleep(0.1) + # Double check the model is still online + if not handler.check_online(): + return return_invalid_model_err(handler.request_json_body['model']) + try: response = generator(msg_to_backend, handler.backend_url) r_headers = dict(request.headers) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 6a6ad4a..4011030 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -58,6 +58,10 @@ class RequestHandler: # "recent_prompters" is only used for stats. redis.zadd('recent_prompters', {self.client_ip: time.time()}) + def check_online(self) -> bool: + self.cluster_backend_info = cluster_config.get_backend(self.backend_url) + return self.cluster_backend_info['online'] + def get_auth_token(self): if self.request_json_body.get('X-API-KEY'): return self.request_json_body['X-API-KEY'] diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index c55e36f..b918106 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -8,7 +8,7 @@ from . import bp from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import priority_queue -from ... import opts +from ... import messages, opts from ...custom_redis import redis from ...database.log_to_db import log_to_db from ...llm.generator import generator @@ -147,9 +147,12 @@ def do_stream(ws, model_name): break time.sleep(0.1) + # Double check the model is still online + if not handler.check_online(): + return messages.BACKEND_OFFLINE, 404 # TODO: format this error + try: response = generator(llm_request, handler.backend_url) - if not response: error_msg = 'Failed to reach backend while streaming.' print('Streaming failed:', error_msg) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 0357eab..0a9d871 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,12 +1,12 @@ import threading import time import traceback -from uuid import uuid4 +from llm_server import messages from llm_server.cluster.cluster_config import cluster_config -from llm_server.custom_redis import redis, RedisCustom +from llm_server.custom_redis import redis from llm_server.llm.generator import generator -from llm_server.routes.queue import DataEvent, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count, RedisPriorityQueue, PriorityQueue, priority_queue +from llm_server.routes.queue import DataEvent, RedisPriorityQueue, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count def worker(backend_url): @@ -14,14 +14,18 @@ def worker(backend_url): while True: (request_json_body, client_ip, token, parameters), event_id, selected_model = queue.get() backend_info = cluster_config.get_backend(backend_url) + + if not backend_info['online']: + event = DataEvent(event_id) + event.set((False, None, messages.BACKEND_OFFLINE)) + return + if not selected_model: selected_model = backend_info['model'] increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - print('Worker starting processing for', client_ip) - try: if not request_json_body: # This was a dummy request from the streaming handlers. @@ -48,7 +52,6 @@ def worker(backend_url): finally: decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) - print('Worker finished processing for', client_ip) def start_workers(cluster: dict): diff --git a/server.py b/server.py index 89c71aa..490eebe 100644 --- a/server.py +++ b/server.py @@ -40,7 +40,7 @@ from llm_server.sock import init_socketio # TODO: if a backend is at its limit of concurrent requests, choose a different one # Lower priority -# TODO: fix moderation freezing after a while +# TODO: make error messages consitient # TODO: support logit_bias on OpenAI and Ooba endpoints. # TODO: add a way to cancel VLLM gens. Maybe use websockets? # TODO: validate openai_silent_trim works as expected and only when enabled -- 2.34.1 From b3f0c4b28f2893ce30f405586df77ee6bb6ba3e5 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 15 Oct 2023 15:14:32 -0600 Subject: [PATCH 119/163] remove debug print --- llm_server/routes/openai_request_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 246c3b6..549cc93 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -85,7 +85,6 @@ class OpenAIRequestHandler(RequestHandler): return backend_response, backend_response_status_code def handle_ratelimited(self, do_log: bool = True): - print('OAI ratelimited:', self.client_ip) model_choices, default_model = get_model_choices() default_model_info = model_choices[default_model] w = int(default_model_info['estimated_wait']) if default_model_info['estimated_wait'] > 0 else 2 -- 2.34.1 From 3ec9b2347f0644df720365babd7d90363365e4a3 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 15 Oct 2023 17:24:18 -0600 Subject: [PATCH 120/163] fix wrong datatype --- llm_server/routes/openai/chat_completions.py | 8 ++++++-- llm_server/routes/openai/completions.py | 8 ++++++-- llm_server/routes/v1/generate_stream.py | 8 ++++++-- llm_server/workers/inferencer.py | 8 +++----- 4 files changed, 21 insertions(+), 11 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 4018fee..426fd98 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -102,8 +102,12 @@ def openai_chat_completions(model_name=None): pubsub = redis.pubsub() pubsub.subscribe(event_id) for item in pubsub.listen(): - if item['type'] == 'message' and item['data'].decode('utf-8') == 'begin': - break + if item['type'] == 'message': + msg = item['data'].decode('utf-8') + if msg == 'begin': + break + elif msg == 'offline': + return return_invalid_model_err(handler.request_json_body['model']) time.sleep(0.1) # Double check the model is still online diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 4dda2f2..2cd8578 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -127,8 +127,12 @@ def openai_completions(model_name=None): pubsub = redis.pubsub() pubsub.subscribe(event_id) for item in pubsub.listen(): - if item['type'] == 'message' and item['data'].decode('utf-8') == 'begin': - break + if item['type'] == 'message': + msg = item['data'].decode('utf-8') + if msg == 'begin': + break + elif msg == 'offline': + return return_invalid_model_err(handler.request_json_body['model']) time.sleep(0.1) # Double check the model is still online diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index b918106..79be511 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -143,8 +143,12 @@ def do_stream(ws, model_name): pubsub = redis.pubsub() pubsub.subscribe(event_id) for item in pubsub.listen(): - if item['type'] == 'message' and item['data'].decode('utf-8') == 'begin': - break + if item['type'] == 'message': + msg = item['data'].decode('utf-8') + if msg == 'begin': + break + elif msg == 'offline': + return messages.BACKEND_OFFLINE, 404 # TODO: format this error time.sleep(0.1) # Double check the model is still online diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 0a9d871..d65d125 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -2,7 +2,6 @@ import threading import time import traceback -from llm_server import messages from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.llm.generator import generator @@ -14,10 +13,11 @@ def worker(backend_url): while True: (request_json_body, client_ip, token, parameters), event_id, selected_model = queue.get() backend_info = cluster_config.get_backend(backend_url) + pubsub = redis.pubsub() + pubsub.subscribe(event_id) if not backend_info['online']: - event = DataEvent(event_id) - event.set((False, None, messages.BACKEND_OFFLINE)) + redis.publish(event_id, 'offline') return if not selected_model: @@ -34,8 +34,6 @@ def worker(backend_url): # is finished. Since a lot of ratelimiting and stats are # based off the number of active workers, we must keep # the generation based off the workers. - pubsub = redis.pubsub() - pubsub.subscribe(event_id) redis.publish(event_id, 'begin') for item in pubsub.listen(): if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': -- 2.34.1 From 31ab4188f1e8bb1329fc40ae4fd9e0a0b132eeac Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 15 Oct 2023 20:45:01 -0600 Subject: [PATCH 121/163] fix issues with queue and streaming --- llm_server/custom_redis.py | 4 +- llm_server/llm/oobabooga/ooba_backend.py | 77 +------------------- llm_server/llm/vllm/tokenize.py | 2 +- llm_server/routes/ooba_request_handler.py | 2 +- llm_server/routes/openai/chat_completions.py | 14 +++- llm_server/routes/queue.py | 45 ++++++++++-- llm_server/workers/inferencer.py | 61 ++++++++++++++-- llm_server/workers/mainer.py | 6 ++ llm_server/workers/printer.py | 6 +- 9 files changed, 119 insertions(+), 98 deletions(-) diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index 485cb58..aacaec0 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -200,10 +200,10 @@ class RedisCustom(Redis): return json.loads(r.decode("utf-8")) def setp(self, name, value): - self.redis.set(name, pickle.dumps(value)) + self.redis.set(self._key(name), pickle.dumps(value)) def getp(self, name: str): - r = self.redis.get(name) + r = self.redis.get(self._key(name)) if r: return pickle.loads(r) return r diff --git a/llm_server/llm/oobabooga/ooba_backend.py b/llm_server/llm/oobabooga/ooba_backend.py index 0e2b2d8..18fe6b1 100644 --- a/llm_server/llm/oobabooga/ooba_backend.py +++ b/llm_server/llm/oobabooga/ooba_backend.py @@ -1,79 +1,6 @@ -from flask import jsonify - -from llm_server.custom_redis import redis from ..llm_backend import LLMBackend -from ...database.database import do_db_log -from ...helpers import safe_list_get -from ...routes.helpers.client import format_sillytavern_err -from ...routes.helpers.http import validate_json class OobaboogaBackend(LLMBackend): - default_params = {} - - def handle_response(self, success, request, response, error_msg, client_ip, token, prompt, elapsed_time, parameters, headers): - raise NotImplementedError('need to implement default_params') - - backend_err = False - response_valid_json, response_json_body = validate_json(response) - if response: - try: - # Be extra careful when getting attributes from the response object - response_status_code = response.status_code - except: - response_status_code = 0 - else: - response_status_code = None - - # =============================================== - - # We encountered an error - if not success or not response or error_msg: - if not error_msg or error_msg == '': - error_msg = 'Unknown error.' - else: - error_msg = error_msg.strip('.') + '.' - backend_response = format_sillytavern_err(error_msg, error_type='error', backend_url=self.backend_url) - log_to_db(client_ip, token, prompt, backend_response, None, parameters, headers, response_status_code, request.url, is_error=True) - return jsonify({ - 'code': 500, - 'msg': error_msg, - 'results': [{'text': backend_response}] - }), 400 - - # =============================================== - - if response_valid_json: - backend_response = safe_list_get(response_json_body.get('results', []), 0, {}).get('text') - if not backend_response: - # Ooba doesn't return any error messages so we will just tell the client an error occurred - backend_err = True - backend_response = format_sillytavern_err( - f'Backend (oobabooga) returned an empty string. This is usually due to an error on the backend during inference. Please check your parameters and try again.', - error_type='error', - backend_url=self.backend_url) - response_json_body['results'][0]['text'] = backend_response - - if not backend_err: - redis.incr('proompts') - - log_to_db(client_ip, token, prompt, backend_response, elapsed_time if not backend_err else None, parameters, headers, response_status_code, request.url, response_tokens=response_json_body.get('details', {}).get('generated_tokens'), is_error=backend_err) - return jsonify({ - **response_json_body - }), 200 - else: - backend_response = format_sillytavern_err(f'The backend did not return valid JSON.', error_type='error', backend_url=self.backend_url) - log_to_db(client_ip, token, prompt, backend_response, elapsed_time, parameters, headers, response.status_code, request.url, is_error=True) - return jsonify({ - 'code': 500, - 'msg': 'the backend did not return valid JSON', - 'results': [{'text': backend_response}] - }), 400 - - def validate_params(self, params_dict: dict): - # No validation required - return True, None - - def get_parameters(self, parameters): - del parameters['prompt'] - return parameters + def __int__(self): + return diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 8b18073..69a2b14 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -33,7 +33,7 @@ def tokenize(prompt: str, backend_url: str) -> int: j = r.json() return j['length'] except Exception as e: - print(f'Failed to tokenize using VLLM -', f'{e.__class__.__name__}: {e}') + print(f'Failed to tokenize using VLLM - {e.__class__.__name__}') return len(tokenizer.encode(chunk)) + 10 # Use a ThreadPoolExecutor to send all chunks to the server at once diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 6966e32..804be74 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -17,7 +17,7 @@ class OobaRequestHandler(RequestHandler): assert not self.used if self.offline: print(messages.BACKEND_OFFLINE) - self.handle_error(messages.BACKEND_OFFLINE) + return self.handle_error(messages.BACKEND_OFFLINE) request_valid, invalid_response = self.validate_request() if not request_valid: diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 426fd98..87a7330 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -79,6 +79,7 @@ def openai_chat_completions(model_name=None): event = None if not handler.is_client_ratelimited(): + start_time = time.time() # Add a dummy event to the queue and wait for it to reach a worker event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) if not event: @@ -102,11 +103,14 @@ def openai_chat_completions(model_name=None): pubsub = redis.pubsub() pubsub.subscribe(event_id) for item in pubsub.listen(): + if time.time() - start_time >= opts.backend_generate_request_timeout: + raise Exception('Inferencer timed out waiting for streaming to complete:', request_json_body) if item['type'] == 'message': msg = item['data'].decode('utf-8') if msg == 'begin': break elif msg == 'offline': + # This shouldn't happen because the best model should be auto-selected. return return_invalid_model_err(handler.request_json_body['model']) time.sleep(0.1) @@ -135,6 +139,7 @@ def openai_chat_completions(model_name=None): json_obj = json.loads(json_str.decode()) new = json_obj['text'][0].split(handler.prompt + generated_text)[1] generated_text = generated_text + new + redis.publish(event_id, 'chunk') # Keepalive except IndexError: # ???? continue @@ -170,9 +175,14 @@ def openai_chat_completions(model_name=None): r_url, handler.backend_url, ) + except GeneratorExit: + yield 'data: [DONE]\n\n' + except: + # AttributeError: 'bool' object has no attribute 'iter_content' + traceback.print_exc() + yield 'data: [DONE]\n\n' finally: - # After completing inference, we need to tell the worker we - # are finished. + # After completing inference, we need to tell the worker we are finished. if event_id: # may be None if ratelimited. redis.publish(event_id, 'finished') else: diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 24bc019..3e4279f 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -6,6 +6,7 @@ from uuid import uuid4 from redis import Redis +from llm_server import opts from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import RedisCustom, redis from llm_server.database.database import get_token_ratelimit @@ -23,9 +24,14 @@ def decrement_ip_count(client_ip: str, redis_key): class RedisPriorityQueue: def __init__(self, name, db: int = 12): + self.name = name self.redis = RedisCustom(name, db=db) def put(self, item, priority, selected_model): + assert item is not None + assert priority is not None + assert selected_model is not None + event = DataEvent() # Check if the IP is already in the dictionary and if it has reached the limit ip_count = self.redis.hget('queued_ip_count', item[1]) @@ -36,7 +42,8 @@ class RedisPriorityQueue: print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') return None # reject the request - self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model)): -priority}) + timestamp = time.time() + self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model, timestamp)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event @@ -52,11 +59,13 @@ class RedisPriorityQueue: def print_all_items(self): items = self.redis.zrange('queue', 0, -1) + to_print = [] for item in items: - print(item.decode('utf-8')) + to_print.append(item.decode('utf-8')) + print(f'ITEMS {self.name} -->', to_print) def increment_ip_count(self, client_ip: str, redis_key): - new_count = self.redis.hincrby(redis_key, client_ip, 1) + self.redis.hincrby(redis_key, client_ip, 1) def decrement_ip_count(self, client_ip: str, redis_key): new_count = self.redis.hincrby(redis_key, client_ip, -1) @@ -75,6 +84,16 @@ class RedisPriorityQueue: def flush(self): self.redis.flush() + def cleanup(self): + now = time.time() + items = self.redis.zrange('queue', 0, -1) + for item in items: + item_data = json.loads(item) + timestamp = item_data[-1] + if now - timestamp > opts.backend_generate_request_timeout * 3: # TODO: config option + self.redis.zrem('queue', item) + print('removed item from queue:', item) + class DataEvent: def __init__(self, event_id=None): @@ -112,7 +131,7 @@ def decr_active_workers(selected_model: str, backend_url: str): class PriorityQueue: - def __init__(self, backends: list = None): + def __init__(self, backends: set = None): """ Only have to load the backends once. :param backends: @@ -120,10 +139,10 @@ class PriorityQueue: self.redis = Redis(host='localhost', port=6379, db=9) if backends: for item in backends: - self.redis.lpush('backends', item) + self.redis.sadd('backends', item) def get_backends(self): - return [x.decode('utf-8') for x in self.redis.lrange('backends', 0, -1)] + return {x.decode('utf-8') for x in self.redis.smembers('backends')} def get_queued_ip_count(self, client_ip: str): count = 0 @@ -136,22 +155,32 @@ class PriorityQueue: queue = RedisPriorityQueue(backend_url) return queue.put(item, priority, selected_model) + def activity(self): + lines = [] + status_redis = RedisCustom('worker_status') + for worker in status_redis.keys(): + lines.append((worker, status_redis.getp(worker))) + return sorted(lines) + def len(self, model_name): count = 0 - backends_with_models = [] + backends_with_models = set() for k in self.get_backends(): info = cluster_config.get_backend(k) if info.get('model') == model_name: - backends_with_models.append(k) + backends_with_models.add(k) for backend_url in backends_with_models: count += len(RedisPriorityQueue(backend_url)) return count def __len__(self): count = 0 + p = set() for backend_url in self.get_backends(): queue = RedisPriorityQueue(backend_url) + p.add((backend_url, len(queue))) count += len(queue) + print(p) return count def flush(self): diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index d65d125..f41d0d3 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,20 +1,48 @@ +import queue import threading import time import traceback +from uuid import uuid4 +from redis.client import PubSub + +from llm_server import opts from llm_server.cluster.cluster_config import cluster_config -from llm_server.custom_redis import redis +from llm_server.custom_redis import RedisCustom, redis from llm_server.llm.generator import generator from llm_server.routes.queue import DataEvent, RedisPriorityQueue, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count +class ListenerThread(threading.Thread): + def __init__(self, pubsub: PubSub, listener_queue: queue.Queue, stop_event: threading.Event): + threading.Thread.__init__(self) + self.pubsub = pubsub + self.listener_queue = listener_queue + self.stop_event = stop_event + + def run(self): + while not self.stop_event.is_set(): + message = self.pubsub.get_message() + if message: + self.listener_queue.put(message) + time.sleep(0.1) + + def worker(backend_url): - queue = RedisPriorityQueue(backend_url) + status_redis = RedisCustom('worker_status') + worker_id = uuid4() + status_redis.setp(str(worker_id), None) + redis_queue = RedisPriorityQueue(backend_url) while True: - (request_json_body, client_ip, token, parameters), event_id, selected_model = queue.get() + (request_json_body, client_ip, token, parameters), event_id, selected_model, timestamp = redis_queue.get() backend_info = cluster_config.get_backend(backend_url) + pubsub = redis.pubsub() pubsub.subscribe(event_id) + stop_event = threading.Event() + q = queue.Queue() + listener = ListenerThread(pubsub, q, stop_event) + listener.start() if not backend_info['online']: redis.publish(event_id, 'offline') @@ -26,6 +54,8 @@ def worker(backend_url): increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) + status_redis.setp(str(worker_id), (backend_url, client_ip)) + try: if not request_json_body: # This was a dummy request from the streaming handlers. @@ -34,13 +64,27 @@ def worker(backend_url): # is finished. Since a lot of ratelimiting and stats are # based off the number of active workers, we must keep # the generation based off the workers. + start_time = time.time() redis.publish(event_id, 'begin') - for item in pubsub.listen(): - if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': - # The streaming endpoint has said that it has finished + while True: + status_redis.setp(str(worker_id), (f'waiting for streaming to complete - {time.time() - start_time} - {opts.backend_generate_request_timeout}', client_ip)) + + try: + item = q.get(timeout=30) + except queue.Empty: + print('Inferencer timed out waiting for chunk from streamer:', (request_json_body, client_ip, token, parameters), event_id, selected_model) + status_redis.setp(str(worker_id), ('streaming chunk timed out', client_ip)) + break + + if time.time() - start_time >= opts.backend_generate_request_timeout: + status_redis.setp(str(worker_id), ('streaming timed out', client_ip)) + print('Inferencer timed out waiting for streaming to complete:', (request_json_body, client_ip, token, parameters), event_id, selected_model) + break + if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': + status_redis.setp(str(worker_id), ('streaming completed', client_ip)) break - time.sleep(0.1) else: + status_redis.setp(str(worker_id), ('generating', client_ip)) # Normal inference (not streaming). success, response, error_msg = generator(request_json_body, backend_url) event = DataEvent(event_id) @@ -48,8 +92,11 @@ def worker(backend_url): except: traceback.print_exc() finally: + stop_event.set() # make sure to stop the listener thread + listener.join() decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) + status_redis.setp(str(worker_id), None) def start_workers(cluster: dict): diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py index 37c1178..d342f4b 100644 --- a/llm_server/workers/mainer.py +++ b/llm_server/workers/mainer.py @@ -7,6 +7,7 @@ from llm_server.cluster.cluster_config import cluster_config, get_backends from llm_server.custom_redis import redis from llm_server.database.database import weighted_average_column_for_model from llm_server.llm.info import get_info +from llm_server.routes.queue import RedisPriorityQueue, priority_queue def main_background_thread(): @@ -35,6 +36,11 @@ def main_background_thread(): except Exception as e: print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}') + backends = priority_queue.get_backends() + for backend_url in backends: + queue = RedisPriorityQueue(backend_url) + queue.cleanup() + time.sleep(30) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index ed6ff65..a3da690 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -24,5 +24,7 @@ def console_printer(): for k in processing: processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] - logger.info(f'REQUEST QUEUE -> Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - time.sleep(10) + activity = priority_queue.activity() + print(activity) + logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') + time.sleep(1) -- 2.34.1 From 381bdb950f3e8e0c24ab118c75b5b8ca680ede58 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 15 Oct 2023 20:46:32 -0600 Subject: [PATCH 122/163] remove debug print --- llm_server/routes/queue.py | 1 - llm_server/workers/printer.py | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 3e4279f..dcbdfc2 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -180,7 +180,6 @@ class PriorityQueue: queue = RedisPriorityQueue(backend_url) p.add((backend_url, len(queue))) count += len(queue) - print(p) return count def flush(self): diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index a3da690..c28e383 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -25,6 +25,8 @@ def console_printer(): processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] activity = priority_queue.activity() - print(activity) + + # TODO: Active Workers and Processing should read the same. If not, that's an issue + logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - time.sleep(1) + time.sleep(10) -- 2.34.1 From 24aab3cd938c3a054ee22d1984b647072167c856 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 15 Oct 2023 20:59:11 -0600 Subject: [PATCH 123/163] fix streaming disabled --- llm_server/routes/openai/chat_completions.py | 2 +- llm_server/routes/openai/completions.py | 2 +- llm_server/routes/v1/generate_stream.py | 2 +- llm_server/workers/mainer.py | 1 + 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 87a7330..b1e74be 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -40,7 +40,7 @@ def openai_chat_completions(model_name=None): return 'Internal server error', 500 else: if not opts.enable_streaming: - return + return 'Streaming disabled', 403 handler.parameters, _ = handler.get_parameters() handler.request_json_body = { diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 2cd8578..dc7f9e6 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -87,7 +87,7 @@ def openai_completions(model_name=None): return response, 200 else: if not opts.enable_streaming: - return 'DISABLED', 401 + return 'Streaming disabled', 403 event_id = None start_time = time.time() diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 79be511..332fe4c 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -61,7 +61,7 @@ def do_stream(ws, model_name): ) if not opts.enable_streaming: - return 'Streaming is disabled', 500 + return 'Streaming disabled', 403 r_headers = dict(request.headers) r_url = request.url diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py index d342f4b..e06e803 100644 --- a/llm_server/workers/mainer.py +++ b/llm_server/workers/mainer.py @@ -36,6 +36,7 @@ def main_background_thread(): except Exception as e: print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}') + # TODO: test backends = priority_queue.get_backends() for backend_url in backends: queue = RedisPriorityQueue(backend_url) -- 2.34.1 From 151b3e476904538f32af2fe94bc313fbb872bed3 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 00:18:05 -0600 Subject: [PATCH 124/163] begin streaming rewrite --- llm_server/routes/openai/chat_completions.py | 124 +++++++------------ llm_server/routes/queue.py | 9 +- llm_server/workers/cleaner.py | 32 +++++ llm_server/workers/inferencer.py | 112 ++++++++--------- llm_server/workers/threader.py | 1 - 5 files changed, 137 insertions(+), 141 deletions(-) create mode 100644 llm_server/workers/cleaner.py diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index b1e74be..a840070 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -1,8 +1,10 @@ import json +import pickle import time import traceback from flask import Response, jsonify, request +from redis import Redis from llm_server.custom_redis import redis from . import openai_bp, openai_model_bp @@ -11,7 +13,6 @@ from ..openai_request_handler import OpenAIRequestHandler from ..queue import priority_queue from ... import opts from ...database.log_to_db import log_to_db -from ...llm.generator import generator from ...llm.openai.oai_to_vllm import oai_to_vllm, return_invalid_model_err, validate_oai from ...llm.openai.transform import generate_oai_string, transform_messages_to_prompt, trim_messages_to_fit @@ -64,24 +65,18 @@ def openai_chat_completions(model_name=None): # Prevent issues on the backend. return 'Invalid prompt', 400 - event_id = None + # Need to set the prompt in the JSON body since that's what the inference worker expects. + handler.request_json_body['prompt'] = handler.prompt + start_time = time.time() request_valid, invalid_response = handler.validate_request() if not request_valid: return invalid_response else: - msg_to_backend = { - **handler.parameters, - 'prompt': handler.prompt, - 'stream': True, - } - event = None if not handler.is_client_ratelimited(): - start_time = time.time() - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) + event = priority_queue.put(handler.backend_url, (handler.request_json_body, handler.client_ip, handler.token, handler.parameters), handler.token_priority, handler.selected_model, do_stream=True) if not event: log_to_db( handler.client_ip, @@ -97,27 +92,6 @@ def openai_chat_completions(model_name=None): ) return handler.handle_ratelimited() - # Once the worker receives our streaming request, it will tell us we are ready - # to begin inference. - event_id = event.event_id - pubsub = redis.pubsub() - pubsub.subscribe(event_id) - for item in pubsub.listen(): - if time.time() - start_time >= opts.backend_generate_request_timeout: - raise Exception('Inferencer timed out waiting for streaming to complete:', request_json_body) - if item['type'] == 'message': - msg = item['data'].decode('utf-8') - if msg == 'begin': - break - elif msg == 'offline': - # This shouldn't happen because the best model should be auto-selected. - return return_invalid_model_err(handler.request_json_body['model']) - time.sleep(0.1) - - # Double check the model is still online - if not handler.check_online(): - return return_invalid_model_err(handler.request_json_body['model']) - try: r_headers = dict(request.headers) r_url = request.url @@ -125,68 +99,62 @@ def openai_chat_completions(model_name=None): oai_string = generate_oai_string(30) def generate(): + stream_name = event.wait() + stream_redis = Redis(db=8) + generated_text = '' try: - response = generator(msg_to_backend, handler.backend_url) - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - redis.publish(event_id, 'chunk') # Keepalive - except IndexError: - # ???? - continue - - data = { + while True: + stream_data = stream_redis.xread({stream_name: '0-0'}, block=30000) + if not stream_data: + print("No message received in 30 seconds, closing stream.") + yield 'data: [DONE]\n\n' + else: + for r_timestamp, item in stream_data[0][1]: + timestamp = int(r_timestamp.decode('utf-8').split('-')[0]) + data = pickle.loads(item[b'data']) + if data['error']: + yield 'data: [DONE]\n\n' + return + elif data['new']: + response = { "id": f"chatcmpl-{oai_string}", "object": "chat.completion.chunk", - "created": int(time.time()), + "created": timestamp, "model": model, "choices": [ { "index": 0, "delta": { - "content": new + "content": data['new'] }, "finish_reason": None } ] } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - 200, - r_url, - handler.backend_url, - ) - except GeneratorExit: - yield 'data: [DONE]\n\n' - except: - # AttributeError: 'bool' object has no attribute 'iter_content' + generated_text = generated_text + data['new'] + yield f'data: {json.dumps(response)}\n\n' + elif data['completed']: + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + 200, + r_url, + handler.backend_url, + ) + return + except (Exception, GeneratorExit): traceback.print_exc() yield 'data: [DONE]\n\n' finally: - # After completing inference, we need to tell the worker we are finished. - if event_id: # may be None if ratelimited. - redis.publish(event_id, 'finished') - else: - print('event_id was None!') + stream_redis.delete(stream_name) return Response(generate(), mimetype='text/event-stream') except Exception: diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index dcbdfc2..cb4aaf5 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -27,7 +27,7 @@ class RedisPriorityQueue: self.name = name self.redis = RedisCustom(name, db=db) - def put(self, item, priority, selected_model): + def put(self, item, priority: int, selected_model: str, do_stream: bool = False): assert item is not None assert priority is not None assert selected_model is not None @@ -43,7 +43,7 @@ class RedisPriorityQueue: return None # reject the request timestamp = time.time() - self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model, timestamp)): -priority}) + self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model, timestamp, do_stream)): -priority}) self.increment_ip_count(item[1], 'queued_ip_count') return event @@ -106,6 +106,7 @@ class DataEvent: self.redis.publish(self.event_id, pickle.dumps(data)) def wait(self): + # TODO: implement timeout for item in self.pubsub.listen(): if item['type'] == 'message': return pickle.loads(item['data']) @@ -151,9 +152,9 @@ class PriorityQueue: count += queue.get_queued_ip_count(client_ip) return count - def put(self, backend_url, item: Tuple[dict, str, str, dict], priority: int, selected_model: str): + def put(self, backend_url, item: Tuple[dict, str, str, dict], priority: int, selected_model: str, do_stream: bool = False): queue = RedisPriorityQueue(backend_url) - return queue.put(item, priority, selected_model) + return queue.put(item, priority, selected_model, do_stream) def activity(self): lines = [] diff --git a/llm_server/workers/cleaner.py b/llm_server/workers/cleaner.py new file mode 100644 index 0000000..95a6a78 --- /dev/null +++ b/llm_server/workers/cleaner.py @@ -0,0 +1,32 @@ +import time + +from redis import Redis + +from llm_server.workers.inferencer import STREAM_NAME_PREFIX + + +# NOT NEEDED + +def cleaner(): + r = Redis(db=8) + stream_info = {} + + while True: + all_streams = r.keys(f'{STREAM_NAME_PREFIX}:*') + processed_streams = [] + for stream in all_streams: + stream = stream.decode() + current_size = r.xlen(stream) + + # If the stream is new or its size has changed, update the size and time in the dictionary + if stream not in stream_info or current_size != stream_info[stream]['size']: + stream_info[stream] = {'size': current_size, 'time': time.time()} + processed_streams.append(stream) + else: + # If the size hasn't changed for 5 minutes, delete the stream + if time.time() - stream_info[stream]['time'] >= 300: + r.delete(stream) + print(f"Stream '{stream}' deleted due to inactivity.") + del stream_info[stream] + + time.sleep(60) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index f41d0d3..a8a73e0 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,90 +1,88 @@ -import queue +import json +import pickle import threading -import time import traceback from uuid import uuid4 -from redis.client import PubSub +from redis import Redis -from llm_server import opts from llm_server.cluster.cluster_config import cluster_config -from llm_server.custom_redis import RedisCustom, redis +from llm_server.custom_redis import RedisCustom from llm_server.llm.generator import generator from llm_server.routes.queue import DataEvent, RedisPriorityQueue, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count +stream_redis = Redis(db=8) -class ListenerThread(threading.Thread): - def __init__(self, pubsub: PubSub, listener_queue: queue.Queue, stop_event: threading.Event): - threading.Thread.__init__(self) - self.pubsub = pubsub - self.listener_queue = listener_queue - self.stop_event = stop_event +STREAM_NAME_PREFIX = 'stream' - def run(self): - while not self.stop_event.is_set(): - message = self.pubsub.get_message() - if message: - self.listener_queue.put(message) - time.sleep(0.1) + +def get_stream_name(name: str): + return f'{STREAM_NAME_PREFIX}:{name}' + + +def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str): + prompt = msg_to_backend['prompt'] + stream_name = get_stream_name(stream_name) + stream_redis.delete(get_stream_name(stream_name)) # be extra sure + try: + response = generator(msg_to_backend, backend_url) + generated_text = '' + partial_response = b'' + for chunk in response.iter_content(chunk_size=1): + partial_response += chunk + if partial_response.endswith(b'\x00'): + json_strs = partial_response.split(b'\x00') + for json_str in json_strs: + if json_str: + try: + json_obj = json.loads(json_str.decode()) + new = json_obj['text'][0].split(prompt + generated_text)[1] + generated_text = generated_text + new + except IndexError: + # ???? + continue + stream_redis.xadd(stream_name, {'data': pickle.dumps({'new': new, 'completed': False, 'error': None})}) + except Exception as e: + stream_redis.xadd(stream_name, {'data': pickle.dumps({'new': None, 'completed': True, 'error': f'{e.__class__.__name__}: {e}'})}) + traceback.print_exc() + finally: + # Publish final message to Redis stream + stream_redis.xadd(stream_name, {'data': pickle.dumps({'new': None, 'completed': True, 'error': None})}) def worker(backend_url): status_redis = RedisCustom('worker_status') - worker_id = uuid4() + worker_id = str(uuid4()) status_redis.setp(str(worker_id), None) redis_queue = RedisPriorityQueue(backend_url) while True: - (request_json_body, client_ip, token, parameters), event_id, selected_model, timestamp = redis_queue.get() + (request_json_body, client_ip, token, parameters), event_id, selected_model, timestamp, do_stream = redis_queue.get() backend_info = cluster_config.get_backend(backend_url) - pubsub = redis.pubsub() - pubsub.subscribe(event_id) - stop_event = threading.Event() - q = queue.Queue() - listener = ListenerThread(pubsub, q, stop_event) - listener.start() - if not backend_info['online']: - redis.publish(event_id, 'offline') + # TODO: communicate to caller + # redis.publish(event_id, 'offline') return if not selected_model: selected_model = backend_info['model'] + stream_redis.delete(get_stream_name(worker_id)) # clean up any old streams increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - - status_redis.setp(str(worker_id), (backend_url, client_ip)) + status_redis.setp(str(worker_id), ('generating', client_ip)) try: - if not request_json_body: - # This was a dummy request from the streaming handlers. - # The worker will let the handler do the streaming instead - # of the worker. The worker will block until the handler - # is finished. Since a lot of ratelimiting and stats are - # based off the number of active workers, we must keep - # the generation based off the workers. - start_time = time.time() - redis.publish(event_id, 'begin') - while True: - status_redis.setp(str(worker_id), (f'waiting for streaming to complete - {time.time() - start_time} - {opts.backend_generate_request_timeout}', client_ip)) - - try: - item = q.get(timeout=30) - except queue.Empty: - print('Inferencer timed out waiting for chunk from streamer:', (request_json_body, client_ip, token, parameters), event_id, selected_model) - status_redis.setp(str(worker_id), ('streaming chunk timed out', client_ip)) - break - - if time.time() - start_time >= opts.backend_generate_request_timeout: - status_redis.setp(str(worker_id), ('streaming timed out', client_ip)) - print('Inferencer timed out waiting for streaming to complete:', (request_json_body, client_ip, token, parameters), event_id, selected_model) - break - if item['type'] == 'message' and item['data'].decode('utf-8') == 'finished': - status_redis.setp(str(worker_id), ('streaming completed', client_ip)) - break + if do_stream: + event = DataEvent(event_id) + event.set(get_stream_name(worker_id)) + msg_to_backend = { + **parameters, + 'prompt': request_json_body['prompt'], + 'stream': True, + } + inference_do_stream(worker_id, msg_to_backend, backend_url) else: - status_redis.setp(str(worker_id), ('generating', client_ip)) # Normal inference (not streaming). success, response, error_msg = generator(request_json_body, backend_url) event = DataEvent(event_id) @@ -92,8 +90,6 @@ def worker(backend_url): except: traceback.print_exc() finally: - stop_event.set() # make sure to stop the listener thread - listener.join() decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) status_redis.setp(str(worker_id), None) diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index f19ce1c..0e47c02 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -2,7 +2,6 @@ import time from threading import Thread from llm_server import opts -from llm_server.cluster.stores import redis_running_models from llm_server.cluster.worker import cluster_worker from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.inferencer import start_workers -- 2.34.1 From 2c7773cc4f147e1f9a07522809291def37c641ee Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 16:22:52 -0600 Subject: [PATCH 125/163] get streaming working again --- llm_server/custom_redis.py | 9 + llm_server/database/log_to_db.py | 3 + llm_server/netdata.py | 52 ----- llm_server/routes/openai/chat_completions.py | 187 ++++++++-------- llm_server/routes/openai/completions.py | 221 ++++++++++--------- llm_server/routes/openai_request_handler.py | 4 - llm_server/routes/request_handler.py | 3 + llm_server/routes/v1/generate_stream.py | 155 ++++--------- llm_server/sock.py | 2 +- llm_server/workers/inferencer.py | 18 +- llm_server/workers/printer.py | 2 +- other/ooba-test-streaming.py | 4 +- server.py | 9 +- 13 files changed, 296 insertions(+), 373 deletions(-) delete mode 100644 llm_server/netdata.py diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index aacaec0..60e4dbd 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -223,5 +223,14 @@ class RedisCustom(Redis): self.flush() return True + def lrange(self, name: str, start: int, end: int): + return self.redis.lrange(self._key(name), start, end) + + def delete(self, *names: KeyT): + return self.redis.delete(*[self._key(i) for i in names]) + + def lpop(self, name: str, count: Optional[int] = None): + return self.redis.lpop(self._key(name), count) + redis = RedisCustom('local_llm') diff --git a/llm_server/database/log_to_db.py b/llm_server/database/log_to_db.py index be6946f..75bcaab 100644 --- a/llm_server/database/log_to_db.py +++ b/llm_server/database/log_to_db.py @@ -5,6 +5,9 @@ from redis import Redis def log_to_db(ip: str, token: str, prompt: str, response: Union[str, None], gen_time: Union[int, float, None], parameters: dict, headers: dict, backend_response_code: int, request_url: str, backend_url: str, response_tokens: int = None, is_error: bool = False): + assert isinstance(prompt, str) + assert isinstance(backend_url, str) + r = Redis(host='localhost', port=6379, db=3) data = { 'function': 'log_prompt', diff --git a/llm_server/netdata.py b/llm_server/netdata.py deleted file mode 100644 index f37c109..0000000 --- a/llm_server/netdata.py +++ /dev/null @@ -1,52 +0,0 @@ -import json -from datetime import datetime, timedelta - -import requests - -from llm_server import opts - - -def get_power_states(): - gpu_num = 0 - output = {} - while True: - url = f"{opts.netdata_root}/api/v1/data?chart=nvidia_smi.gpu{gpu_num}_power_state" - try: - response = requests.get(url, timeout=10) - if response.status_code != 200: - break - data = json.loads(response.text) - power_state_data = data['data'][0] - power_state = None - for i in range(1, len(power_state_data)): - if power_state_data[i] == 1: - power_state = data['labels'][i] - break - output[f'gpu{gpu_num}'] = int(power_state.lower().strip('p')) - except Exception as e: - print('Failed to fetch Netdata metrics:', e) - return output - gpu_num += 1 - return output - - -def get_gpu_wh(gpu_id: int): - chart_name = f"nvidia_smi.gpu{gpu_id}_power" - now = datetime.now() - one_hour_ago = now - timedelta(hours=1) - num_seconds = int((now - one_hour_ago).total_seconds()) - params = { - "chart": chart_name, - "after": int(one_hour_ago.timestamp()), - "before": int(now.timestamp()), - "points": num_seconds, - "group": "second", - "format": "json", - "options": "absolute|jsonwrap" - } - response = requests.get(f'{opts.netdata_root}/api/v1/data', params=params, timeout=10) - data = json.loads(response.text) - total_power_usage_watts = sum(point[1] for point in data['result']['data']) - # total_power_usage_watt_hours = round(total_power_usage_watts / 3600, 1) - total_power_usage_kwh = round(total_power_usage_watts / 1000 / 3600, 3) - return total_power_usage_kwh diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index a840070..44d5172 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -43,24 +43,23 @@ def openai_chat_completions(model_name=None): if not opts.enable_streaming: return 'Streaming disabled', 403 - handler.parameters, _ = handler.get_parameters() - handler.request_json_body = { - 'messages': handler.request_json_body['messages'], - 'model': handler.request_json_body['model'], - **handler.parameters - } - invalid_oai_err_msg = validate_oai(handler.request_json_body) if invalid_oai_err_msg: return invalid_oai_err_msg handler.request_json_body = oai_to_vllm(handler.request_json_body, stop_hashes=True, mode=handler.cluster_backend_info['mode']) + handler.parameters, e = handler.get_parameters() + handler.request_json_body = { + 'messages': handler.request_json_body['messages'], + 'model': handler.request_json_body['model'], + **handler.parameters + } + if opts.openai_silent_trim: handler.prompt = transform_messages_to_prompt(trim_messages_to_fit(handler.request.json['messages'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url)) else: handler.prompt = transform_messages_to_prompt(handler.request.json['messages']) - if not handler.prompt: # Prevent issues on the backend. return 'Invalid prompt', 400 @@ -73,90 +72,94 @@ def openai_chat_completions(model_name=None): request_valid, invalid_response = handler.validate_request() if not request_valid: return invalid_response - else: - event = None - if not handler.is_client_ratelimited(): - event = priority_queue.put(handler.backend_url, (handler.request_json_body, handler.client_ip, handler.token, handler.parameters), handler.token_priority, handler.selected_model, do_stream=True) - if not event: - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - None, - None, - handler.parameters, - request.headers, - 429, - request.url, - handler.backend_url, - ) - return handler.handle_ratelimited() - try: - r_headers = dict(request.headers) - r_url = request.url - model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') - oai_string = generate_oai_string(30) + event = None + if not handler.is_client_ratelimited(): + event = priority_queue.put(handler.backend_url, (handler.request_json_body, handler.client_ip, handler.token, handler.parameters), handler.token_priority, handler.selected_model, do_stream=True) + if not event: + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + None, + None, + handler.parameters, + request.headers, + 429, + request.url, + handler.backend_url, + ) + return handler.handle_ratelimited() - def generate(): - stream_name = event.wait() - stream_redis = Redis(db=8) - generated_text = '' - try: - while True: - stream_data = stream_redis.xread({stream_name: '0-0'}, block=30000) - if not stream_data: - print("No message received in 30 seconds, closing stream.") - yield 'data: [DONE]\n\n' - else: - for r_timestamp, item in stream_data[0][1]: - timestamp = int(r_timestamp.decode('utf-8').split('-')[0]) - data = pickle.loads(item[b'data']) - if data['error']: - yield 'data: [DONE]\n\n' - return - elif data['new']: - response = { - "id": f"chatcmpl-{oai_string}", - "object": "chat.completion.chunk", - "created": timestamp, - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": data['new'] - }, - "finish_reason": None - } - ] - } - generated_text = generated_text + data['new'] - yield f'data: {json.dumps(response)}\n\n' - elif data['completed']: - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - 200, - r_url, - handler.backend_url, - ) - return - except (Exception, GeneratorExit): - traceback.print_exc() - yield 'data: [DONE]\n\n' - finally: - stream_redis.delete(stream_name) + try: + r_headers = dict(request.headers) + r_url = request.url + model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') + oai_string = generate_oai_string(30) - return Response(generate(), mimetype='text/event-stream') - except Exception: - traceback.print_exc() - return 'INTERNAL SERVER', 500 + def generate(): + stream_name = event.wait() + stream_redis = Redis(db=8) + generated_text = '' + try: + last_id = '0-0' + while True: + stream_data = stream_redis.xread({stream_name: last_id}, block=30000) + if not stream_data: + print("No message received in 30 seconds, closing stream.") + yield 'data: [DONE]\n\n' + else: + for stream_index, item in stream_data[0][1]: + last_id = stream_index + timestamp = int(stream_index.decode('utf-8').split('-')[0]) + data = pickle.loads(item[b'data']) + if data['error']: + yield 'data: [DONE]\n\n' + return + elif data['new']: + response = { + "id": f"chatcmpl-{oai_string}", + "object": "chat.completion.chunk", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": data['new'] + }, + "finish_reason": None + } + ] + } + generated_text = generated_text + data['new'] + yield f'data: {json.dumps(response)}\n\n' + elif data['completed']: + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + 200, + r_url, + handler.backend_url, + ) + return + except (Exception, GeneratorExit): + traceback.print_exc() + yield 'data: [DONE]\n\n' + finally: + if event: + redis.lpush(f'notifications:{event.event_id}', 'canceled') + stream_redis.delete(stream_name) + + return Response(generate(), mimetype='text/event-stream') + except Exception: + traceback.print_exc() + return 'INTERNAL SERVER', 500 diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index dc7f9e6..374cdc2 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -1,8 +1,10 @@ +import pickle import time import traceback import simplejson as json from flask import Response, jsonify, request +from redis import Redis from llm_server.custom_redis import redis from . import openai_bp, openai_model_bp @@ -12,7 +14,6 @@ from ..queue import priority_queue from ... import opts from ...database.log_to_db import log_to_db from ...llm import get_token_count -from ...llm.generator import generator from ...llm.openai.oai_to_vllm import oai_to_vllm, return_invalid_model_err, validate_oai from ...llm.openai.transform import generate_oai_string, trim_string_to_fit @@ -42,12 +43,14 @@ def openai_completions(model_name=None): handler.request_json_body = oai_to_vllm(handler.request_json_body, stop_hashes=False, mode=handler.cluster_backend_info['mode']) if opts.openai_silent_trim: - handler.request_json_body['prompt'] = trim_string_to_fit(request_json_body['prompt'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) + handler.prompt = trim_string_to_fit(request_json_body['prompt'], handler.cluster_backend_info['model_config']['max_position_embeddings'], handler.backend_url) else: # The handle_request() call below will load the prompt so we don't have # to do anything else here. pass + handler.request_json_body['prompt'] = handler.prompt + if not request_json_body.get('stream'): invalid_oai_err_msg = validate_oai(request_json_body) if invalid_oai_err_msg: @@ -89,120 +92,120 @@ def openai_completions(model_name=None): if not opts.enable_streaming: return 'Streaming disabled', 403 - event_id = None + request_valid, invalid_response = handler.validate_request() + if not request_valid: + return invalid_response + + handler.parameters, _ = handler.get_parameters() + handler.request_json_body = { + 'prompt': handler.request_json_body['prompt'], + 'model': handler.request_json_body['model'], + **handler.parameters + } + + invalid_oai_err_msg = validate_oai(handler.request_json_body) + if invalid_oai_err_msg: + return invalid_oai_err_msg + + if opts.openai_silent_trim: + handler.request_json_body['prompt'] = handler.request_json_body['prompt'][:handler.cluster_backend_info['model_config']['max_position_embeddings']] + if not handler.prompt: + # Prevent issues on the backend. + return 'Invalid prompt', 400 + start_time = time.time() request_valid, invalid_response = handler.validate_request() if not request_valid: return invalid_response - else: - handler.prompt = handler.request_json_body['prompt'] - msg_to_backend = { - **handler.parameters, - 'prompt': handler.prompt, - 'stream': True, - } - event = None - if not handler.is_client_ratelimited(): - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) - if not event: - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - None, - None, - handler.parameters, - request.headers, - 429, - request.url, - handler.backend_url, - ) - return handler.handle_ratelimited() + event = None + if not handler.is_client_ratelimited(): + event = priority_queue.put(handler.backend_url, (handler.request_json_body, handler.client_ip, handler.token, handler.parameters), handler.token_priority, handler.selected_model, do_stream=True) + if not event: + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + None, + None, + handler.parameters, + request.headers, + 429, + request.url, + handler.backend_url, + ) + return handler.handle_ratelimited() - # Wait for permission to begin. - event_id = event.event_id - pubsub = redis.pubsub() - pubsub.subscribe(event_id) - for item in pubsub.listen(): - if item['type'] == 'message': - msg = item['data'].decode('utf-8') - if msg == 'begin': - break - elif msg == 'offline': - return return_invalid_model_err(handler.request_json_body['model']) - time.sleep(0.1) + try: + r_headers = dict(request.headers) + r_url = request.url + model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') + oai_string = generate_oai_string(30) - # Double check the model is still online - if not handler.check_online(): - return return_invalid_model_err(handler.request_json_body['model']) - - try: - response = generator(msg_to_backend, handler.backend_url) - r_headers = dict(request.headers) - r_url = request.url - model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') - oai_string = generate_oai_string(30) - - def generate(): - try: - generated_text = '' - partial_response = b'' - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(handler.prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue - - data = { - "id": f"cmpl-{oai_string}", - "object": "text_completion", - "created": int(time.time()), - "model": model, - "choices": [ - { - "index": 0, - "delta": { - "content": new - }, - "finish_reason": None - } - ] - } - yield f'data: {json.dumps(data)}\n\n' - yield 'data: [DONE]\n\n' - end_time = time.time() - elapsed_time = end_time - start_time - - log_to_db( - handler.client_ip, - handler.token, - handler.prompt, - generated_text, - elapsed_time, - handler.parameters, - r_headers, - 200, - r_url, - handler.backend_url, - ) - finally: - if event_id: - redis.publish(event_id, 'finished') + def generate(): + stream_name = event.wait() + stream_redis = Redis(db=8) + generated_text = '' + try: + last_id = '0-0' + while True: + stream_data = stream_redis.xread({stream_name: last_id}, block=30000) + if not stream_data: + print("No message received in 30 seconds, closing stream.") + yield 'data: [DONE]\n\n' else: - print('event_id was None!') + for stream_index, item in stream_data[0][1]: + last_id = stream_index + timestamp = int(stream_index.decode('utf-8').split('-')[0]) + data = pickle.loads(item[b'data']) + if data['error']: + yield 'data: [DONE]\n\n' + return + elif data['new']: + response = { + "id": f"cmpl-{oai_string}", + "object": "text_completion", + "created": timestamp, + "model": model, + "choices": [ + { + "index": 0, + "delta": { + "content": data['new'] + }, + "finish_reason": None + } + ] + } + generated_text = generated_text + data['new'] + yield f'data: {json.dumps(response)}\n\n' + elif data['completed']: + yield 'data: [DONE]\n\n' + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db( + handler.client_ip, + handler.token, + handler.prompt, + generated_text, + elapsed_time, + handler.parameters, + r_headers, + 200, + r_url, + handler.backend_url, + ) + return + except (Exception, GeneratorExit): + traceback.print_exc() + yield 'data: [DONE]\n\n' + finally: + if event: + redis.lpush(f'notifications:{event.event_id}', 'canceled') + stream_redis.delete(stream_name) - return Response(generate(), mimetype='text/event-stream') - except Exception: - traceback.print_exc() - return 'INTERNAL SERVER', 500 + return Response(generate(), mimetype='text/event-stream') + except Exception: + traceback.print_exc() + return 'INTERNAL SERVER', 500 diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 549cc93..9cbb11c 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -150,10 +150,6 @@ class OpenAIRequestHandler(RequestHandler): "total_tokens": prompt_tokens + response_tokens } }), 200) - - stats = redis.get('proxy_stats', dtype=dict) - if stats: - response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] return response def validate_request(self, prompt: str = None, do_log: bool = False) -> Tuple[bool, Tuple[Response | None, int]]: diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 4011030..0fe94ec 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -37,6 +37,9 @@ class RequestHandler: self.parameters = None self.used = False + # This is null by default since most handlers need to transform the prompt in a specific way. + self.prompt = None + self.selected_model = selected_model self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 332fe4c..7c02cc9 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -1,17 +1,18 @@ import json +import pickle import time import traceback from flask import request +from redis import Redis from . import bp from ..helpers.http import require_api_key, validate_json from ..ooba_request_handler import OobaRequestHandler from ..queue import priority_queue -from ... import messages, opts +from ... import opts from ...custom_redis import redis from ...database.log_to_db import log_to_db -from ...llm.generator import generator from ...sock import sock @@ -35,6 +36,7 @@ def stream_with_model(ws, model_name=None): def do_stream(ws, model_name): + event_id = None try: def send_err_and_quit(quitting_err_msg): ws.send(json.dumps({ @@ -46,6 +48,7 @@ def do_stream(ws, model_name): 'event': 'stream_end', 'message_num': 1 })) + ws.close() log_to_db(ip=handler.client_ip, token=handler.token, prompt=input_prompt, @@ -55,7 +58,7 @@ def do_stream(ws, model_name): headers=r_headers, backend_response_code=response_status_code, request_url=r_url, - backend_url=handler.cluster_backend_info, + backend_url=handler.backend_url, response_tokens=None, is_error=True ) @@ -74,6 +77,7 @@ def do_stream(ws, model_name): if not request_valid_json or not request_json_body.get('prompt'): return 'Invalid JSON', 400 else: + # We have to do auth ourselves since the details are sent in the message. auth_failure = require_api_key(request_json_body) if auth_failure: return auth_failure @@ -89,14 +93,10 @@ def do_stream(ws, model_name): })) return - assert not handler.offline - if handler.cluster_backend_info['mode'] != 'vllm': # TODO: implement other backends raise NotImplementedError - event_id = None - generated_text = '' input_prompt = request_json_body['prompt'] response_status_code = 0 start_time = time.time() @@ -113,119 +113,55 @@ def do_stream(ws, model_name): send_err_and_quit(err_msg) return - llm_request = { - **handler.parameters, - 'prompt': input_prompt, - 'stream': True, + handler.parameters, _ = handler.get_parameters() + handler.prompt = input_prompt + handler.request_json_body = { + 'prompt': handler.prompt, + **handler.parameters } event = None if not handler.is_client_ratelimited(): - # Add a dummy event to the queue and wait for it to reach a worker - event = priority_queue.put(handler.backend_url, (None, handler.client_ip, handler.token, None), handler.token_priority, handler.selected_model) + event = priority_queue.put(handler.backend_url, (handler.request_json_body, handler.client_ip, handler.token, handler.parameters), handler.token_priority, handler.selected_model, do_stream=True) if not event: - log_to_db( - handler.client_ip, - handler.token, - handler.request_json_body.get('prompt'), - None, - None, - handler.parameters, - request.headers, - response_status_code, - request.url, - handler.backend_url, - ) - return handler.handle_ratelimited() - - # Wait for permission to begin. + r = handler.handle_ratelimited() + send_err_and_quit(r[0].data) + return event_id = event.event_id - pubsub = redis.pubsub() - pubsub.subscribe(event_id) - for item in pubsub.listen(): - if item['type'] == 'message': - msg = item['data'].decode('utf-8') - if msg == 'begin': - break - elif msg == 'offline': - return messages.BACKEND_OFFLINE, 404 # TODO: format this error - time.sleep(0.1) - # Double check the model is still online - if not handler.check_online(): - return messages.BACKEND_OFFLINE, 404 # TODO: format this error + stream_name = event.wait() + stream_redis = Redis(db=8) + generated_text = '' try: - response = generator(llm_request, handler.backend_url) - if not response: - error_msg = 'Failed to reach backend while streaming.' - print('Streaming failed:', error_msg) - msg = handler.handle_error(error_msg)[0].json['results'][0]['text'] - ws.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'text': msg - })) - else: - # Be extra careful when getting attributes from the response object - try: - response_status_code = response.status_code - except: - response_status_code = 0 - - partial_response = b'' - - for chunk in response.iter_content(chunk_size=1): - partial_response += chunk - if partial_response.endswith(b'\x00'): - json_strs = partial_response.split(b'\x00') - for json_str in json_strs: - if json_str: - try: - json_obj = json.loads(json_str.decode()) - new = json_obj['text'][0].split(input_prompt + generated_text)[1] - generated_text = generated_text + new - except IndexError: - # ???? - continue - try: - ws.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'text': new - })) - except: - # The has client closed the stream. - if response: - # Cancel the backend? - response.close() - # used to log here - return - - message_num += 1 - partial_response = b'' # Reset the partial response - - # If there is no more data, break the loop - if not chunk: - break - if response: - response.close() - # used to log here + last_id = '0-0' # The ID of the last entry we read. + while True: + stream_data = stream_redis.xread({stream_name: last_id}, block=30000) + if not stream_data: + print("No message received in 30 seconds, closing stream.") + return + else: + for stream_index, item in stream_data[0][1]: + last_id = stream_index + data = pickle.loads(item[b'data']) + if data['error']: + print(data['error']) + send_err_and_quit('Encountered exception while streaming.') + return + elif data['new']: + ws.send(json.dumps({ + 'event': 'text_stream', + 'message_num': message_num, + 'text': data['new'] + })) + message_num += 1 + generated_text = generated_text + data['new'] + elif data['completed']: + return except: + send_err_and_quit('Encountered exception while streaming.') traceback.print_exc() - generated_text = generated_text + '\n\n' + handler.handle_error('Encountered error while streaming.', 'exception')[0].json['results'][0]['text'] - ws.send(json.dumps({ - 'event': 'text_stream', - 'message_num': message_num, - 'text': generated_text - })) - # used to log here finally: - if event_id: - redis.publish(event_id, 'finished') - else: - print('event_id was None!') - try: ws.send(json.dumps({ 'event': 'stream_end', @@ -234,6 +170,7 @@ def do_stream(ws, model_name): except: # The client closed the stream. pass + stream_redis.delete(stream_name) end_time = time.time() elapsed_time = end_time - start_time log_to_db(ip=handler.client_ip, @@ -248,6 +185,8 @@ def do_stream(ws, model_name): backend_url=handler.backend_url ) finally: + if event_id: + redis.lpush(f'notifications:{event_id}', 'canceled') try: # Must close the connection or greenlets will complain. ws.close() diff --git a/llm_server/sock.py b/llm_server/sock.py index 8ac2fc1..2f1a17d 100644 --- a/llm_server/sock.py +++ b/llm_server/sock.py @@ -3,6 +3,6 @@ from flask_sock import Sock sock = Sock() -def init_socketio(app): +def init_wssocket(app): global sock sock.init_app(app) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index a8a73e0..b190b4d 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -7,7 +7,7 @@ from uuid import uuid4 from redis import Redis from llm_server.cluster.cluster_config import cluster_config -from llm_server.custom_redis import RedisCustom +from llm_server.custom_redis import RedisCustom, redis from llm_server.llm.generator import generator from llm_server.routes.queue import DataEvent, RedisPriorityQueue, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count @@ -20,15 +20,25 @@ def get_stream_name(name: str): return f'{STREAM_NAME_PREFIX}:{name}' -def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str): +def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str, event_id: str): prompt = msg_to_backend['prompt'] stream_name = get_stream_name(stream_name) + redis.delete(f'notifications:{event_id}') stream_redis.delete(get_stream_name(stream_name)) # be extra sure try: response = generator(msg_to_backend, backend_url) generated_text = '' partial_response = b'' for chunk in response.iter_content(chunk_size=1): + # If there is no more data, break the loop + if not chunk: + break + message = redis.lpop(f'notifications:{event_id}') + if message and message.decode('utf-8') == 'canceled': + print('Client canceled generation') + response.close() + return + partial_response += chunk if partial_response.endswith(b'\x00'): json_strs = partial_response.split(b'\x00') @@ -74,14 +84,16 @@ def worker(backend_url): try: if do_stream: + # Return the name of the stream that the slave should connect to. event = DataEvent(event_id) event.set(get_stream_name(worker_id)) + msg_to_backend = { **parameters, 'prompt': request_json_body['prompt'], 'stream': True, } - inference_do_stream(worker_id, msg_to_backend, backend_url) + inference_do_stream(worker_id, msg_to_backend, backend_url, event_id) else: # Normal inference (not streaming). success, response, error_msg = generator(request_json_body, backend_url) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index c28e383..4025df3 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -29,4 +29,4 @@ def console_printer(): # TODO: Active Workers and Processing should read the same. If not, that's an issue logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - time.sleep(10) + time.sleep(2) diff --git a/other/ooba-test-streaming.py b/other/ooba-test-streaming.py index 883c2f5..7f5185d 100644 --- a/other/ooba-test-streaming.py +++ b/other/ooba-test-streaming.py @@ -11,6 +11,7 @@ except ImportError: HOST = 'localhost:5000' URI = f'ws://{HOST}/api/v1/stream' + # For reverse-proxied streaming, the remote will likely host with ssl - wss:// # URI = 'wss://your-uri-here.trycloudflare.com/api/v1/stream' @@ -82,5 +83,6 @@ async def print_response_stream(prompt): if __name__ == '__main__': - prompt = "In order to make homemade bread, follow these steps:\n1)" + # prompt = "In order to make homemade bread, follow these steps:\n1)" + prompt = "Write a 300 word description of how an apple tree grows.\n\n" asyncio.run(print_response_stream(prompt)) diff --git a/server.py b/server.py index 490eebe..e33d55a 100644 --- a/server.py +++ b/server.py @@ -28,7 +28,7 @@ from llm_server.routes.openai import openai_bp, openai_model_bp from llm_server.routes.server_error import handle_server_error from llm_server.routes.v1 import bp from llm_server.routes.v1.generate_stats import generate_stats -from llm_server.sock import init_socketio +from llm_server.sock import init_wssocket # TODO: queue item timeout # TODO: return an `error: True`, error code, and error message rather than just a formatted message @@ -68,10 +68,15 @@ except ModuleNotFoundError as e: sys.exit(1) app = Flask(__name__) + +# Fixes ConcurrentObjectUseError +# https://github.com/miguelgrinberg/simple-websocket/issues/24 +app.config['SOCK_SERVER_OPTIONS'] = {'ping_interval': 25} + app.register_blueprint(bp, url_prefix='/api/') app.register_blueprint(openai_bp, url_prefix='/api/openai/v1/') app.register_blueprint(openai_model_bp, url_prefix='/api/openai/') -init_socketio(app) +init_wssocket(app) flask_cache.init_app(app) flask_cache.clear() -- 2.34.1 From f421436048bd2b34a1f4c214a716509fc172bc60 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 17:43:37 -0600 Subject: [PATCH 126/163] add nginx config --- other/nginx-site.conf | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 other/nginx-site.conf diff --git a/other/nginx-site.conf b/other/nginx-site.conf new file mode 100644 index 0000000..1c81d3d --- /dev/null +++ b/other/nginx-site.conf @@ -0,0 +1,38 @@ +server +{ + listen 443 ssl http2 default_server; + server_name _; + + proxy_set_header Host $host; + proxy_set_header Connection $http_connection; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Scheme $scheme; + + location ~* ^/api/(.*?|v1|openai)/(v1|(generate|stream)|(chat/completions|completions))$ + { + # Route to inference endpoints + proxy_pass http://127.0.0.1:5000; + + # Required for streaming (both websockets and SSE). + proxy_buffering off; + proxy_cache off; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Set long timeouts for inference operations. + # Cloudflare has a timeout of 100 seconds. + proxy_read_timeout 120; + proxy_connect_timeout 120; + proxy_send_timeout 120; + } + + location / + { + proxy_pass http://127.0.0.1:5000; + } + + ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt; + ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key; + include /etc/nginx/snippets/ssl-params.conf; +} -- 2.34.1 From 19a193b792d0f473259e5df342b8dd0b5d21070e Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 17:59:21 -0600 Subject: [PATCH 127/163] increase tokenization chunk size --- llm_server/llm/vllm/tokenize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 69a2b14..d32eb68 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -22,8 +22,8 @@ def tokenize(prompt: str, backend_url: str) -> int: tokenizer = tiktoken.get_encoding("cl100k_base") - # Split the prompt into 300 character chunks - chunk_size = 300 + # Split the prompt into 1000 character chunks + chunk_size = 1000 chunks = [prompt[i:i + chunk_size] for i in range(0, len(prompt), chunk_size)] # Define a function to send a chunk to the server -- 2.34.1 From 20047fa0e434fbea2db958dc8d2bdf04eefbfbf8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 18:01:17 -0600 Subject: [PATCH 128/163] 2000 chunk size --- llm_server/llm/vllm/tokenize.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index d32eb68..0a2f5c6 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -22,8 +22,8 @@ def tokenize(prompt: str, backend_url: str) -> int: tokenizer = tiktoken.get_encoding("cl100k_base") - # Split the prompt into 1000 character chunks - chunk_size = 1000 + # Split the prompt into 2000 character chunks + chunk_size = 2000 chunks = [prompt[i:i + chunk_size] for i in range(0, len(prompt), chunk_size)] # Define a function to send a chunk to the server -- 2.34.1 From 1e68e10b6220f2bd24d857445bbf2f79f1402810 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 18:04:49 -0600 Subject: [PATCH 129/163] fix GeneratorExit --- llm_server/routes/openai/chat_completions.py | 4 +++- llm_server/routes/openai/completions.py | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 44d5172..e863636 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -151,7 +151,9 @@ def openai_chat_completions(model_name=None): handler.backend_url, ) return - except (Exception, GeneratorExit): + except GeneratorExit: + return + except Exception: traceback.print_exc() yield 'data: [DONE]\n\n' finally: diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 374cdc2..4df336f 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -197,7 +197,10 @@ def openai_completions(model_name=None): handler.backend_url, ) return - except (Exception, GeneratorExit): + except GeneratorExit: + # This should be triggered if a client disconnects early. + return + except Exception: traceback.print_exc() yield 'data: [DONE]\n\n' finally: -- 2.34.1 From 21755450a341f5d62805b6dc8328ba6966edf80c Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 18:10:21 -0600 Subject: [PATCH 130/163] test --- llm_server/llm/vllm/tokenize.py | 1 + 1 file changed, 1 insertion(+) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 0a2f5c6..6fcca52 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -29,6 +29,7 @@ def tokenize(prompt: str, backend_url: str) -> int: # Define a function to send a chunk to the server def send_chunk(chunk): try: + XXX r = requests.post(f'{backend_url}/tokenize', json={'input': chunk}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) j = r.json() return j['length'] -- 2.34.1 From 81baf9616f2ddaa05af926534a96d5c7b562fc85 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 18:16:19 -0600 Subject: [PATCH 131/163] revert --- llm_server/llm/vllm/tokenize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 6fcca52..0a2f5c6 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -29,7 +29,6 @@ def tokenize(prompt: str, backend_url: str) -> int: # Define a function to send a chunk to the server def send_chunk(chunk): try: - XXX r = requests.post(f'{backend_url}/tokenize', json={'input': chunk}, verify=opts.verify_ssl, timeout=opts.backend_generate_request_timeout) j = r.json() return j['length'] -- 2.34.1 From 806e522d166baff7ade0973d55cbbe3e46f7f5a2 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 18:35:10 -0600 Subject: [PATCH 132/163] don't pickle streaming --- llm_server/opts.py | 3 ++- llm_server/routes/openai/chat_completions.py | 8 ++++---- llm_server/routes/openai/completions.py | 8 ++++---- llm_server/routes/v1/generate_stream.py | 8 ++++---- llm_server/workers/inferencer.py | 8 ++++---- requirements.txt | 3 ++- 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/llm_server/opts.py b/llm_server/opts.py index 69b25eb..ada54a8 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -37,4 +37,5 @@ show_backends = True background_homepage_cacher = True openai_moderation_timeout = 5 prioritize_by_size = False -cluster_workers = 0 \ No newline at end of file +cluster_workers = 0 +redis_stream_timeout = 25000 diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index e863636..5e5921a 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -1,5 +1,5 @@ import json -import pickle +import ujson import time import traceback @@ -104,15 +104,15 @@ def openai_chat_completions(model_name=None): try: last_id = '0-0' while True: - stream_data = stream_redis.xread({stream_name: last_id}, block=30000) + stream_data = stream_redis.xread({stream_name: last_id}, block=opts.redis_stream_timeout) if not stream_data: - print("No message received in 30 seconds, closing stream.") + print(f"No message received in {opts.redis_stream_timeout / 1000} seconds, closing stream.") yield 'data: [DONE]\n\n' else: for stream_index, item in stream_data[0][1]: last_id = stream_index timestamp = int(stream_index.decode('utf-8').split('-')[0]) - data = pickle.loads(item[b'data']) + data = ujson.loads(item[b'data']) if data['error']: yield 'data: [DONE]\n\n' return diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 4df336f..b8efb07 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -1,8 +1,8 @@ -import pickle import time import traceback import simplejson as json +import ujson from flask import Response, jsonify, request from redis import Redis @@ -150,15 +150,15 @@ def openai_completions(model_name=None): try: last_id = '0-0' while True: - stream_data = stream_redis.xread({stream_name: last_id}, block=30000) + stream_data = stream_redis.xread({stream_name: last_id}, block=opts.redis_stream_timeout) if not stream_data: - print("No message received in 30 seconds, closing stream.") + print(f"No message received in {opts.redis_stream_timeout / 1000} seconds, closing stream.") yield 'data: [DONE]\n\n' else: for stream_index, item in stream_data[0][1]: last_id = stream_index timestamp = int(stream_index.decode('utf-8').split('-')[0]) - data = pickle.loads(item[b'data']) + data = ujson.loads(item[b'data']) if data['error']: yield 'data: [DONE]\n\n' return diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 7c02cc9..29eb281 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -1,8 +1,8 @@ import json -import pickle import time import traceback +import ujson from flask import request from redis import Redis @@ -136,14 +136,14 @@ def do_stream(ws, model_name): try: last_id = '0-0' # The ID of the last entry we read. while True: - stream_data = stream_redis.xread({stream_name: last_id}, block=30000) + stream_data = stream_redis.xread({stream_name: last_id}, block=opts.redis_stream_timeout) if not stream_data: - print("No message received in 30 seconds, closing stream.") + print(f"No message received in {opts.redis_stream_timeout / 1000} seconds, closing stream.") return else: for stream_index, item in stream_data[0][1]: last_id = stream_index - data = pickle.loads(item[b'data']) + data = ujson.loads(item[b'data']) if data['error']: print(data['error']) send_err_and_quit('Encountered exception while streaming.') diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index b190b4d..b6e94dc 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,9 +1,9 @@ import json -import pickle import threading import traceback from uuid import uuid4 +import ujson from redis import Redis from llm_server.cluster.cluster_config import cluster_config @@ -51,13 +51,13 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str except IndexError: # ???? continue - stream_redis.xadd(stream_name, {'data': pickle.dumps({'new': new, 'completed': False, 'error': None})}) + stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': new, 'completed': False, 'error': None})}) except Exception as e: - stream_redis.xadd(stream_name, {'data': pickle.dumps({'new': None, 'completed': True, 'error': f'{e.__class__.__name__}: {e}'})}) + stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': None, 'completed': True, 'error': f'{e.__class__.__name__}: {e}'})}) traceback.print_exc() finally: # Publish final message to Redis stream - stream_redis.xadd(stream_name, {'data': pickle.dumps({'new': None, 'completed': True, 'error': None})}) + stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': None, 'completed': True, 'error': None})}) def worker(backend_url): diff --git a/requirements.txt b/requirements.txt index 28e818f..802d6f2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,4 +13,5 @@ openai~=0.28.0 urllib3~=2.0.4 flask-sock==0.6.0 gunicorn==21.2.0 -redis==5.0.1 \ No newline at end of file +redis==5.0.1 +ujson==5.8.0 \ No newline at end of file -- 2.34.1 From 70cf6843e51c9714371d860984fa423ff0b7cd1c Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:12:41 -0600 Subject: [PATCH 133/163] update requiorements --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 802d6f2..c1103d7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,6 @@ Flask-Caching==2.0.2 requests~=2.31.0 tiktoken~=0.5.0 gevent~=23.9.0.post1 -torch~=2.0.1 PyMySQL~=1.1.0 simplejson~=3.19.1 websockets~=11.0.3 @@ -14,4 +13,5 @@ urllib3~=2.0.4 flask-sock==0.6.0 gunicorn==21.2.0 redis==5.0.1 -ujson==5.8.0 \ No newline at end of file +ujson==5.8.0 +vllm -- 2.34.1 From c3c053e071120caf3e95ea3e10dc886c2cd28980 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:29:17 -0600 Subject: [PATCH 134/163] test --- llm_server/routes/openai/chat_completions.py | 4 ++-- llm_server/routes/v1/generate_stream.py | 4 ++-- llm_server/workers/inferencer.py | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 5e5921a..fcad3a0 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -157,8 +157,8 @@ def openai_chat_completions(model_name=None): traceback.print_exc() yield 'data: [DONE]\n\n' finally: - if event: - redis.lpush(f'notifications:{event.event_id}', 'canceled') + # if event: + # redis.lpush(f'notifications:{event.event_id}', 'canceled') stream_redis.delete(stream_name) return Response(generate(), mimetype='text/event-stream') diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 29eb281..e329cd8 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -185,8 +185,8 @@ def do_stream(ws, model_name): backend_url=handler.backend_url ) finally: - if event_id: - redis.lpush(f'notifications:{event_id}', 'canceled') + # if event_id: + # redis.lpush(f'notifications:{event_id}', 'canceled') try: # Must close the connection or greenlets will complain. ws.close() diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index b6e94dc..3d05dc2 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -33,11 +33,11 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str # If there is no more data, break the loop if not chunk: break - message = redis.lpop(f'notifications:{event_id}') - if message and message.decode('utf-8') == 'canceled': - print('Client canceled generation') - response.close() - return + # message = redis.lpop(f'notifications:{event_id}') + # if message and message.decode('utf-8') == 'canceled': + # print('Client canceled generation') + # response.close() + # return partial_response += chunk if partial_response.endswith(b'\x00'): -- 2.34.1 From 9e3cbc9d2ecf7817c3771bd6026998f70e048d01 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:36:25 -0600 Subject: [PATCH 135/163] fix streaming slowdown? --- llm_server/routes/openai/chat_completions.py | 4 ++-- llm_server/routes/openai/completions.py | 2 +- llm_server/routes/v1/generate_stream.py | 4 ++-- llm_server/workers/inferencer.py | 13 +++++++------ 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index fcad3a0..fbac971 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -157,8 +157,8 @@ def openai_chat_completions(model_name=None): traceback.print_exc() yield 'data: [DONE]\n\n' finally: - # if event: - # redis.lpush(f'notifications:{event.event_id}', 'canceled') + if event: + redis.publish(f'notifications:{event.event_id}', 'canceled') stream_redis.delete(stream_name) return Response(generate(), mimetype='text/event-stream') diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index b8efb07..e7b85ea 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -205,7 +205,7 @@ def openai_completions(model_name=None): yield 'data: [DONE]\n\n' finally: if event: - redis.lpush(f'notifications:{event.event_id}', 'canceled') + redis.publish(f'notifications:{event.event_id}', 'canceled') stream_redis.delete(stream_name) return Response(generate(), mimetype='text/event-stream') diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index e329cd8..72f4bad 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -185,8 +185,8 @@ def do_stream(ws, model_name): backend_url=handler.backend_url ) finally: - # if event_id: - # redis.lpush(f'notifications:{event_id}', 'canceled') + if event_id: + redis.publish(f'notifications:{event_id}', 'canceled') try: # Must close the connection or greenlets will complain. ws.close() diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 3d05dc2..765ec8b 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -23,7 +23,8 @@ def get_stream_name(name: str): def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str, event_id: str): prompt = msg_to_backend['prompt'] stream_name = get_stream_name(stream_name) - redis.delete(f'notifications:{event_id}') + pubsub = redis.pubsub() + pubsub.subscribe(f'notifications:{event_id}') stream_redis.delete(get_stream_name(stream_name)) # be extra sure try: response = generator(msg_to_backend, backend_url) @@ -33,11 +34,11 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str # If there is no more data, break the loop if not chunk: break - # message = redis.lpop(f'notifications:{event_id}') - # if message and message.decode('utf-8') == 'canceled': - # print('Client canceled generation') - # response.close() - # return + message = pubsub.get_message() + if message and message['data'] == b'canceled': + print('Client canceled generation') + response.close() + return partial_response += chunk if partial_response.endswith(b'\x00'): -- 2.34.1 From 6f657917955d0afdd8f0712a27142afadeae0197 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:40:07 -0600 Subject: [PATCH 136/163] adjust --- llm_server/workers/inferencer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 765ec8b..4bc1472 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -34,7 +34,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str # If there is no more data, break the loop if not chunk: break - message = pubsub.get_message() + message = pubsub.get_message(timeout=0.001) if message and message['data'] == b'canceled': print('Client canceled generation') response.close() -- 2.34.1 From 2ed0e01db6bbcfd7784542f029ff8ff8f90a6ac8 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:44:11 -0600 Subject: [PATCH 137/163] background thread --- llm_server/workers/inferencer.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 4bc1472..b4bff23 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -1,5 +1,6 @@ import json import threading +import time import traceback from uuid import uuid4 @@ -16,6 +17,23 @@ stream_redis = Redis(db=8) STREAM_NAME_PREFIX = 'stream' +def check_cancellation(event, event_id): + """ + This thread checks the pub/sub channel in the background so the main process + isn't bogged down with Redis calls. + :param event: + :param event_id: + :return: + """ + pubsub = redis.pubsub() + pubsub.subscribe(f'notifications:{event_id}') + while not event.is_set(): + message = pubsub.get_message() + if message and message['data'] == b'canceled': + event.set() + time.sleep(0.5) # check every half second + + def get_stream_name(name: str): return f'{STREAM_NAME_PREFIX}:{name}' @@ -23,9 +41,10 @@ def get_stream_name(name: str): def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str, event_id: str): prompt = msg_to_backend['prompt'] stream_name = get_stream_name(stream_name) - pubsub = redis.pubsub() - pubsub.subscribe(f'notifications:{event_id}') stream_redis.delete(get_stream_name(stream_name)) # be extra sure + event = threading.Event() + t = threading.Thread(target=check_cancellation, args=(event, event_id)) + t.start() try: response = generator(msg_to_backend, backend_url) generated_text = '' @@ -34,8 +53,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str # If there is no more data, break the loop if not chunk: break - message = pubsub.get_message(timeout=0.001) - if message and message['data'] == b'canceled': + if event.is_set(): print('Client canceled generation') response.close() return @@ -59,6 +77,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str finally: # Publish final message to Redis stream stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': None, 'completed': True, 'error': None})}) + event.set() # stop the cancellation checking thread def worker(backend_url): -- 2.34.1 From 7998cfca87f2336ed68aa1ccfa7fd011345b166f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 16 Oct 2023 23:47:34 -0600 Subject: [PATCH 138/163] cleanup --- llm_server/workers/inferencer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index b4bff23..d1a3ceb 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -20,7 +20,7 @@ STREAM_NAME_PREFIX = 'stream' def check_cancellation(event, event_id): """ This thread checks the pub/sub channel in the background so the main process - isn't bogged down with Redis calls. + isn't bogged down with Redis calls. Otherwise, the main process slows down to 1 token/sec. :param event: :param event_id: :return: @@ -43,8 +43,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str stream_name = get_stream_name(stream_name) stream_redis.delete(get_stream_name(stream_name)) # be extra sure event = threading.Event() - t = threading.Thread(target=check_cancellation, args=(event, event_id)) - t.start() + threading.Thread(target=check_cancellation, args=(event, event_id)).start() try: response = generator(msg_to_backend, backend_url) generated_text = '' -- 2.34.1 From 2fed87d340e5cffc0313ed15a3975680ce1f1a82 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 17 Oct 2023 11:46:39 -0600 Subject: [PATCH 139/163] remove timed-out items from queue --- llm_server/custom_redis.py | 17 ++++++++++ llm_server/llm/vllm/vllm_backend.py | 1 + llm_server/routes/openai/chat_completions.py | 13 ++++++-- llm_server/routes/openai/completions.py | 9 +++-- llm_server/routes/queue.py | 32 +++++++++++------- llm_server/routes/request_handler.py | 2 ++ llm_server/routes/v1/generate_stream.py | 35 +++++++++++--------- llm_server/workers/inferencer.py | 2 +- llm_server/workers/mainer.py | 1 - llm_server/workers/printer.py | 5 ++- 10 files changed, 80 insertions(+), 37 deletions(-) diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index 60e4dbd..886230a 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -232,5 +232,22 @@ class RedisCustom(Redis): def lpop(self, name: str, count: Optional[int] = None): return self.redis.lpop(self._key(name), count) + def zrange( + self, + name: KeyT, + start: int, + end: int, + desc: bool = False, + withscores: bool = False, + score_cast_func: Union[type, Callable] = float, + byscore: bool = False, + bylex: bool = False, + offset: int = None, + num: int = None, + ): + return self.redis.zrange(self._key(name), start, end, desc, withscores, score_cast_func, byscore, bylex, offset, num) + + def zrem(self, name: KeyT, *values: FieldT): + return self.redis.zrem(self._key(name), *values) redis = RedisCustom('local_llm') diff --git a/llm_server/llm/vllm/vllm_backend.py b/llm_server/llm/vllm/vllm_backend.py index 9665547..5c12b45 100644 --- a/llm_server/llm/vllm/vllm_backend.py +++ b/llm_server/llm/vllm/vllm_backend.py @@ -30,6 +30,7 @@ class VLLMBackend(LLMBackend): if top_k <= 0: top_k = -1 + # TODO: support more params sampling_params = SamplingParams( temperature=parameters.get('temperature', self._default_params['temperature']), top_p=parameters.get('top_p', self._default_params['top_p']), diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index fbac971..475ff00 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -1,8 +1,8 @@ import json -import ujson import time import traceback +import ujson from flask import Response, jsonify, request from redis import Redis @@ -97,8 +97,14 @@ def openai_chat_completions(model_name=None): model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') oai_string = generate_oai_string(30) + # Need to do this before we enter generate() since we want to be able to + # return a 408 if necessary. + _, stream_name, error_msg = event.wait() + if error_msg == 'closed': + stream_name = None # set to null so that the Finally ignores it. + return 'Request Timeout', 408 + def generate(): - stream_name = event.wait() stream_redis = Redis(db=8) generated_text = '' try: @@ -159,7 +165,8 @@ def openai_chat_completions(model_name=None): finally: if event: redis.publish(f'notifications:{event.event_id}', 'canceled') - stream_redis.delete(stream_name) + if stream_name: + stream_redis.delete(stream_name) return Response(generate(), mimetype='text/event-stream') except Exception: diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index e7b85ea..5dfacf3 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -143,8 +143,12 @@ def openai_completions(model_name=None): model = redis.get('running_model', 'ERROR', dtype=str) if opts.openai_expose_our_model else request_json_body.get('model') oai_string = generate_oai_string(30) + _, stream_name, error_msg = event.wait() + if error_msg == 'closed': + stream_name = None + return 'Request Timeout', 408 + def generate(): - stream_name = event.wait() stream_redis = Redis(db=8) generated_text = '' try: @@ -206,7 +210,8 @@ def openai_completions(model_name=None): finally: if event: redis.publish(f'notifications:{event.event_id}', 'canceled') - stream_redis.delete(stream_name) + if stream_name: + stream_redis.delete(stream_name) return Response(generate(), mimetype='text/event-stream') except Exception: diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index cb4aaf5..834c844 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -57,12 +57,12 @@ class RedisPriorityQueue: return item time.sleep(0.1) # wait for something to be added to the queue - def print_all_items(self): - items = self.redis.zrange('queue', 0, -1) - to_print = [] - for item in items: - to_print.append(item.decode('utf-8')) - print(f'ITEMS {self.name} -->', to_print) + # def print_all_items(self): + # items = self.redis.zrange('queue', 0, -1) + # to_print = [] + # for item in items: + # to_print.append(item.decode('utf-8')) + # print(f'ITEMS {self.name} -->', to_print) def increment_ip_count(self, client_ip: str, redis_key): self.redis.hincrby(redis_key, client_ip, 1) @@ -84,15 +84,23 @@ class RedisPriorityQueue: def flush(self): self.redis.flush() + def items(self): + return self.redis.zrange('queue', 0, -1) + def cleanup(self): now = time.time() - items = self.redis.zrange('queue', 0, -1) - for item in items: + for item in self.items(): item_data = json.loads(item) - timestamp = item_data[-1] - if now - timestamp > opts.backend_generate_request_timeout * 3: # TODO: config option - self.redis.zrem('queue', item) - print('removed item from queue:', item) + timestamp = item_data[-2] + if now - timestamp > opts.backend_generate_request_timeout: + self.redis.zrem('queue', 0, item) + data = json.loads(item.decode('utf-8')) + event_id = data[1] + client_ip = data[0][1] + self.decrement_ip_count(client_ip, 'queued_ip_count') + event = DataEvent(event_id) + event.set((False, None, 'closed')) + print('Removed timed-out item from queue:', event_id) class DataEvent: diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 0fe94ec..df07c29 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -148,6 +148,8 @@ class RequestHandler: # TODO: add wait timeout success, response, error_msg = event.wait() + if error_msg == 'closed': + return (False, None, None, 0), (self.handle_error('Request Timeout')[0], 408) end_time = time.time() elapsed_time = end_time - self.start_time diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index 72f4bad..cdf939d 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -129,7 +129,11 @@ def do_stream(ws, model_name): return event_id = event.event_id - stream_name = event.wait() + _, stream_name, error_msg = event.wait() + if error_msg == 'closed': + ws.close(reason=1014, message='Request Timeout') + return + stream_redis = Redis(db=8) generated_text = '' @@ -170,20 +174,21 @@ def do_stream(ws, model_name): except: # The client closed the stream. pass - stream_redis.delete(stream_name) - end_time = time.time() - elapsed_time = end_time - start_time - log_to_db(ip=handler.client_ip, - token=handler.token, - prompt=input_prompt, - response=generated_text, - gen_time=elapsed_time, - parameters=handler.parameters, - headers=r_headers, - backend_response_code=response_status_code, - request_url=r_url, - backend_url=handler.backend_url - ) + if stream_name: + stream_redis.delete(stream_name) + end_time = time.time() + elapsed_time = end_time - start_time + log_to_db(ip=handler.client_ip, + token=handler.token, + prompt=input_prompt, + response=generated_text, + gen_time=elapsed_time, + parameters=handler.parameters, + headers=r_headers, + backend_response_code=response_status_code, + request_url=r_url, + backend_url=handler.backend_url + ) finally: if event_id: redis.publish(f'notifications:{event_id}', 'canceled') diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index d1a3ceb..0738c1b 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -105,7 +105,7 @@ def worker(backend_url): if do_stream: # Return the name of the stream that the slave should connect to. event = DataEvent(event_id) - event.set(get_stream_name(worker_id)) + event.set((True, get_stream_name(worker_id), None)) msg_to_backend = { **parameters, diff --git a/llm_server/workers/mainer.py b/llm_server/workers/mainer.py index e06e803..d342f4b 100644 --- a/llm_server/workers/mainer.py +++ b/llm_server/workers/mainer.py @@ -36,7 +36,6 @@ def main_background_thread(): except Exception as e: print(f'Failed fetch the homepage - {e.__class__.__name__}: {e}') - # TODO: test backends = priority_queue.get_backends() for backend_url in backends: queue = RedisPriorityQueue(backend_url) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index 4025df3..fe0d129 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -26,7 +26,6 @@ def console_printer(): backends = [k for k, v in cluster_config.all().items() if v['online']] activity = priority_queue.activity() - # TODO: Active Workers and Processing should read the same. If not, that's an issue - + # Active Workers and Processing should read the same. If not, that's an issue. logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - time.sleep(2) + time.sleep(10) -- 2.34.1 From 4c2c164ce1990b30ad115a1e9afa7fb9695029bf Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 17 Oct 2023 12:29:12 -0600 Subject: [PATCH 140/163] test --- llm_server/routes/openai/chat_completions.py | 1 + llm_server/routes/openai/completions.py | 1 + 2 files changed, 2 insertions(+) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 475ff00..520652e 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -120,6 +120,7 @@ def openai_chat_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: + print(data['error']) yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 5dfacf3..c78ac64 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -164,6 +164,7 @@ def openai_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: + print(data['error']) yield 'data: [DONE]\n\n' return elif data['new']: -- 2.34.1 From 90adffaec8951e87d4a35b6ee7c8cb4b9d2cba30 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 17 Oct 2023 12:32:41 -0600 Subject: [PATCH 141/163] test --- llm_server/routes/openai/chat_completions.py | 2 +- llm_server/routes/openai/completions.py | 2 +- llm_server/workers/inferencer.py | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 520652e..0ef28ff 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -120,7 +120,7 @@ def openai_chat_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: - print(data['error']) + print('OAI streaming error:', data['error']) yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index c78ac64..d77ff0d 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -164,7 +164,7 @@ def openai_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: - print(data['error']) + print('OAI streaming error:', data['error']) yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 0738c1b..a1dd749 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -66,6 +66,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str json_obj = json.loads(json_str.decode()) new = json_obj['text'][0].split(prompt + generated_text)[1] generated_text = generated_text + new + print(new) except IndexError: # ???? continue -- 2.34.1 From be035691656b6f9c08a2756e5b67ba93eef0a45e Mon Sep 17 00:00:00 2001 From: Cyberes Date: Tue, 17 Oct 2023 12:42:48 -0600 Subject: [PATCH 142/163] use backend handler to build parameters when sending test prompt --- llm_server/cluster/backend.py | 9 +++++++-- llm_server/workers/inferencer.py | 1 - 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index 2a7edc3..62dc2c5 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -6,6 +6,7 @@ from llm_server.cluster.stores import redis_running_models from llm_server.custom_redis import redis from llm_server.llm.generator import generator from llm_server.llm.info import get_info +from llm_server.llm.vllm.vllm_backend import VLLMBackend from llm_server.routes.queue import priority_queue from llm_server.routes.stats import calculate_wait_time, get_active_gen_workers_model @@ -33,11 +34,15 @@ def is_valid_model(model_name: str): def test_backend(backend_url: str, test_prompt: bool = False): backend_info = cluster_config.get_backend(backend_url) if test_prompt: - data = { - "prompt": "Test prompt", + handler = VLLMBackend(backend_url) + parameters, _ = handler.get_parameters({ "stream": False, "temperature": 0, "max_new_tokens": 3, + }) + data = { + 'prompt': 'test prompt', + **parameters } try: success, response, err = generator(data, backend_url, timeout=10) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index a1dd749..0738c1b 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -66,7 +66,6 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str json_obj = json.loads(json_str.decode()) new = json_obj['text'][0].split(prompt + generated_text)[1] generated_text = generated_text + new - print(new) except IndexError: # ???? continue -- 2.34.1 From 92e4ecd8a138965ef371ac508eaca9b266cf8c33 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 18 Oct 2023 09:03:10 -0600 Subject: [PATCH 143/163] refer to queue for tracking IP count rather than seperate value --- llm_server/routes/queue.py | 49 +++++++++++------------------------ llm_server/workers/printer.py | 2 +- server.py | 2 +- 3 files changed, 17 insertions(+), 36 deletions(-) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 834c844..66659d4 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -1,9 +1,9 @@ -import json import pickle import time from typing import Tuple from uuid import uuid4 +import ujson as json from redis import Redis from llm_server import opts @@ -28,23 +28,22 @@ class RedisPriorityQueue: self.redis = RedisCustom(name, db=db) def put(self, item, priority: int, selected_model: str, do_stream: bool = False): + # TODO: remove this when we're sure nothing strange is happening assert item is not None assert priority is not None assert selected_model is not None event = DataEvent() + # Check if the IP is already in the dictionary and if it has reached the limit - ip_count = self.redis.hget('queued_ip_count', item[1]) - if ip_count: - ip_count = int(ip_count) + ip_count = self.get_ip_request_count(item[1]) _, simultaneous_ip = get_token_ratelimit(item[2]) if ip_count and int(ip_count) >= simultaneous_ip and priority != 0: - print(f'Rejecting request from {item[1]} - {ip_count} requests in progress.') + print(f'Rejecting request from {item[1]} - {ip_count} request queued.') return None # reject the request timestamp = time.time() self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model, timestamp, do_stream)): -priority}) - self.increment_ip_count(item[1], 'queued_ip_count') return event def get(self): @@ -52,34 +51,20 @@ class RedisPriorityQueue: data = self.redis.zpopmin('queue') if data: item = json.loads(data[0][0]) - client_ip = item[0][1] - self.decrement_ip_count(client_ip, 'queued_ip_count') return item time.sleep(0.1) # wait for something to be added to the queue - # def print_all_items(self): - # items = self.redis.zrange('queue', 0, -1) - # to_print = [] - # for item in items: - # to_print.append(item.decode('utf-8')) - # print(f'ITEMS {self.name} -->', to_print) - - def increment_ip_count(self, client_ip: str, redis_key): - self.redis.hincrby(redis_key, client_ip, 1) - - def decrement_ip_count(self, client_ip: str, redis_key): - new_count = self.redis.hincrby(redis_key, client_ip, -1) - if new_count <= 0: - self.redis.hdel(redis_key, client_ip) - def __len__(self): return self.redis.zcard('queue') - def get_queued_ip_count(self, client_ip: str): - q = self.redis.hget('queued_ip_count', client_ip) - if not q: - return 0 - return 0 + def get_ip_request_count(self, client_ip: str): + items = self.redis.zrange('queue', 0, -1) + count = 0 + for item in items: + item_data = json.loads(item) + if item_data[0][1] == client_ip: + count += 1 + return count def flush(self): self.redis.flush() @@ -94,10 +79,7 @@ class RedisPriorityQueue: timestamp = item_data[-2] if now - timestamp > opts.backend_generate_request_timeout: self.redis.zrem('queue', 0, item) - data = json.loads(item.decode('utf-8')) - event_id = data[1] - client_ip = data[0][1] - self.decrement_ip_count(client_ip, 'queued_ip_count') + event_id = item_data[1] event = DataEvent(event_id) event.set((False, None, 'closed')) print('Removed timed-out item from queue:', event_id) @@ -114,7 +96,6 @@ class DataEvent: self.redis.publish(self.event_id, pickle.dumps(data)) def wait(self): - # TODO: implement timeout for item in self.pubsub.listen(): if item['type'] == 'message': return pickle.loads(item['data']) @@ -157,7 +138,7 @@ class PriorityQueue: count = 0 for backend_url in self.get_backends(): queue = RedisPriorityQueue(backend_url) - count += queue.get_queued_ip_count(client_ip) + count += queue.get_ip_request_count(client_ip) return count def put(self, backend_url, item: Tuple[dict, str, str, dict], priority: int, selected_model: str, do_stream: bool = False): diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index fe0d129..c9c421e 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -28,4 +28,4 @@ def console_printer(): # Active Workers and Processing should read the same. If not, that's an issue. logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') - time.sleep(10) + time.sleep(2) diff --git a/server.py b/server.py index e33d55a..43aa9d2 100644 --- a/server.py +++ b/server.py @@ -30,7 +30,7 @@ from llm_server.routes.v1 import bp from llm_server.routes.v1.generate_stats import generate_stats from llm_server.sock import init_wssocket -# TODO: queue item timeout +# TODO: seperate queue item timeout for websockets (make longer, like 5 minutes) # TODO: return an `error: True`, error code, and error message rather than just a formatted message # TODO: what happens when all backends are offline? What about the "online" key in the stats page? # TODO: redis SCAN vs KEYS?? -- 2.34.1 From 50377eca224a92d1f920f45feb69617d5a0b2479 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 18 Oct 2023 09:09:22 -0600 Subject: [PATCH 144/163] track lag on get_ip_request_count() --- llm_server/routes/queue.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 66659d4..62c9c42 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -58,12 +58,25 @@ class RedisPriorityQueue: return self.redis.zcard('queue') def get_ip_request_count(self, client_ip: str): + """ + Get the number of requests in the queue from a specific IP. + This is a bit inefficient since we iterate over the entire queue, but + keeps the queue as a single point of truth instead of tracking a separate hashed + set which can get confusing. + If we run into slowdowns in the future, we should go back to the hashed set approach. + :param client_ip: + :return: + """ + start_time = time.time() items = self.redis.zrange('queue', 0, -1) count = 0 for item in items: item_data = json.loads(item) if item_data[0][1] == client_ip: count += 1 + elapsed_time = time.time() - start_time + if elapsed_time > 0.5: + raise Exception(f"!!! get_ip_request_count took {elapsed_time} seconds to execute !!!") return count def flush(self): -- 2.34.1 From 56a2ca464bc80118dd0c3df94ff9a56a6c4ece50 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 18 Oct 2023 09:12:30 -0600 Subject: [PATCH 145/163] change print --- llm_server/routes/request_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index df07c29..8e25b1b 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -222,7 +222,7 @@ class RequestHandler: processing_ip = 0 if queued_ip_count + processing_ip >= self.token_simultaneous_ip: - print(f'Rejecting request from {self.client_ip} - {queued_ip_count + processing_ip} already queued/processing.') + print(f'Rejecting request from {self.client_ip} - {queued_ip_count} queued, {processing_ip} processing.') return True else: return False -- 2.34.1 From b9566e9db7ea7a1589a75a6e1c72847822976523 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Wed, 18 Oct 2023 09:23:54 -0600 Subject: [PATCH 146/163] docs and stuff --- llm_server/custom_redis.py | 26 ++++++++++++++------------ llm_server/routes/queue.py | 12 ++++++++++++ llm_server/routes/server_error.py | 2 +- 3 files changed, 27 insertions(+), 13 deletions(-) diff --git a/llm_server/custom_redis.py b/llm_server/custom_redis.py index 886230a..a055537 100644 --- a/llm_server/custom_redis.py +++ b/llm_server/custom_redis.py @@ -16,7 +16,8 @@ ONE_MONTH_SECONDS = 2678000 class RedisCustom(Redis): """ - A wrapper class to set prefixes to keys. + A simple wrapper class for Redis to create a "namespace" within a DB, + which simplyifies key management. """ def __init__(self, prefix, **kwargs): @@ -233,21 +234,22 @@ class RedisCustom(Redis): return self.redis.lpop(self._key(name), count) def zrange( - self, - name: KeyT, - start: int, - end: int, - desc: bool = False, - withscores: bool = False, - score_cast_func: Union[type, Callable] = float, - byscore: bool = False, - bylex: bool = False, - offset: int = None, - num: int = None, + self, + name: KeyT, + start: int, + end: int, + desc: bool = False, + withscores: bool = False, + score_cast_func: Union[type, Callable] = float, + byscore: bool = False, + bylex: bool = False, + offset: int = None, + num: int = None, ): return self.redis.zrange(self._key(name), start, end, desc, withscores, score_cast_func, byscore, bylex, offset, num) def zrem(self, name: KeyT, *values: FieldT): return self.redis.zrem(self._key(name), *values) + redis = RedisCustom('local_llm') diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 62c9c42..3bbdd1c 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -23,6 +23,10 @@ def decrement_ip_count(client_ip: str, redis_key): class RedisPriorityQueue: + """ + A queue for a specific backend. + """ + def __init__(self, name, db: int = 12): self.name = name self.redis = RedisCustom(name, db=db) @@ -99,6 +103,10 @@ class RedisPriorityQueue: class DataEvent: + """ + Class to simplify pub/sub communication between consumers and producers (MASTERS and SLAVES lololololol). + """ + def __init__(self, event_id=None): self.event_id = event_id if event_id else str(uuid4()) self.redis = Redis(host='localhost', port=6379, db=14) @@ -134,6 +142,10 @@ def decr_active_workers(selected_model: str, backend_url: str): class PriorityQueue: + """ + Helper class to wrangler all the different queues. + """ + def __init__(self, backends: set = None): """ Only have to load the backends once. diff --git a/llm_server/routes/server_error.py b/llm_server/routes/server_error.py index fec3836..b18d7f1 100644 --- a/llm_server/routes/server_error.py +++ b/llm_server/routes/server_error.py @@ -1,3 +1,3 @@ def handle_server_error(e): print(e) - return {'error': True}, 500 + return {'error': True, 'code': 500, 'message': 'Internal Server Error :('}, 500 -- 2.34.1 From 6e74ce7c285af4288512a94b61b011b48950596a Mon Sep 17 00:00:00 2001 From: Cyberes Date: Thu, 19 Oct 2023 17:59:27 -0600 Subject: [PATCH 147/163] fix old code in completions --- llm_server/routes/openai/completions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index d77ff0d..5d5721e 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -84,9 +84,10 @@ def openai_completions(model_name=None): } }) - stats = redis.get('proxy_stats', dtype=dict) - if stats: - response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] + # TODO: + # stats = redis.get('proxy_stats', dtype=dict) + # if stats: + # response.headers['x-ratelimit-reset-requests'] = stats['queue']['estimated_wait_sec'] return response, 200 else: if not opts.enable_streaming: -- 2.34.1 From 4f5b2dbecba98eec5a536696c48e3b47620d974c Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 12:14:34 -0600 Subject: [PATCH 148/163] add tests --- llm_server/routes/openai/__init__.py | 15 ++++- other/tests/config.sh | 11 ++++ other/tests/generate.sh | 58 +++++++++++++++++ other/tests/oai-chat-completion.sh | 52 +++++++++++++++ other/tests/oai-completion.sh | 52 +++++++++++++++ other/tests/start-bulk.sh | 64 +++++++++++++++++++ .../stream.py} | 60 +++++++++++------ 7 files changed, 290 insertions(+), 22 deletions(-) create mode 100644 other/tests/config.sh create mode 100755 other/tests/generate.sh create mode 100755 other/tests/oai-chat-completion.sh create mode 100755 other/tests/oai-completion.sh create mode 100755 other/tests/start-bulk.sh rename other/{ooba-test-streaming.py => tests/stream.py} (54%) mode change 100644 => 100755 diff --git a/llm_server/routes/openai/__init__.py b/llm_server/routes/openai/__init__.py index 3a69aa7..6ca4d92 100644 --- a/llm_server/routes/openai/__init__.py +++ b/llm_server/routes/openai/__init__.py @@ -19,7 +19,20 @@ def before_oai_request(): @openai_bp.errorhandler(500) @openai_model_bp.errorhandler(500) def handle_error(e): - return handle_server_error(e) + """ + Found Codes: + "auth_subrequest_error" + """ + + print(e) + return jsonify({ + "error": { + "message": "Internal server error", + "type": "auth_subrequest_error", + "param": None, + "code": "internal_error" + } + }), 500 from .models import openai_list_models diff --git a/other/tests/config.sh b/other/tests/config.sh new file mode 100644 index 0000000..64bea46 --- /dev/null +++ b/other/tests/config.sh @@ -0,0 +1,11 @@ +HOST="proxy.chub-archive.evulid.cc" + +AUTH_KEY="TEST_1df979f0-6df1-41bd-814a-e99b1680e727" + +PROXY_SERVERS=( + "http://172.0.4.7:3128" + "http://172.0.4.8:3128" + "http://172.0.4.10:3128" + "http://172.0.4.12:3128" + "http://172.0.4.13:3128" +) diff --git a/other/tests/generate.sh b/other/tests/generate.sh new file mode 100755 index 0000000..f36d73e --- /dev/null +++ b/other/tests/generate.sh @@ -0,0 +1,58 @@ +#!/bin/bash + +SLEEP_TIME=2 + +while getopts p:t: flag; do + case "${flag}" in + p) PROXY_CHOICE=${OPTARG} ;; + t) SLEEP_TIME=${OPTARG} ;; + *) ;; + esac +done + +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + +source "$DIR/config.sh" + +if [ -n "$PROXY_CHOICE" ]; then + our_proxy_server="${PROXY_SERVERS[$PROXY_CHOICE]}" + echo "Using $our_proxy_server" +else + our_proxy_server="" +fi + +while true; do + echo "--> START <--" + + DATA=$( + cat < DONE <--\n" + sleep $SLEEP_TIME +done diff --git a/other/tests/oai-chat-completion.sh b/other/tests/oai-chat-completion.sh new file mode 100755 index 0000000..5355a8a --- /dev/null +++ b/other/tests/oai-chat-completion.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +DO_STREAM=false +SLEEP_TIME=2 + +while getopts p:t:s flag; do + case "${flag}" in + s) DO_STREAM=true ;; + p) PROXY_CHOICE=${OPTARG} ;; + t) SLEEP_TIME=${OPTARG} ;; + *) ;; + esac +done + +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + +source "$DIR/config.sh" + +if [ ! -z "$PROXY_CHOICE" ]; then + our_proxy_server="${PROXY_SERVERS[$PROXY_CHOICE]}" + echo "Using $our_proxy_server" +else + our_proxy_server="" +fi + +while true; do + echo "--> START <--" + + DATA=$( + cat < DONE <--\n" + sleep $SLEEP_TIME +done diff --git a/other/tests/oai-completion.sh b/other/tests/oai-completion.sh new file mode 100755 index 0000000..f89d1c8 --- /dev/null +++ b/other/tests/oai-completion.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +DO_STREAM=false +SLEEP_TIME=2 + +while getopts p:t:s flag; do + case "${flag}" in + s) DO_STREAM=true ;; + p) PROXY_CHOICE=${OPTARG} ;; + t) SLEEP_TIME=${OPTARG} ;; + *) ;; + esac +done + +SOURCE=${BASH_SOURCE[0]} +while [ -L "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + SOURCE=$(readlink "$SOURCE") + [[ $SOURCE != /* ]] && SOURCE=$DIR/$SOURCE # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +DIR=$(cd -P "$(dirname "$SOURCE")" >/dev/null 2>&1 && pwd) + +source "$DIR/config.sh" + +if [ ! -z "$PROXY_CHOICE" ]; then + our_proxy_server="${PROXY_SERVERS[$PROXY_CHOICE]}" + echo "Using $our_proxy_server" +else + our_proxy_server="" +fi + +while true; do + echo "--> START <--" + + DATA=$( + cat < DONE <--\n" + sleep $SLEEP_TIME +done diff --git a/other/tests/start-bulk.sh b/other/tests/start-bulk.sh new file mode 100755 index 0000000..49e92a6 --- /dev/null +++ b/other/tests/start-bulk.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +# Function to display help message +function display_help { + echo "Usage: $0 -n num_windows -c command" + echo + echo " -n, --number Number of windows to create" + echo " -c, --command Command to run in each window" + echo + exit 1 +} + +# Parse command line arguments +while getopts "n:c:h" opt; do + case ${opt} in + n) + num_windows=${OPTARG} + ;; + c) + command=${OPTARG} + ;; + h) + display_help + ;; + \?) + echo "Invalid option: -$OPTARG" 1>&2 + display_help + ;; + :) + echo "Option -$OPTARG requires an argument." 1>&2 + display_help + ;; + esac +done + +# Check if number of windows and command are provided +if [ -z "$num_windows" ] || [ -z "$command" ]; then + echo "Both number of windows and command are required." + display_help +fi + +# Calculate rows and columns +rows=$(echo "sqrt($num_windows)" | bc) +columns=$(echo "($num_windows + $rows - 1) / $rows" | bc) + +# Create a new tmux session +tmux new-session -d -s my_session "$command -p 0" + +# Create the remaining windows +for ((i = 1; i < $num_windows; i++)); do + if ((i % $columns == 0)); then + tmux select-layout -t my_session:0 tiled + tmux select-pane -t 0 + tmux split-window -t my_session:0 -v "$command -p $i" + else + tmux split-window -t my_session:0 -h "$command -p $i" + fi +done + +# Balance the windows +tmux select-layout -t my_session:0 tiled + +# Attach to the session +tmux attach-session -t my_session diff --git a/other/ooba-test-streaming.py b/other/tests/stream.py old mode 100644 new mode 100755 similarity index 54% rename from other/ooba-test-streaming.py rename to other/tests/stream.py index 7f5185d..f5c4023 --- a/other/ooba-test-streaming.py +++ b/other/tests/stream.py @@ -1,38 +1,50 @@ import asyncio import json import sys +import os +import time +from pathlib import Path try: import websockets except ImportError: print("Websockets package not found. Make sure it's installed.") -# For local streaming, the websockets are hosted without ssl - ws:// -HOST = 'localhost:5000' -URI = f'ws://{HOST}/api/v1/stream' +script_path = os.path.dirname(os.path.realpath(__file__)) -# For reverse-proxied streaming, the remote will likely host with ssl - wss:// -# URI = 'wss://your-uri-here.trycloudflare.com/api/v1/stream' +def parse_bash_config(file_path): + config = {} + with open(file_path, 'r') as f: + for line in f: + if line.startswith('#') or '=' not in line: + continue + key, value = line.strip().split('=', 1) + if value.startswith('"') and value.endswith('"'): + value = value[1:-1] + elif value.startswith('(') and value.endswith(')'): + value = value[1:-1].split() + value = [v.strip('"') for v in value] + config[key] = value + return config + + +config = parse_bash_config(Path(script_path, 'config.sh')) async def run(context): - # Note: the selected defaults change from time to time. request = { 'prompt': context, 'max_new_tokens': 250, 'auto_max_new_tokens': False, 'max_tokens_second': 0, - - # Generation params. If 'preset' is set to different than 'None', the values - # in presets/preset-name.yaml are used instead of the individual numbers. 'preset': 'None', 'do_sample': True, 'temperature': 0.7, 'top_p': 0.1, 'typical_p': 1, - 'epsilon_cutoff': 0, # In units of 1e-4 - 'eta_cutoff': 0, # In units of 1e-4 + 'epsilon_cutoff': 0, + 'eta_cutoff': 0, 'tfs': 1, 'top_a': 0, 'repetition_penalty': 1.18, @@ -49,7 +61,6 @@ async def run(context): 'mirostat_eta': 0.1, 'guidance_scale': 1, 'negative_prompt': '', - 'seed': -1, 'add_bos_token': True, 'truncation_length': 2048, @@ -59,7 +70,7 @@ async def run(context): 'stopping_strings': [] } - async with websockets.connect(URI, ping_interval=None) as websocket: + async with websockets.connect(f'wss://{config["HOST"]}/api/v1/stream', ping_interval=None) as websocket: await websocket.send(json.dumps(request)) yield context # Remove this if you just want to see the reply @@ -68,21 +79,28 @@ async def run(context): incoming_data = await websocket.recv() incoming_data = json.loads(incoming_data) + print(incoming_data) + match incoming_data['event']: - case 'text_stream': - yield incoming_data['text'] + # case 'text_stream': + # yield incoming_data['text'] case 'stream_end': return async def print_response_stream(prompt): - async for response in run(prompt): - print(response, end='') - sys.stdout.flush() # If we don't flush, we won't see tokens in realtime. - print('\n\nfinished') + try: + async for response in run(prompt): + print(response, end='') + sys.stdout.flush() # If we don't flush, we won't see tokens in realtime. + except Exception as e: + print(e) if __name__ == '__main__': - # prompt = "In order to make homemade bread, follow these steps:\n1)" prompt = "Write a 300 word description of how an apple tree grows.\n\n" - asyncio.run(print_response_stream(prompt)) + while True: + print('--> START <--') + asyncio.run(print_response_stream(prompt)) + print('--> DONE <--') + time.sleep(2) -- 2.34.1 From 0abd4b94fb7bbe7135110fc3a2773ac1d8ca2c75 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 13:57:20 -0600 Subject: [PATCH 149/163] track down keyerror --- llm_server/cluster/cluster_config.py | 35 ++++++++++++++++------------ 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/llm_server/cluster/cluster_config.py b/llm_server/cluster/cluster_config.py index 453fa33..891dfc1 100644 --- a/llm_server/cluster/cluster_config.py +++ b/llm_server/cluster/cluster_config.py @@ -1,5 +1,6 @@ import hashlib import pickle +import traceback from llm_server import opts from llm_server.cluster.redis_cycle import add_backend_cycler, redis_cycle @@ -75,24 +76,28 @@ def get_backends(): priority = b['priority'] result[k] = {'status': status, 'priority': priority} - if not opts.prioritize_by_size: - online_backends = sorted( - ((url, info) for url, info in backends.items() if info['online']), + try: + if not opts.prioritize_by_size: + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: -kv[1]['priority'], + reverse=True + ) + else: + online_backends = sorted( + ((url, info) for url, info in backends.items() if info['online']), + key=lambda kv: estimate_model_size(kv[1]['model_config']), + reverse=True + ) + offline_backends = sorted( + ((url, info) for url, info in backends.items() if not info['online']), key=lambda kv: -kv[1]['priority'], reverse=True ) - else: - online_backends = sorted( - ((url, info) for url, info in backends.items() if info['online']), - key=lambda kv: estimate_model_size(kv[1]['model_config']), - reverse=True - ) - offline_backends = sorted( - ((url, info) for url, info in backends.items() if not info['online']), - key=lambda kv: -kv[1]['priority'], - reverse=True - ) - return [url for url, info in online_backends], [url for url, info in offline_backends] + return [url for url, info in online_backends], [url for url, info in offline_backends] + except KeyError: + traceback.print_exc() + print(backends) def get_a_cluster_backend(model=None): -- 2.34.1 From e838f591aa76688f033621295bc51b3067b48c2a Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 14:00:24 -0600 Subject: [PATCH 150/163] fix keyerror? --- llm_server/routes/helpers/model.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/llm_server/routes/helpers/model.py b/llm_server/routes/helpers/model.py index ca35867..bf18b66 100644 --- a/llm_server/routes/helpers/model.py +++ b/llm_server/routes/helpers/model.py @@ -5,9 +5,11 @@ def estimate_model_size(config: dict): :param config: :return: """ - vocab_size = config['vocab_size'] - hidden_size = config['hidden_size'] - num_hidden_layers = config['num_hidden_layers'] - intermediate_size = config['intermediate_size'] - total_params = (vocab_size * hidden_size) + (num_hidden_layers * ((hidden_size * intermediate_size * 4) + (hidden_size * hidden_size * 3))) - return int(total_params / 1e9) + vocab_size = config.get('vocab_size') + hidden_size = config.get('hidden_size') + num_hidden_layers = config.get('num_hidden_layers') + intermediate_size = config.get('intermediate_size') + if vocab_size and hidden_size and num_hidden_layers and intermediate_size: + total_params = (vocab_size * hidden_size) + (num_hidden_layers * ((hidden_size * intermediate_size * 4) + (hidden_size * hidden_size * 3))) + return int(total_params / 1e9) + return 0 -- 2.34.1 From 763139c949b56413ccd015333e32771572c818cc Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 14:02:30 -0600 Subject: [PATCH 151/163] fix keyerror --- llm_server/routes/v1/generate_stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index 3bfbca6..c5fa456 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -2,8 +2,8 @@ import time from datetime import datetime from llm_server import opts -from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.backend import get_model_choices +from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.database.database import get_distinct_ips_24h, sum_column from llm_server.helpers import deep_sort @@ -67,7 +67,7 @@ def generate_stats(regen: bool = False): backend_uptime = int((datetime.now() - datetime.fromtimestamp(backend_info['startup_time'])).total_seconds()) if opts.show_uptime else None output['backends'][backend_info['hash']] = { 'uptime': backend_uptime, - 'max_tokens': backend_info['model_config']['max_position_embeddings'], + 'max_tokens': backend_info['model_config'].get('max_position_embeddings', -1), 'model': backend_info['model'], 'mode': backend_info['mode'], 'nvidia': backend_info['nvidia'], -- 2.34.1 From 1a1523240041c5f294ecbe8eb68888b003b5e99f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 17:35:21 -0600 Subject: [PATCH 152/163] tests: make sure all prompts are the same --- other/tests/generate.sh | 2 +- other/tests/oai-completion.sh | 2 +- other/tests/stream.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/other/tests/generate.sh b/other/tests/generate.sh index f36d73e..b1443c0 100755 --- a/other/tests/generate.sh +++ b/other/tests/generate.sh @@ -33,7 +33,7 @@ while true; do DATA=$( cat < START <--') asyncio.run(print_response_stream(prompt)) -- 2.34.1 From f39e976b34988539e0dfdb27b6e15bf20beb13f5 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 17:41:47 -0600 Subject: [PATCH 153/163] dameon printer: Calculate the queue size the same way it's done on the stats --- llm_server/workers/printer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index c9c421e..a62f85c 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -1,6 +1,7 @@ import logging import time +from llm_server.cluster.backend import get_running_models from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue @@ -24,8 +25,15 @@ def console_printer(): for k in processing: processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] + + # Calculate the queue size the same way it's done on the stats. + queue_size = 0 + running_models = get_running_models() + for model in running_models: + queue_size += priority_queue.len(model) + activity = priority_queue.activity() # Active Workers and Processing should read the same. If not, that's an issue. - logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {len(priority_queue)} | Backends Online: {len(backends)}') + logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {queue_size} | Backends Online: {len(backends)}') time.sleep(2) -- 2.34.1 From e236e93a79a43383ddc567c3ac61effe063c12f1 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 20 Oct 2023 18:26:41 -0600 Subject: [PATCH 154/163] clean up a bit --- llm_server/routes/request_handler.py | 2 +- llm_server/workers/printer.py | 5 ++--- other/tests/start-bulk.sh | 12 ++++++------ 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 8e25b1b..2d81d6a 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -222,7 +222,7 @@ class RequestHandler: processing_ip = 0 if queued_ip_count + processing_ip >= self.token_simultaneous_ip: - print(f'Rejecting request from {self.client_ip} - {queued_ip_count} queued, {processing_ip} processing.') + print(f'Rejecting request from {self.client_ip} - {processing_ip} processing, {queued_ip_count} queued') return True else: return False diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index a62f85c..759ae67 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -25,6 +25,7 @@ def console_printer(): for k in processing: processing_count += redis.get(k, default=0, dtype=int) backends = [k for k, v in cluster_config.all().items() if v['online']] + activity = priority_queue.activity() # Calculate the queue size the same way it's done on the stats. queue_size = 0 @@ -32,8 +33,6 @@ def console_printer(): for model in running_models: queue_size += priority_queue.len(model) - activity = priority_queue.activity() - # Active Workers and Processing should read the same. If not, that's an issue. logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {queue_size} | Backends Online: {len(backends)}') - time.sleep(2) + time.sleep(10) diff --git a/other/tests/start-bulk.sh b/other/tests/start-bulk.sh index 49e92a6..6f254d5 100755 --- a/other/tests/start-bulk.sh +++ b/other/tests/start-bulk.sh @@ -44,21 +44,21 @@ rows=$(echo "sqrt($num_windows)" | bc) columns=$(echo "($num_windows + $rows - 1) / $rows" | bc) # Create a new tmux session -tmux new-session -d -s my_session "$command -p 0" +tmux new-session -d -s llm_tester "$command -p 0" # Create the remaining windows for ((i = 1; i < $num_windows; i++)); do if ((i % $columns == 0)); then - tmux select-layout -t my_session:0 tiled + tmux select-layout -t llm_tester:0 tiled tmux select-pane -t 0 - tmux split-window -t my_session:0 -v "$command -p $i" + tmux split-window -t llm_tester:0 -v "$command -p $i" else - tmux split-window -t my_session:0 -h "$command -p $i" + tmux split-window -t llm_tester:0 -h "$command -p $i" fi done # Balance the windows -tmux select-layout -t my_session:0 tiled +tmux select-layout -t llm_tester:0 tiled # Attach to the session -tmux attach-session -t my_session +tmux attach-session -t llm_tester -- 2.34.1 From d43f110a14ed2acf3592681fbc17aa90e2738c49 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Sun, 22 Oct 2023 12:19:20 -0600 Subject: [PATCH 155/163] fix redis cycle and add no reset to daemon --- daemon.py | 11 +++++++++-- llm_server/cluster/redis_cycle.py | 10 +++++++--- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/daemon.py b/daemon.py index 35c1d59..305c202 100644 --- a/daemon.py +++ b/daemon.py @@ -1,3 +1,4 @@ +import argparse import os import sys import time @@ -20,8 +21,14 @@ else: config_path = Path(script_path, 'config', 'config.yml') if __name__ == "__main__": - Redis().flushall() - print('Flushed Redis.') + parser = argparse.ArgumentParser( + description='Daemon microservice.') + parser.add_argument('--no-reset', action='store_true', help="Don't clear the Redis server databases.") + args = parser.parse_args() + + if not args.no_reset: + Redis().flushall() + print('Flushed Redis.') success, config, msg = load_config(config_path) if not success: diff --git a/llm_server/cluster/redis_cycle.py b/llm_server/cluster/redis_cycle.py index 7cff2c4..266241d 100644 --- a/llm_server/cluster/redis_cycle.py +++ b/llm_server/cluster/redis_cycle.py @@ -9,11 +9,15 @@ def redis_cycle(list_name): :param list_name: :return: """ - to_move = redis_cycler_db.rpop(list_name) + pipeline = redis_cycler_db.pipeline() + pipeline.lpop(list_name) + to_move = pipeline.execute()[0] if not to_move: return [] - redis_cycler_db.lpush(list_name, to_move) - new_list = redis_cycler_db.lrange(list_name, 0, -1) + pipeline.rpush(list_name, to_move) + pipeline.lrange(list_name, 0, -1) + results = pipeline.execute() + new_list = results[-1] return [x.decode('utf-8') for x in new_list] -- 2.34.1 From 3cf73fec9bd527f42a9427b680780a9a09c24745 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 23 Oct 2023 15:22:57 -0600 Subject: [PATCH 156/163] fix a few exceptions when all backends go offline --- llm_server/cluster/backend.py | 2 +- llm_server/routes/queue.py | 1 + llm_server/routes/v1/generate_stats.py | 2 -- llm_server/workers/inferencer.py | 30 ++++++++--------- llm_server/workers/printer.py | 45 +++++++++++++++++--------- server.py | 26 ++++++++++----- 6 files changed, 65 insertions(+), 41 deletions(-) diff --git a/llm_server/cluster/backend.py b/llm_server/cluster/backend.py index 62dc2c5..9e2e19b 100644 --- a/llm_server/cluster/backend.py +++ b/llm_server/cluster/backend.py @@ -110,7 +110,7 @@ def get_model_choices(regen: bool = False): default_backend_url = get_a_cluster_backend() default_backend_info = cluster_config.get_backend(default_backend_url) if not default_backend_info.get('model'): - return None, None + return {}, None default_model = default_backend_info['model'] redis.setp('model_choices', (model_choices, default_model)) diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 3bbdd1c..6bf66cd 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -118,6 +118,7 @@ class DataEvent: def wait(self): for item in self.pubsub.listen(): + print(item) if item['type'] == 'message': return pickle.loads(item['data']) diff --git a/llm_server/routes/v1/generate_stats.py b/llm_server/routes/v1/generate_stats.py index c5fa456..a9148b3 100644 --- a/llm_server/routes/v1/generate_stats.py +++ b/llm_server/routes/v1/generate_stats.py @@ -17,8 +17,6 @@ def generate_stats(regen: bool = False): return c model_choices, default_model = get_model_choices(regen=True) - if not model_choices or not default_model: - return 'Please wait for Redis to be populated...' base_client_api = redis.get('base_client_api', dtype=str) proompters_5_min = len(redis.zrangebyscore('recent_prompters', time.time() - 5 * 60, '+inf')) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 0738c1b..879e422 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -86,22 +86,21 @@ def worker(backend_url): redis_queue = RedisPriorityQueue(backend_url) while True: (request_json_body, client_ip, token, parameters), event_id, selected_model, timestamp, do_stream = redis_queue.get() - backend_info = cluster_config.get_backend(backend_url) - - if not backend_info['online']: - # TODO: communicate to caller - # redis.publish(event_id, 'offline') - return - - if not selected_model: - selected_model = backend_info['model'] - - stream_redis.delete(get_stream_name(worker_id)) # clean up any old streams - increment_ip_count(client_ip, 'processing_ips') - incr_active_workers(selected_model, backend_url) - status_redis.setp(str(worker_id), ('generating', client_ip)) - try: + backend_info = cluster_config.get_backend(backend_url) + + if not backend_info['online']: + redis.publish(event_id, 'canceled') + return + + if not selected_model: + selected_model = backend_info['model'] + + stream_redis.delete(get_stream_name(worker_id)) # clean up any old streams + increment_ip_count(client_ip, 'processing_ips') + incr_active_workers(selected_model, backend_url) + status_redis.setp(str(worker_id), ('generating', client_ip)) + if do_stream: # Return the name of the stream that the slave should connect to. event = DataEvent(event_id) @@ -120,6 +119,7 @@ def worker(backend_url): event.set((success, response, error_msg)) except: traceback.print_exc() + redis.publish(event_id, 'canceled') finally: decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index 759ae67..bfc62f8 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -1,10 +1,12 @@ import logging import time +import traceback -from llm_server.cluster.backend import get_running_models +from llm_server.cluster.backend import get_model_choices, get_running_models from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis from llm_server.routes.queue import priority_queue +from llm_server.routes.v1.generate_stats import generate_stats logger = logging.getLogger('console_printer') if not logger.handlers: @@ -19,20 +21,33 @@ if not logger.handlers: def console_printer(): time.sleep(3) while True: - processing = redis.keys('active_gen_workers:http*') # backends always start with http - processing_count = 0 - if len(processing): - for k in processing: - processing_count += redis.get(k, default=0, dtype=int) - backends = [k for k, v in cluster_config.all().items() if v['online']] - activity = priority_queue.activity() + try: + stats = generate_stats() + model_choices, default_model = get_model_choices() - # Calculate the queue size the same way it's done on the stats. - queue_size = 0 - running_models = get_running_models() - for model in running_models: - queue_size += priority_queue.len(model) + processing_count = 0 + backend_count = len(stats['backends']) - # Active Workers and Processing should read the same. If not, that's an issue. - logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {queue_size} | Backends Online: {len(backends)}') + if model_choices and default_model: + for model, info in model_choices.items(): + processing_count += info['processing'] + + # processing = redis.keys('active_gen_workers:http*') # backends always start with http + # processing_count = 0 + # if len(processing): + # for k in processing: + # processing_count += redis.get(k, default=0, dtype=int) + # backends = [k for k, v in cluster_config.all().items() if v['online']] + activity = priority_queue.activity() + + # Calculate the queue size the same way it's done on the stats. + queue_size = 0 + running_models = get_running_models() + for model in running_models: + queue_size += priority_queue.len(model) + + # Active Workers and Processing should read the same. If not, that's an issue. + logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {queue_size} | Backends Online: {backend_count}') + except: + traceback.print_exc() time.sleep(10) diff --git a/server.py b/server.py index 43aa9d2..37db35b 100644 --- a/server.py +++ b/server.py @@ -105,17 +105,27 @@ def home(): stats = generate_stats() model_choices, default_model = get_model_choices() - if not model_choices.get(default_model): - return 'The server is still starting up. Please wait...' + if default_model: + if not model_choices.get(default_model): + return 'The server is still starting up. Please wait...' - default_model_info = model_choices[default_model] + default_model_info = model_choices[default_model] - if default_model_info['queued'] == 0 and default_model_info['queued'] >= default_model_info['concurrent_gens']: - # There will be a wait if the queue is empty but prompts are processing, but we don't - # know how long. - default_estimated_wait_sec = f"less than {int(default_model_info['estimated_wait'])} seconds" + if default_model_info['queued'] == 0 and default_model_info['queued'] >= default_model_info['concurrent_gens']: + # There will be a wait if the queue is empty but prompts are processing, but we don't + # know how long. + default_estimated_wait_sec = f"less than {int(default_model_info['estimated_wait'])} seconds" + else: + default_estimated_wait_sec = f"{int(default_model_info['estimated_wait'])} seconds" else: - default_estimated_wait_sec = f"{int(default_model_info['estimated_wait'])} seconds" + default_model_info = { + 'model': 'OFFLINE', + 'processing': 0, + 'queued': 0, + + } + default_estimated_wait_sec = 'OFFLINE' + if len(config['analytics_tracking_code']): analytics_tracking_code = f"" -- 2.34.1 From 0771c2325cf95ac8c72c753eb7bf27b923cb7b70 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 23 Oct 2023 17:24:20 -0600 Subject: [PATCH 157/163] fix inference workers quitting when a backend is offline, start adding logging, improve tokenizer error handling --- daemon.py | 22 ++++++--- llm_server/cluster/worker.py | 6 +-- llm_server/llm/__init__.py | 20 +++++++- llm_server/llm/vllm/tokenize.py | 7 ++- llm_server/logging.py | 52 ++++++++++++++++++++ llm_server/opts.py | 5 ++ llm_server/routes/ooba_request_handler.py | 2 +- llm_server/routes/openai/__init__.py | 2 +- llm_server/routes/openai/chat_completions.py | 10 ++-- llm_server/routes/openai/completions.py | 9 ++-- llm_server/routes/openai_request_handler.py | 4 +- llm_server/routes/queue.py | 6 +-- llm_server/routes/request_handler.py | 13 ++--- llm_server/routes/server_error.py | 2 +- llm_server/routes/v1/generate_stream.py | 3 +- llm_server/workers/inferencer.py | 49 +++++++++++++----- llm_server/workers/moderator.py | 7 +-- llm_server/workers/printer.py | 39 ++++----------- llm_server/workers/threader.py | 18 ++++--- requirements.txt | 5 +- 20 files changed, 187 insertions(+), 94 deletions(-) create mode 100644 llm_server/logging.py diff --git a/daemon.py b/daemon.py index 305c202..0a5be10 100644 --- a/daemon.py +++ b/daemon.py @@ -1,4 +1,5 @@ import argparse +import logging import os import sys import time @@ -10,6 +11,7 @@ from llm_server.cluster.cluster_config import cluster_config from llm_server.config.load import load_config, parse_backends from llm_server.custom_redis import redis from llm_server.database.create import create_db +from llm_server.logging import create_logger, logging_info, init_logging from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.threader import start_background @@ -21,18 +23,26 @@ else: config_path = Path(script_path, 'config', 'config.yml') if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Daemon microservice.') + parser = argparse.ArgumentParser(description='Daemon microservice.') parser.add_argument('--no-reset', action='store_true', help="Don't clear the Redis server databases.") + parser.add_argument('-d', '--debug', action='store_true', help='Enable debug logging.') args = parser.parse_args() + # TODO: have this be set by either the arg or a config value + if args.debug: + logging_info.level = logging.DEBUG + + init_logging() + logger = create_logger('daemon') + logger.debug('Debug logging enabled.') + if not args.no_reset: Redis().flushall() - print('Flushed Redis.') + logger.info('Flushed Redis.') success, config, msg = load_config(config_path) if not success: - print('Failed to load config:', msg) + logger.info(f'Failed to load config: {msg}') sys.exit(1) create_db() @@ -40,13 +50,13 @@ if __name__ == "__main__": cluster_config.clear() cluster_config.load(parse_backends(config)) - print('Loading backend stats...') + logger.info('Loading backend stats...') generate_stats() start_background() redis.set('daemon_started', 1) - print('== Daemon Setup Complete ==\n') + logger.info('== Daemon Setup Complete ==') try: while True: diff --git a/llm_server/cluster/worker.py b/llm_server/cluster/worker.py index 951f18d..9652db9 100644 --- a/llm_server/cluster/worker.py +++ b/llm_server/cluster/worker.py @@ -1,8 +1,8 @@ import time from threading import Thread -from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.backend import test_backend +from llm_server.cluster.cluster_config import cluster_config from llm_server.cluster.stores import redis_running_models @@ -26,7 +26,6 @@ def cluster_worker(): def check_backend(n, v, test_prompt): online, backend_info = test_backend(v['backend_url'], test_prompt=test_prompt) - # purge_backend_from_running_models(n) if online: running_model = backend_info['model'] for k, v in backend_info.items(): @@ -36,7 +35,4 @@ def check_backend(n, v, test_prompt): for model in redis_running_models.keys(): redis_running_models.srem(model, n) - # redis_running_models.srem(backend_info['model'], n) - # backend_cycler_store.lrem(backend_info['model'], 1, n) - cluster_config.set_backend_value(n, 'online', online) diff --git a/llm_server/llm/__init__.py b/llm_server/llm/__init__.py index 73e439f..ac6702b 100644 --- a/llm_server/llm/__init__.py +++ b/llm_server/llm/__init__.py @@ -1,10 +1,28 @@ +import tiktoken + from llm_server.cluster.cluster_config import cluster_config from llm_server.llm import oobabooga, vllm +from llm_server.logging import create_logger + + +def fallback_tokenizer(prompt: str): + tokenizer = tiktoken.get_encoding("cl100k_base") + return len(tokenizer.encode(prompt)) + 10 def get_token_count(prompt: str, backend_url: str): backend_url = cluster_config.validate_backend(backend_url) - backend_mode = cluster_config.get_backend(backend_url)['mode'] + if not backend_url: + logger = create_logger('tokenizer') + logger.warning('using fallback tokenizer as there is no valid backend') + return fallback_tokenizer(prompt) + + backend_mode = cluster_config.get_backend(backend_url).get('mode') + if not backend_mode: + logger = create_logger('tokenizer') + logger.warning("using fallback tokenizer as the backend isn't initalized") + return fallback_tokenizer(prompt) + if backend_mode == 'vllm': return vllm.tokenize(prompt, backend_url) elif backend_mode == 'ooba': diff --git a/llm_server/llm/vllm/tokenize.py b/llm_server/llm/vllm/tokenize.py index 0a2f5c6..bdb6650 100644 --- a/llm_server/llm/vllm/tokenize.py +++ b/llm_server/llm/vllm/tokenize.py @@ -5,6 +5,7 @@ import tiktoken from llm_server import opts from llm_server.cluster.cluster_config import cluster_config +from llm_server.logging import create_logger def tokenize(prompt: str, backend_url: str) -> int: @@ -16,6 +17,8 @@ def tokenize(prompt: str, backend_url: str) -> int: return 0 assert isinstance(prompt, str) + logger = create_logger('tokenizer') + # The backend could have died between when the request was # submitted and now, so let's double check it's still online. backend_url = cluster_config.validate_backend(backend_url) @@ -33,7 +36,7 @@ def tokenize(prompt: str, backend_url: str) -> int: j = r.json() return j['length'] except Exception as e: - print(f'Failed to tokenize using VLLM - {e.__class__.__name__}') + logger.debug(f'Failed to tokenize using VLLM - {e.__class__.__name__}') return len(tokenizer.encode(chunk)) + 10 # Use a ThreadPoolExecutor to send all chunks to the server at once @@ -44,5 +47,5 @@ def tokenize(prompt: str, backend_url: str) -> int: try: data = future.result() except Exception as exc: - print('%r generated an exception: %s' % (chunk, exc)) + logger.warning('%r generated an exception: %s' % (chunk, exc)) return sum(future.result() for future in future_to_chunk) diff --git a/llm_server/logging.py b/llm_server/logging.py new file mode 100644 index 0000000..7e9aa74 --- /dev/null +++ b/llm_server/logging.py @@ -0,0 +1,52 @@ +import logging + +import coloredlogs + +from llm_server import opts + + +class LoggingInfo: + def __init__(self): + self._level = logging.INFO + self._format = opts.LOGGING_FORMAT + + @property + def level(self): + return self._level + + @level.setter + def level(self, value): + self._level = value + + @property + def format(self): + return self._format + + @format.setter + def format(self, value): + self._format = value + + +logging_info = LoggingInfo() + + +def init_logging(): + """ + Set up the parent logger. + :return: + """ + logger = logging.getLogger('llm_server') + logger.setLevel(logging_info.level) + + +def create_logger(name): + logger = logging.getLogger('llm_server').getChild(name) + logger.setLevel(logging_info.level) + if not logger.handlers: + handler = logging.StreamHandler() + handler.setLevel(logging_info.level) + formatter = logging.Formatter(logging_info.format) + handler.setFormatter(formatter) + logger.addHandler(handler) + coloredlogs.install(logger=logger, level=logging_info.level) + return logger diff --git a/llm_server/opts.py b/llm_server/opts.py index ada54a8..f75ba94 100644 --- a/llm_server/opts.py +++ b/llm_server/opts.py @@ -1,5 +1,8 @@ # Read-only global variables +# Uppercase variables are read-only globals. +# Lowercase variables are ones that are set on startup and are never changed. + # TODO: rewrite the config system so I don't have to add every single config default here frontend_api_mode = 'ooba' @@ -39,3 +42,5 @@ openai_moderation_timeout = 5 prioritize_by_size = False cluster_workers = 0 redis_stream_timeout = 25000 + +LOGGING_FORMAT = "%(asctime)s: %(levelname)s:%(name)s - %(message)s" diff --git a/llm_server/routes/ooba_request_handler.py b/llm_server/routes/ooba_request_handler.py index 804be74..aadda78 100644 --- a/llm_server/routes/ooba_request_handler.py +++ b/llm_server/routes/ooba_request_handler.py @@ -16,7 +16,7 @@ class OobaRequestHandler(RequestHandler): def handle_request(self, return_ok: bool = True): assert not self.used if self.offline: - print(messages.BACKEND_OFFLINE) + print('This backend is offline:', messages.BACKEND_OFFLINE) return self.handle_error(messages.BACKEND_OFFLINE) request_valid, invalid_response = self.validate_request() diff --git a/llm_server/routes/openai/__init__.py b/llm_server/routes/openai/__init__.py index 6ca4d92..c3837e4 100644 --- a/llm_server/routes/openai/__init__.py +++ b/llm_server/routes/openai/__init__.py @@ -24,7 +24,7 @@ def handle_error(e): "auth_subrequest_error" """ - print(e) + print('OAI returning error:', e) return jsonify({ "error": { "message": "Internal server error", diff --git a/llm_server/routes/openai/chat_completions.py b/llm_server/routes/openai/chat_completions.py index 0ef28ff..9ccc15f 100644 --- a/llm_server/routes/openai/chat_completions.py +++ b/llm_server/routes/openai/chat_completions.py @@ -29,9 +29,7 @@ def openai_chat_completions(model_name=None): else: handler = OpenAIRequestHandler(incoming_request=request, incoming_json=request_json_body, selected_model=model_name) if handler.offline: - msg = return_invalid_model_err(model_name) - print(msg) - return handler.handle_error(msg) + return return_invalid_model_err(model_name) if not request_json_body.get('stream'): try: @@ -100,7 +98,8 @@ def openai_chat_completions(model_name=None): # Need to do this before we enter generate() since we want to be able to # return a 408 if necessary. _, stream_name, error_msg = event.wait() - if error_msg == 'closed': + if error_msg: + print('OAI failed to start streaming:', error_msg) stream_name = None # set to null so that the Finally ignores it. return 'Request Timeout', 408 @@ -120,7 +119,8 @@ def openai_chat_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: - print('OAI streaming error:', data['error']) + # Not printing error since we can just check the daemon log. + print('OAI streaming encountered error') yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/routes/openai/completions.py b/llm_server/routes/openai/completions.py index 5d5721e..2524b17 100644 --- a/llm_server/routes/openai/completions.py +++ b/llm_server/routes/openai/completions.py @@ -29,9 +29,7 @@ def openai_completions(model_name=None): else: handler = OobaRequestHandler(incoming_request=request, incoming_json=request_json_body, selected_model=model_name) if handler.offline: - msg = return_invalid_model_err(model_name) - print(msg) - return handler.handle_error(msg) + return return_invalid_model_err(model_name) if handler.cluster_backend_info['mode'] != 'vllm': # TODO: implement other backends @@ -145,7 +143,8 @@ def openai_completions(model_name=None): oai_string = generate_oai_string(30) _, stream_name, error_msg = event.wait() - if error_msg == 'closed': + if error_msg: + print('OAI failed to start streaming:', error_msg) stream_name = None return 'Request Timeout', 408 @@ -165,7 +164,7 @@ def openai_completions(model_name=None): timestamp = int(stream_index.decode('utf-8').split('-')[0]) data = ujson.loads(item[b'data']) if data['error']: - print('OAI streaming error:', data['error']) + print('OAI streaming encountered error') yield 'data: [DONE]\n\n' return elif data['new']: diff --git a/llm_server/routes/openai_request_handler.py b/llm_server/routes/openai_request_handler.py index 9cbb11c..170eb77 100644 --- a/llm_server/routes/openai_request_handler.py +++ b/llm_server/routes/openai_request_handler.py @@ -29,7 +29,7 @@ class OpenAIRequestHandler(RequestHandler): assert not self.used if self.offline: msg = return_invalid_model_err(self.selected_model) - print(msg) + print('OAI Offline:', msg) return self.handle_error(msg) if opts.openai_silent_trim: @@ -106,7 +106,7 @@ class OpenAIRequestHandler(RequestHandler): return response, 429 def handle_error(self, error_msg: str, error_type: str = 'error') -> Tuple[flask.Response, int]: - print(error_msg) + print('OAI Error:', error_msg) return jsonify({ "error": { "message": "Invalid request, check your parameters and try again.", diff --git a/llm_server/routes/queue.py b/llm_server/routes/queue.py index 6bf66cd..ee66580 100644 --- a/llm_server/routes/queue.py +++ b/llm_server/routes/queue.py @@ -37,8 +37,6 @@ class RedisPriorityQueue: assert priority is not None assert selected_model is not None - event = DataEvent() - # Check if the IP is already in the dictionary and if it has reached the limit ip_count = self.get_ip_request_count(item[1]) _, simultaneous_ip = get_token_ratelimit(item[2]) @@ -47,6 +45,7 @@ class RedisPriorityQueue: return None # reject the request timestamp = time.time() + event = DataEvent() self.redis.zadd('queue', {json.dumps((item, event.event_id, selected_model, timestamp, do_stream)): -priority}) return event @@ -107,7 +106,7 @@ class DataEvent: Class to simplify pub/sub communication between consumers and producers (MASTERS and SLAVES lololololol). """ - def __init__(self, event_id=None): + def __init__(self, event_id: str = None): self.event_id = event_id if event_id else str(uuid4()) self.redis = Redis(host='localhost', port=6379, db=14) self.pubsub = self.redis.pubsub() @@ -118,7 +117,6 @@ class DataEvent: def wait(self): for item in self.pubsub.listen(): - print(item) if item['type'] == 'message': return pickle.loads(item['data']) diff --git a/llm_server/routes/request_handler.py b/llm_server/routes/request_handler.py index 2d81d6a..f4abfa6 100644 --- a/llm_server/routes/request_handler.py +++ b/llm_server/routes/request_handler.py @@ -44,12 +44,13 @@ class RequestHandler: self.backend_url = get_a_cluster_backend(selected_model) self.cluster_backend_info = cluster_config.get_backend(self.backend_url) - if not self.cluster_backend_info.get('mode'): - print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) - if not self.cluster_backend_info.get('model'): - print('keyerror: model -', selected_model, self.backend_url, self.cluster_backend_info) - if not self.cluster_backend_info.get('model_config'): - print('keyerror: model_config -', selected_model, self.backend_url, self.cluster_backend_info) + # Debug stuff + # if not self.cluster_backend_info.get('mode'): + # print('keyerror: mode -', selected_model, self.backend_url, self.cluster_backend_info) + # if not self.cluster_backend_info.get('model'): + # print('keyerror: model -', selected_model, self.backend_url, self.cluster_backend_info) + # if not self.cluster_backend_info.get('model_config'): + # print('keyerror: model_config -', selected_model, self.backend_url, self.cluster_backend_info) if not self.cluster_backend_info.get('mode') or not self.cluster_backend_info.get('model') or not self.cluster_backend_info.get('model_config'): self.offline = True diff --git a/llm_server/routes/server_error.py b/llm_server/routes/server_error.py index b18d7f1..a6d6f99 100644 --- a/llm_server/routes/server_error.py +++ b/llm_server/routes/server_error.py @@ -1,3 +1,3 @@ def handle_server_error(e): - print(e) + print('Internal Error:', e) return {'error': True, 'code': 500, 'message': 'Internal Server Error :('}, 500 diff --git a/llm_server/routes/v1/generate_stream.py b/llm_server/routes/v1/generate_stream.py index cdf939d..3ed2f58 100644 --- a/llm_server/routes/v1/generate_stream.py +++ b/llm_server/routes/v1/generate_stream.py @@ -130,7 +130,8 @@ def do_stream(ws, model_name): event_id = event.event_id _, stream_name, error_msg = event.wait() - if error_msg == 'closed': + if error_msg: + print('Stream failed to start streaming:', error_msg) ws.close(reason=1014, message='Request Timeout') return diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index 879e422..c9425b4 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -10,6 +10,7 @@ from redis import Redis from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import RedisCustom, redis from llm_server.llm.generator import generator +from llm_server.logging import create_logger from llm_server.routes.queue import DataEvent, RedisPriorityQueue, decr_active_workers, decrement_ip_count, incr_active_workers, increment_ip_count stream_redis = Redis(db=8) @@ -39,6 +40,7 @@ def get_stream_name(name: str): def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str, event_id: str): + logger = create_logger('inferencer') prompt = msg_to_backend['prompt'] stream_name = get_stream_name(stream_name) stream_redis.delete(get_stream_name(stream_name)) # be extra sure @@ -53,7 +55,7 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str if not chunk: break if event.is_set(): - print('Client canceled generation') + logger.debug('Client canceled generation') response.close() return @@ -70,40 +72,60 @@ def inference_do_stream(stream_name: str, msg_to_backend: dict, backend_url: str # ???? continue stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': new, 'completed': False, 'error': None})}) + except AttributeError as e: + if str(e) == "'bool' object has no attribute 'iter_content'": + # We don't care about these errors. + logger.debug('failed to stream from backend - no response') + else: + raise except Exception as e: stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': None, 'completed': True, 'error': f'{e.__class__.__name__}: {e}'})}) - traceback.print_exc() + raise # We won't handle the exception here. finally: # Publish final message to Redis stream stream_redis.xadd(stream_name, {'data': ujson.dumps({'new': None, 'completed': True, 'error': None})}) event.set() # stop the cancellation checking thread +# def worker(backend_url): + logger = create_logger('inferencer') status_redis = RedisCustom('worker_status') worker_id = str(uuid4()) status_redis.setp(str(worker_id), None) redis_queue = RedisPriorityQueue(backend_url) while True: + status_redis.setp(str(worker_id), 'waiting...') (request_json_body, client_ip, token, parameters), event_id, selected_model, timestamp, do_stream = redis_queue.get() + event = DataEvent(event_id) + try: backend_info = cluster_config.get_backend(backend_url) + except: + # This is not a critical error because it usually means that the backend is + # offline and this backend is in a state of transition from online to offline. + logger.debug(f'got an exception while getting info for backend {backend_url} - ', traceback.format_exc()) + event.set((False, None, 'exception')) + continue - if not backend_info['online']: - redis.publish(event_id, 'canceled') - return + if not backend_info['online']: + event.set((False, None, 'canceled')) + continue - if not selected_model: - selected_model = backend_info['model'] + if not selected_model: + selected_model = backend_info['model'] + logger.debug(f"Starting using {backend_url} and {selected_model}. Online: {backend_info['online']}") + + try: stream_redis.delete(get_stream_name(worker_id)) # clean up any old streams increment_ip_count(client_ip, 'processing_ips') incr_active_workers(selected_model, backend_url) - status_redis.setp(str(worker_id), ('generating', client_ip)) if do_stream: + status_redis.setp(str(worker_id), ('streaming', client_ip)) + # Return the name of the stream that the slave should connect to. - event = DataEvent(event_id) event.set((True, get_stream_name(worker_id), None)) msg_to_backend = { @@ -114,12 +136,12 @@ def worker(backend_url): inference_do_stream(worker_id, msg_to_backend, backend_url, event_id) else: # Normal inference (not streaming). + status_redis.setp(str(worker_id), ('generating', client_ip)) success, response, error_msg = generator(request_json_body, backend_url) - event = DataEvent(event_id) event.set((success, response, error_msg)) except: - traceback.print_exc() - redis.publish(event_id, 'canceled') + logger.error(traceback.format_exc()) + event.set((False, None, 'exception')) finally: decrement_ip_count(client_ip, 'processing_ips') decr_active_workers(selected_model, backend_url) @@ -127,6 +149,7 @@ def worker(backend_url): def start_workers(cluster: dict): + logger = create_logger('inferencer') i = 0 for item in cluster: for _ in range(item['concurrent_gens']): @@ -134,4 +157,4 @@ def start_workers(cluster: dict): t.daemon = True t.start() i += 1 - print(f'Started {i} inference workers.') + logger.info(f'Started {i} inference workers.') diff --git a/llm_server/workers/moderator.py b/llm_server/workers/moderator.py index 16ad0a2..6d56eee 100644 --- a/llm_server/workers/moderator.py +++ b/llm_server/workers/moderator.py @@ -7,6 +7,7 @@ import redis as redis_redis from llm_server import opts from llm_server.llm.openai.moderation import check_moderation_endpoint +from llm_server.logging import create_logger redis_moderation = redis_redis.Redis() @@ -18,7 +19,6 @@ def start_moderation_workers(num_workers): t.daemon = True t.start() i += 1 - print(f'Started {i} moderation workers.') # TODO: don't use UUID tags to identify items. Use native redis @@ -39,12 +39,13 @@ def get_results(tag, num_tasks): flagged_categories.add(item) num_results += 1 if time.time() - start_time > opts.openai_moderation_timeout: - print('----> Timed out waiting for result from moderator.') + logger.warning('Timed out waiting for result from moderator') break return list(flagged_categories) def moderation_worker(): + logger = create_logger('moderator') while True: result = redis_moderation.blpop(['queue:msgs_to_check']) try: @@ -52,7 +53,7 @@ def moderation_worker(): _, categories = check_moderation_endpoint(msg) redis_moderation.rpush('queue:flagged_categories', json.dumps((tag, categories))) except: - traceback.print_exc() + logger.error(traceback.format_exc()) continue diff --git a/llm_server/workers/printer.py b/llm_server/workers/printer.py index bfc62f8..deb3246 100644 --- a/llm_server/workers/printer.py +++ b/llm_server/workers/printer.py @@ -1,43 +1,24 @@ -import logging import time import traceback -from llm_server.cluster.backend import get_model_choices, get_running_models +from llm_server.cluster.backend import get_running_models from llm_server.cluster.cluster_config import cluster_config from llm_server.custom_redis import redis +from llm_server.logging import create_logger from llm_server.routes.queue import priority_queue -from llm_server.routes.v1.generate_stats import generate_stats - -logger = logging.getLogger('console_printer') -if not logger.handlers: - handler = logging.StreamHandler() - handler.setLevel(logging.INFO) - logger.setLevel(logging.INFO) - formatter = logging.Formatter("%(asctime)s: %(levelname)s:%(name)s - %(message)s") - handler.setFormatter(formatter) - logger.addHandler(handler) def console_printer(): + logger = create_logger('console_printer') time.sleep(3) while True: try: - stats = generate_stats() - model_choices, default_model = get_model_choices() - + processing = redis.keys('active_gen_workers:http*') # backends always start with http processing_count = 0 - backend_count = len(stats['backends']) - - if model_choices and default_model: - for model, info in model_choices.items(): - processing_count += info['processing'] - - # processing = redis.keys('active_gen_workers:http*') # backends always start with http - # processing_count = 0 - # if len(processing): - # for k in processing: - # processing_count += redis.get(k, default=0, dtype=int) - # backends = [k for k, v in cluster_config.all().items() if v['online']] + if len(processing): + for k in processing: + processing_count += redis.get(k, default=0, dtype=int) + backends = [k for k, v in cluster_config.all().items() if v['online']] activity = priority_queue.activity() # Calculate the queue size the same way it's done on the stats. @@ -47,7 +28,7 @@ def console_printer(): queue_size += priority_queue.len(model) # Active Workers and Processing should read the same. If not, that's an issue. - logger.info(f'REQUEST QUEUE -> Active Workers: {len([i for i in activity if i[1]])} | Processing: {processing_count} | Queued: {queue_size} | Backends Online: {backend_count}') + logger.info(f'Active Workers: {len([i for i in activity if (i[1] and i[1] != "waiting...")])} | Processing: {processing_count} | Queued: {queue_size} | Backends Online: {len(backends)}') except: - traceback.print_exc() + logger.error(traceback.format_exc()) time.sleep(10) diff --git a/llm_server/workers/threader.py b/llm_server/workers/threader.py index 0e47c02..542a630 100644 --- a/llm_server/workers/threader.py +++ b/llm_server/workers/threader.py @@ -3,6 +3,7 @@ from threading import Thread from llm_server import opts from llm_server.cluster.worker import cluster_worker +from llm_server.logging import create_logger from llm_server.routes.v1.generate_stats import generate_stats from llm_server.workers.inferencer import start_workers from llm_server.workers.logger import db_logger @@ -19,36 +20,39 @@ def cache_stats(): def start_background(): + logger = create_logger('threader') start_workers(opts.cluster) t = Thread(target=main_background_thread) t.daemon = True t.start() - print('Started the main background thread.') + logger.info('Started the main background thread.') - start_moderation_workers(opts.cluster_workers * 3) + num_moderators = opts.cluster_workers * 3 + start_moderation_workers(num_moderators) + logger.info(f'Started {num_moderators} moderation workers.') t = Thread(target=cache_stats) t.daemon = True t.start() - print('Started the stats cacher.') + logger.info('Started the stats cacher.') t = Thread(target=recent_prompters_thread) t.daemon = True t.start() - print('Started the recent proompters thread.') + logger.info('Started the recent proompters thread.') t = Thread(target=console_printer) t.daemon = True t.start() - print('Started the console printer.') + logger.info('Started the console logger.infoer.') t = Thread(target=cluster_worker) t.daemon = True t.start() - print('Started the cluster worker.') + logger.info('Started the cluster worker.') t = Thread(target=db_logger) t.daemon = True t.start() - print('Started background logger.') + logger.info('Started background logger.') diff --git a/requirements.txt b/requirements.txt index c1103d7..89f4be7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,9 +9,10 @@ simplejson~=3.19.1 websockets~=11.0.3 basicauth~=1.0.0 openai~=0.28.0 -urllib3~=2.0.4 flask-sock==0.6.0 gunicorn==21.2.0 redis==5.0.1 ujson==5.8.0 -vllm +vllm==0.2.1.post1 +gradio~=3.46.1 +coloredlogs~=15.0.1 \ No newline at end of file -- 2.34.1 From 177dabd20933825a1dd1c177d9d4ca262306835f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 23 Oct 2023 17:25:38 -0600 Subject: [PATCH 158/163] Give some time for the background threads to get themselves ready to go --- daemon.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/daemon.py b/daemon.py index 0a5be10..400efc6 100644 --- a/daemon.py +++ b/daemon.py @@ -55,6 +55,9 @@ if __name__ == "__main__": start_background() + # Give some time for the background threads to get themselves ready to go. + time.sleep(2) + redis.set('daemon_started', 1) logger.info('== Daemon Setup Complete ==') -- 2.34.1 From 96ba48affc6d6e1858459efd225becfa3599cf7f Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 23 Oct 2023 17:26:15 -0600 Subject: [PATCH 159/163] make sure to regen stats on startup --- daemon.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daemon.py b/daemon.py index 400efc6..69e8532 100644 --- a/daemon.py +++ b/daemon.py @@ -51,7 +51,7 @@ if __name__ == "__main__": cluster_config.load(parse_backends(config)) logger.info('Loading backend stats...') - generate_stats() + generate_stats(regen=True) start_background() -- 2.34.1 From b4e01e129de219b09481b44ba1ebb7d5aae74b05 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 23 Oct 2023 17:28:59 -0600 Subject: [PATCH 160/163] fix when all offline --- server.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/server.py b/server.py index 37db35b..f65cf79 100644 --- a/server.py +++ b/server.py @@ -120,13 +120,12 @@ def home(): else: default_model_info = { 'model': 'OFFLINE', - 'processing': 0, - 'queued': 0, - + 'processing': '-', + 'queued': '-', + 'context_size': '-', } default_estimated_wait_sec = 'OFFLINE' - if len(config['analytics_tracking_code']): analytics_tracking_code = f"" else: -- 2.34.1 From 563630547ab57d6c339e6abe021ba5c6e82ce993 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Mon, 23 Oct 2023 17:32:33 -0600 Subject: [PATCH 161/163] add robots.txt --- llm_server/workers/inferencer.py | 2 +- server.py | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/llm_server/workers/inferencer.py b/llm_server/workers/inferencer.py index c9425b4..21e45d0 100644 --- a/llm_server/workers/inferencer.py +++ b/llm_server/workers/inferencer.py @@ -115,7 +115,7 @@ def worker(backend_url): if not selected_model: selected_model = backend_info['model'] - logger.debug(f"Starting using {backend_url} and {selected_model}. Online: {backend_info['online']}") + logger.debug(f"Starting using {backend_url} and {selected_model}. Online: {backend_info['online']}. Streaming: {do_stream}") try: stream_redis.delete(get_stream_name(worker_id)) # clean up any old streams diff --git a/server.py b/server.py index f65cf79..9603350 100644 --- a/server.py +++ b/server.py @@ -10,7 +10,7 @@ import sys from pathlib import Path import simplejson as json -from flask import Flask, jsonify, render_template, request +from flask import Flask, jsonify, render_template, request, Response import config from llm_server import opts @@ -168,6 +168,17 @@ def home(): ) +@app.route('/robots.txt') +def robots(): + # TODO: have config value to deny all + # TODO: https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt + t = """User-agent: * +Allow: /""" + r = Response(t) + r.headers['Content-Type'] = 'text/plain' + return r + + @app.route('/') @app.route('//') def fallback(first=None, rest=None): -- 2.34.1 From 28c250385df691e8fc610e864434dd71c047be21 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 27 Oct 2023 19:00:49 -0600 Subject: [PATCH 162/163] add todo --- server.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server.py b/server.py index 9603350..aa8ef1a 100644 --- a/server.py +++ b/server.py @@ -34,12 +34,11 @@ from llm_server.sock import init_wssocket # TODO: return an `error: True`, error code, and error message rather than just a formatted message # TODO: what happens when all backends are offline? What about the "online" key in the stats page? # TODO: redis SCAN vs KEYS?? -# TODO: implement blind RRD controlled via header and only used when there is a queue on the primary backend(s) # TODO: is frequency penalty the same as ooba repetition penalty??? # TODO: make sure openai_moderation_enabled works on websockets, completions, and chat completions -# TODO: if a backend is at its limit of concurrent requests, choose a different one # Lower priority +# TODO: if a backend is at its limit of concurrent requests, choose a different one # TODO: make error messages consitient # TODO: support logit_bias on OpenAI and Ooba endpoints. # TODO: add a way to cancel VLLM gens. Maybe use websockets? -- 2.34.1 From ee44371fdfdf22c17eaed28f64bc9ffec5c4dc43 Mon Sep 17 00:00:00 2001 From: Cyberes Date: Fri, 27 Oct 2023 19:05:27 -0600 Subject: [PATCH 163/163] Merge branch 'master' into cluster --- other/vllm/Docker/DOCKER.md | 15 ----- other/vllm/Docker/Dockerfile | 81 ++++++++------------------- other/vllm/Docker/Dockerfile.base | 43 ++++++++++++++ other/vllm/Docker/README.md | 47 ++++++++++++++++ other/vllm/Docker/build-docker.sh | 7 +++ other/vllm/Docker/idle.ipynb | 40 +++++++++++++ other/vllm/Docker/init-container.sh | 22 ++++++++ other/vllm/Docker/start-container.sh | 13 +---- other/vllm/Docker/start-vllm.sh | 5 -- other/vllm/Docker/supervisord.conf | 37 +++++++++++- other/vllm/Docker/update-container.sh | 11 ++++ 11 files changed, 228 insertions(+), 93 deletions(-) delete mode 100644 other/vllm/Docker/DOCKER.md create mode 100644 other/vllm/Docker/Dockerfile.base create mode 100644 other/vllm/Docker/README.md create mode 100644 other/vllm/Docker/build-docker.sh create mode 100644 other/vllm/Docker/idle.ipynb create mode 100644 other/vllm/Docker/init-container.sh create mode 100755 other/vllm/Docker/update-container.sh diff --git a/other/vllm/Docker/DOCKER.md b/other/vllm/Docker/DOCKER.md deleted file mode 100644 index 6abf6bf..0000000 --- a/other/vllm/Docker/DOCKER.md +++ /dev/null @@ -1,15 +0,0 @@ -**A Docker container for running VLLM on Paperspace Gradient notebooks.** - -1. Run `jupyter server --generate-config` and `jupyter server password` on your local machine, then copy Jupyter's config directory to `./jupyter` -2. Place your Rathole client config at `./rathole-client.toml` -3. `docker build . -t "paperspace-vllm"` - -To test on your local machine, run this command: - -```bash -docker run --shm-size 14g --gpus all \ - -v /storage/models/awq/MythoMax-L2-13B-AWQ:/models/MythoMax-L2-13B-AWQ \ - -p 7000:7000 -p 8888:8888 \ - -e API_SERVER_ARGS="--model /models/MythoMax-L2-13B-AWQ --quantization awq --max-num-batched-tokens 99999 --gpu-memory-utilization 1" \ - vllm-cloud -``` \ No newline at end of file diff --git a/other/vllm/Docker/Dockerfile b/other/vllm/Docker/Dockerfile index d3c02e8..7ebe7b0 100644 --- a/other/vllm/Docker/Dockerfile +++ b/other/vllm/Docker/Dockerfile @@ -1,87 +1,50 @@ -FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as build - -RUN apt-get update && \ - apt-get install -y git python3-pip python3-venv wget unzip && \ - rm -rf /var/lib/apt/lists/* -RUN pip3 install --upgrade pip setuptools wheel - -RUN git clone https://git.evulid.cc/cyberes/local-llm-server.git /local-llm-server - -WORKDIR /local-llm-server - -RUN python3 -m venv /venv -RUN /venv/bin/pip install git+https://github.com/vllm-project/vllm - -RUN python3 -m venv /jupyterlab -RUN /jupyterlab/bin/pip install jupyterlab -RUN /jupyterlab/bin/jupyter labextension disable "@jupyterlab/apputils-extension:announcements" - -RUN mkdir -p /app -RUN wget https://github.com/rapiz1/rathole/releases/download/v0.4.8/rathole-x86_64-unknown-linux-gnu.zip -O /tmp/rathole.zip -RUN unzip -j /tmp/rathole.zip -d /tmp -RUN rm /tmp/rathole.zip -RUN cp /tmp/rathole /app - -# The local local-llm-server repo may be cached, so we will fetch and reset to the remote every time. -# Also, make sure there weren't any pip deps added. -ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache -RUN git fetch; git reset --hard origin/master -RUN /venv/bin/pip install -r requirements.txt - -FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as runtime - -RUN apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/* +FROM cyberes/vllm-paperspace-base as runtime RUN useradd -ms /bin/bash apiserver RUN usermod -s /bin/bash root +# Required packages RUN apt-get update && \ - apt-get install -y python3 python3-pip wget aria2 git-lfs git openssh-server openssh-client nano tmux file && \ + apt-get install -y python3 python3-pip supervisor && \ + rm -rf /var/lib/apt/lists/* +RUN pip3 install --upgrade pip setuptools wheel + +# Useful Python packages +RUN pip3 install glances + +# Useful tools +RUN apt-get update && \ + apt-get install -y wget aria2 git-lfs git openssh-server openssh-client nano tmux file && \ rm -rf /var/lib/apt/lists/* -RUN pip3 install --upgrade pip setuptools wheel -RUN pip3 install glances +# Update the git repo +RUN cd /local-llm-server && git reset --hard && git pull # Enable root SSH login RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config - # Disable password SSH login RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config - -# Create the necessary directory for SSH +# Create the necessary directory for sshd RUN mkdir /var/run/sshd -ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache - -COPY --from=build /local-llm-server /local-llm-server -COPY --from=build /venv /venv -COPY --from=build /app /app -COPY --from=build /jupyterlab /jupyterlab - -RUN cp /local-llm-server/other/vllm/Docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -RUN cp /local-llm-server/other/vllm/Docker/start-vllm.sh /app/start-vllm.sh -RUN cp /local-llm-server/other/vllm/Docker/start-container.sh /app/start.sh - -# Copy your secrets in -# COPY ./jupyter /app/jupyter +COPY supervisord.conf /etc/supervisor/supervisord.conf +COPY start-vllm.sh /app/start-vllm.sh +COPY init-container.sh /app/init.sh +COPY start-container.sh /app/start.sh RUN mkdir -p /var/log/app/ RUN chown -R apiserver:apiserver /local-llm-server && \ chown -R apiserver:apiserver /app && \ chown -R apiserver:apiserver /var/log/app/ +RUN git config --global --add safe.directory /local-llm-server +RUN chmod +x /app/init.sh RUN chmod +x /app/start.sh ENV SHELL="/bin/bash" -# SSH -EXPOSE 22 - -# VLLM -EXPOSE 7000 - -# Jupyter +# Expose Jupyter. We don't need to expose VLLM or SSH since rathole will tunnel those. EXPOSE 8888 CMD /app/start.sh diff --git a/other/vllm/Docker/Dockerfile.base b/other/vllm/Docker/Dockerfile.base new file mode 100644 index 0000000..bcd4d6f --- /dev/null +++ b/other/vllm/Docker/Dockerfile.base @@ -0,0 +1,43 @@ +# This container builds and assembles the Python parts of the Docker container. +# It is used as the base for the resulting container, which avoids having to re-push +# the large PyTorch parts every time the application is rebuilt. + +FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as build + +RUN apt-get update && \ + apt-get install -y git python3-pip python3-venv wget unzip && \ + rm -rf /var/lib/apt/lists/* +RUN pip install --upgrade pip setuptools wheel + +RUN git clone https://git.evulid.cc/cyberes/local-llm-server.git /local-llm-server + +RUN python3 -m venv /jupyterlab +RUN /jupyterlab/bin/pip install jupyterlab +RUN /jupyterlab/bin/jupyter labextension disable "@jupyterlab/apputils-extension:announcements" + +RUN mkdir -p /app +RUN wget https://github.com/rapiz1/rathole/releases/download/v0.4.8/rathole-x86_64-unknown-linux-gnu.zip -O /tmp/rathole.zip +RUN unzip -j /tmp/rathole.zip -d /tmp +RUN rm /tmp/rathole.zip +RUN cp /tmp/rathole /app + +RUN python3 -m venv /venv +RUN /venv/bin/pip3 install --upgrade pip setuptools wheel + +# Install PyTorch before installing VLLM to ensure we use the right version for our CUDA install. +RUN wget -q -O - https://raw.githubusercontent.com/vllm-project/vllm/main/requirements.txt | grep -E 'torch*' > /tmp/torch_version +RUN /venv/bin/pip3 install "$(cat /tmp/torch_version)" --index-url https://download.pytorch.org/whl/cu118 + +# WORKDIR /local-llm-server + +# Don't build VLLM because we don't do that on the inference server. Just install from pip. +# RUN /venv/bin/pip install git+https://github.com/vllm-project/vllm + +RUN /venv/bin/pip install vllm + +FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as base + +COPY --from=build /local-llm-server /local-llm-server +COPY --from=build /venv /venv +COPY --from=build /app /app +COPY --from=build /jupyterlab /jupyterlab diff --git a/other/vllm/Docker/README.md b/other/vllm/Docker/README.md new file mode 100644 index 0000000..97faf32 --- /dev/null +++ b/other/vllm/Docker/README.md @@ -0,0 +1,47 @@ +**A Docker container for running VLLM on Paperspace Gradient notebooks.** + +### Running + +1. In Paperspace, create a new notebook. +2. Click `Start from Scratch`. +3. Select your GPU and set the auto-shutdown timeout to 6 hours. +4. Click the `View Advanced Options` button at the bottom of the page. Enter these details in the form that appears: + - Container Name: `cyberes/vllm-paperspace:latest` + - Container Command: `/app/start.sh` +5. Start the notebook. It may take up to five minutes for them to pull and start the custom image. +6. Once the container is started, open the log viewer by clicking the icon in the bottom left of the screen. You should see errors from rathole and VLLM as a result of the blank config files. The container will create a new directory in your mounted + storage: `/storage/vllm/`. +7. Enter your rathole client config in `/storage/vllm/rathole-client.toml`. If you need a visual text editor, first link the directory back to the Jupyter home: `ln -s /storage/vllm /notebooks` +8. Restart rathole with `supervisorctl restart rathole` and then view the log: `tail -f /var/log/app/rathole.log`. If you see lines that start with `INFO` and end with `Control channel established`, rathole has connected and is working. Error mesasges will begin + with `ERROR`. +9. Download an AWQ quantization from [TheBloke](https://huggingface.co/TheBloke) to `/storage/vllm/models/`. +10. Enter your VLLM commandline args in `/storage/vllm/cmd.txt`. You need to set `--model` to the path of the model you want to load. +11. Restart VLLM with `supervisorctl restart vllm` and then view the log: `tail -f /var/log/app/vllm.log`. It may take up to three minutes to load. When you see the line: + ``` + INFO: Uvicorn running on http://0.0.0.0:7000 (Press CTRL+C to quit) + ``` +       VLLM is running and ready for queries. + +12. In `/notebooks` (the home directory of Jupyter), the notebook `idle.ipynb` will automatically be created. Run this notebook so Paperspace does not shut down your machine due to "inactivity". You **must** keep the running notebook open in a + browser tab. + +### Building + +You **must** have a GPU attached to your system when building the container (required for building VLLM). + +1. Install the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html) and CUDA 11.8. +2. `bash build-docker.sh` + +To run the container on your local machine: + +```bash +sudo docker run -it --shm-size 14g --gpus all -v /home/user/testing123/notebooks:/notebooks -v /home/user/testing123/storage:/storage -p 8888:8888 cyberes/vllm-paperspace:latest +``` + +You will need to create a directory to mount inside the container (for example: `/home/user/testing123/`). Within this should be the folder `models` that holds the model to load, `rathole-client.toml`, and `cmd.txt`. + +If you need to debug something, you can start a shell inside the container: + +```bash +sudo docker run -it --shm-size 14g --gpus all -v /home/user/testing123/notebooks:/notebooks -v /home/user/testing123/storage:/storage -p 8888:8888 --entrypoint bash cyberes/vllm-paperspace:latest +``` diff --git a/other/vllm/Docker/build-docker.sh b/other/vllm/Docker/build-docker.sh new file mode 100644 index 0000000..f95ad4f --- /dev/null +++ b/other/vllm/Docker/build-docker.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# Build and push the container. + +git pull || exit +sudo docker build . -f Dockerfile.base -t cyberes/vllm-paperspace-base --no-cache && sudo docker push cyberes/vllm-paperspace-base:latest || exit +sudo docker build . -t cyberes/vllm-paperspace && sudo docker push cyberes/vllm-paperspace:latest diff --git a/other/vllm/Docker/idle.ipynb b/other/vllm/Docker/idle.ipynb new file mode 100644 index 0000000..057e227 --- /dev/null +++ b/other/vllm/Docker/idle.ipynb @@ -0,0 +1,40 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "49ae6555-572b-4463-ba01-cc4331932a6c", + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "i = 0\n", + "while True:\n", + " print(i)\n", + " i += 1\n", + " time.sleep(1)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/other/vllm/Docker/init-container.sh b/other/vllm/Docker/init-container.sh new file mode 100644 index 0000000..111646c --- /dev/null +++ b/other/vllm/Docker/init-container.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +# Create the required directories and files. +echo "SETTING UP FILE SYSTEM..." +mkdir -p /storage/vllm/ +chown -R apiserver:apiserver /storage/vllm +touch /storage/vllm/cmd.txt +touch /storage/vllm/rathole-client.toml + +# The user can store SSH auth and authorized_keys to streamline SSH login. +if [ -f /storage/vllm/ssh ]; then + cp -r /storage/vllm/ssh /root/.ssh + echo "Copied ssh from /storage" +fi + +# If the user has not created the VLLM commandline arg file, create the default. +if [ ! -f /storage/vllm/cmd.txt ]; then + echo "--max-num-batched-tokens 4098 --quantization awq --model /storage/vllm/models/model-path" >/storage/vllm/cmd.txt +fi + +# Copy the idling notebook to storage. This will create a blank notebook every time the container is started. +cp /local-llm-server/other/vllm/Docker/idle.ipynb /notebooks/idle.ipynb diff --git a/other/vllm/Docker/start-container.sh b/other/vllm/Docker/start-container.sh index 0b98702..05587a1 100644 --- a/other/vllm/Docker/start-container.sh +++ b/other/vllm/Docker/start-container.sh @@ -1,13 +1,4 @@ #!/bin/bash -mkdir -p /storage/vllm/ -chown -R apiserver:apiserver /storage/vllm -touch /storage/vllm/cmd.txt -touch /storage/vllm/rathole-client.toml - -if [ -f /storage/vllm/ssh ]; then - cp -r /storage/vllm/ssh /root/.ssh - echo "Copied ssh from /storage" -fi - -/usr/bin/supervisord +# Start the services and launch the container. +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/other/vllm/Docker/start-vllm.sh b/other/vllm/Docker/start-vllm.sh index 906bc30..209e90a 100644 --- a/other/vllm/Docker/start-vllm.sh +++ b/other/vllm/Docker/start-vllm.sh @@ -6,9 +6,4 @@ for pid in $vllm_pid; do kill -9 $pid done -cd /local-llm-server -git fetch -git reset --hard origin/master -/venv/bin/pip install -r requirements.txt - /venv/bin/python /local-llm-server/other/vllm/vllm_api_server.py --host 0.0.0.0 --port 7000 --max-log-len 100 $(cat /storage/vllm/cmd.txt) diff --git a/other/vllm/Docker/supervisord.conf b/other/vllm/Docker/supervisord.conf index 9361bdb..800cb27 100644 --- a/other/vllm/Docker/supervisord.conf +++ b/other/vllm/Docker/supervisord.conf @@ -1,5 +1,25 @@ [supervisord] -nodaemon=true +nodaemon = true +user=root +pidfile = /var/run/supervisord.pid +logfile = /var/log/app/supervisord.log +directory = /tmp + +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0770 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + +[program:startup] +command=/app/init.sh +autostart=true +autorestart=false +startsecs=0 [program:vllm] command=/bin/bash -c 'bash /app/start-vllm.sh 2>&1 | tee -a /var/log/app/vllm.log' @@ -24,9 +44,20 @@ user=apiserver environment=HOME="/home/apiserver",USER="apiserver" [program:jupyter] -command=/jupyterlab/bin/jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True +command=/jupyterlab/bin/jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True --notebook-dir /notebooks environment=SHELL="/bin/bash" -; JUPYTER_CONFIG_DIR="/app/jupyter" +autostart=true +autorestart=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/fd/2 +stderr_logfile_maxbytes=0 [program:ssh] command=/usr/sbin/sshd -D +autostart=true +autorestart=true +stdout_logfile=/dev/fd/1 +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/fd/2 +stderr_logfile_maxbytes=0 diff --git a/other/vllm/Docker/update-container.sh b/other/vllm/Docker/update-container.sh new file mode 100755 index 0000000..d44d6d9 --- /dev/null +++ b/other/vllm/Docker/update-container.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +# Run this script to update the container. +# Will restart VLLM as well. + +cd /local-llm-server || exit + +git fetch +git reset --hard origin/master + +supervisorctl restart vllm -- 2.34.1