diff --git a/llm_server/pre_fork.py b/llm_server/pre_fork.py index 8d6030e..60e96a1 100644 --- a/llm_server/pre_fork.py +++ b/llm_server/pre_fork.py @@ -1,3 +1,4 @@ +import time import sys from llm_server.custom_redis import redis @@ -5,7 +6,7 @@ from llm_server.logging import create_logger def server_startup(s): - if not redis.get('daemon_started', dtype=bool): - logger = create_logger('gunicorn') - logger.fatal('Could not find the key daemon_started in Redis. Did you forget to start the daemon process?') - sys.exit(1) + logger = create_logger('gunicorn') + while not redis.get('daemon_started', dtype=bool): + logger.warning('Could not find the key daemon_started in Redis. Did you forget to start the daemon process?') + time.sleep(10) diff --git a/other/local-llm-daemon.service b/other/local-llm-daemon.service new file mode 100644 index 0000000..97e5b1d --- /dev/null +++ b/other/local-llm-daemon.service @@ -0,0 +1,15 @@ +[Unit] +Description=Local LLM Proxy Daemon +Wants=basic.target +After=basic.target network.target + +[Service] +User=server +Group=server +ExecStart=/srv/server/local-llm-server/venv/bin/python /srv/server/local-llm-server/daemon.py +Restart=always +RestartSec=2 +SyslogIdentifier=local-llm-daemon + +[Install] +WantedBy=multi-user.target diff --git a/other/local-llm.service b/other/local-llm-server.service similarity index 63% rename from other/local-llm.service rename to other/local-llm-server.service index b20e19b..2a18b77 100644 --- a/other/local-llm.service +++ b/other/local-llm-server.service @@ -1,7 +1,8 @@ [Unit] -Description=Local LLM Proxy +Description=Local LLM Proxy Server Wants=basic.target -After=basic.target network.target +After=basic.target network.target local-llm-daemon.service +Requires=local-llm-daemon.service [Service] User=server @@ -9,9 +10,10 @@ Group=server WorkingDirectory=/srv/server/local-llm-server # Need a lot of workers since we have long-running requests # Takes about 3.5G memory -ExecStart=/srv/server/local-llm-server/venv/bin/gunicorn --workers 20 --bind 0.0.0.0:5000 server:app --timeout 60 --worker-class gevent +ExecStart=/srv/server/local-llm-server/venv/bin/gunicorn --workers 20 --bind 0.0.0.0:5000 server:app --timeout 60 --worker-class gevent -c /srv/server/local-llm-server/other/server.conf Restart=always RestartSec=2 +SyslogIdentifier=local-llm-server [Install] WantedBy=multi-user.target diff --git a/other/server.conf b/other/server.conf new file mode 100644 index 0000000..5e30e7a --- /dev/null +++ b/other/server.conf @@ -0,0 +1,2 @@ +accesslog="-" +errorlog="-"