fix server exception

llm_server/cluster/backend.py

@@ -98,12 +98,10 @@ def get_model_choices(regen: bool = False):
             'queued': proompters_in_queue,
             'processing': active_gen_workers,
             'avg_generation_time': average_generation_elapsed_sec,
-            'concurrent_gens': concurrent_gens
+            'concurrent_gens': concurrent_gens,
+            'context_size': min(context_size) if len(context_size) else None
         }
 
-        if len(context_size):
-            model_choices[model]['context_size'] = min(context_size)
-
     # Python wants to sort lowercase vs. uppercase letters differently.
     model_choices = dict(sorted(model_choices.items(), key=lambda item: item[0].upper()))
 

llm_server/pre_fork.py

@@ -1,9 +1,11 @@
 import sys
 
 from llm_server.custom_redis import redis
+from llm_server.logging import create_logger
 
 
 def server_startup(s):
     if not redis.get('daemon_started', dtype=bool):
-        print('Could not find the key daemon_started in Redis. Did you forget to start the daemon process?')
+        logger = create_logger('gunicorn')
+        logger.fatal('Could not find the key daemon_started in Redis. Did you forget to start the daemon process?')
         sys.exit(1)

other/gunicorn.py

@@ -2,6 +2,7 @@
 This file is used to run certain tasks when the HTTP server starts.
 It's located here so it doesn't get imported with daemon.py
 """
+from llm_server.logging import create_logger
 
 try:
     import gevent.monkey
@@ -15,4 +16,5 @@ from llm_server.pre_fork import server_startup
 
 def on_starting(s):
     server_startup(s)
-    print('Startup complete!')
+    logger = create_logger('gunicorn')
+    logger.info('Startup complete!')

other/vllm/Docker/Dockerfile

@@ -23,7 +23,7 @@ RUN cd /local-llm-server && git reset --hard && git pull
 # Enable root SSH login
 RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
 # Disable password SSH login
-RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config
+RUN sed -i 's/#PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
 # Create the necessary directory for sshd
 RUN mkdir /var/run/sshd
 

server.py

@@ -125,6 +125,11 @@ def home():
         }
         default_estimated_wait_sec = 'OFFLINE'
 
+    if default_model_info['context_size'] is None:
+        # Sometimes a model doesn't provide the correct config, so the context size is set
+        # to None by the daemon.
+        default_model_info['context_size'] = '-'
+
     if len(config['analytics_tracking_code']):
         analytics_tracking_code = f"<script>\n{config['analytics_tracking_code']}\n</script>"
     else: