local-llm-server/other/vllm/Docker/start-container.sh

#!/bin/bash

echo "LAUNCHING CONTAINER..."

# Update the container repository and make sure pip dependancies are up to date.
cd /local-llm-server || exit
git fetch
git reset --hard origin/master
/venv/bin/pip install -r requirements.txt

mkdir -p /storage/vllm/
chown -R apiserver:apiserver /storage/vllm
touch /storage/vllm/cmd.txt
touch /storage/vllm/rathole-client.toml

if [ -f /storage/vllm/ssh ]; then
  cp -r /storage/vllm/ssh /root/.ssh
  echo "Copied ssh from /storage"
fi

if [ ! -f /storage/vllm/cmd.txt ]; then
  echo "--max-num-batched-tokens 4098 --quantization awq --model /storage/vllm/models/model-path" >/storage/vllm/cmd.txt
fi

cp /local-llm-server/other/vllm/Docker/idle.ipynb /notebooks/idle.ipynb

# Start the services
/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf