23 lines
839 B
Bash
23 lines
839 B
Bash
#!/bin/bash
|
|
|
|
# Create the required directories and files.
|
|
echo "SETTING UP FILE SYSTEM..."
|
|
mkdir -p /storage/vllm/
|
|
chown -R apiserver:apiserver /storage/vllm
|
|
touch /storage/vllm/cmd.txt
|
|
touch /storage/vllm/rathole-client.toml
|
|
|
|
# The user can store SSH auth and authorized_keys to streamline SSH login.
|
|
if [ -f /storage/vllm/ssh ]; then
|
|
cp -r /storage/vllm/ssh /root/.ssh
|
|
echo "Copied ssh from /storage"
|
|
fi
|
|
|
|
# If the user has not created the VLLM commandline arg file, create the default.
|
|
if [ ! -f /storage/vllm/cmd.txt ]; then
|
|
echo "--max-num-batched-tokens 4098 --quantization awq --model /storage/vllm/models/model-path" >/storage/vllm/cmd.txt
|
|
fi
|
|
|
|
# Copy the idling notebook to storage. This will create a blank notebook every time the container is started.
|
|
cp /local-llm-server/other/vllm/Docker/idle.ipynb /notebooks/idle.ipynb
|