26 lines
934 B
Bash
26 lines
934 B
Bash
|
#!/bin/bash
|
||
|
|
|
||
|
|
# Create the required directories and files.
|
||
|
|
echo "SETTING UP FILE SYSTEM..."
|
||
|
|
mkdir -p /storage/vllm/
|
||
|
|
chown -R apiserver:apiserver /storage/vllm
|
||
|
|
touch /storage/vllm/cmd.txt
|
||
|
|
touch /storage/vllm/rathole-client.toml
|
||
|
|
|
||
|
|
# The user can store SSH auth and authorized_keys to streamline SSH login.
|
||
|
|
if [ -f /storage/vllm/ssh ]; then
|
||
|
|
cp -r /storage/vllm/ssh /root/.ssh
|
||
|
|
echo "Copied ssh from /storage"
|
||
|
|
fi
|
||
|
|
|
||
|
|
# If the user has not created the VLLM commandline arg file, create the default.
|
||
|
|
if [ ! -f /storage/vllm/cmd.txt ]; then
|
||
|
|
echo "--max-num-batched-tokens 4098 --quantization awq --model /storage/vllm/models/model-path" >/storage/vllm/cmd.txt
|
||
|
|
fi
|
||
|
|
|
||
|
|
# Copy the idling notebook to storage. This will create a blank notebook every
|
||
|
|
# time the container is started.
|
||
|
|
cp /local-llm-server/other/vllm/Docker/idle.ipynb /notebooks/idle.ipynb
|
||
|
|
|
||
|
|
# cp /local-llm-server/other/vllm/Docker/update-container.sh /notebooks/update-container.sh
|