modify dockerfile for paperspace

other/vllm/Docker/DOCKER.md

@@ -1 +1,15 @@
-`docker run --shm-size 14g --gpus all -v /storage/models/awq/MythoMax-L2-13B-AWQ:/models/MythoMax-L2-13B-AWQ -e ENV_API_SERVER_ARGS="--model /models/MythoMax-L2-13B-AWQ --quantization awq --host 0.0.0.0 --port 7000 --max-num-batched-tokens 8192 --gpu-memory-utilization 1" -d cyberes_vllm_cloud`
+**A Docker container for running VLLM on Paperspace Gradient notebooks.**
+
+1. Run `jupyter server --generate-config` and `jupyter server password` on your local machine, then copy Jupyter's config directory to `./jupyter`
+2. Place your Rathole client config at `./rathole-client.toml`
+3. `docker build . -t "paperspace-vllm"`
+
+To test on your local machine, run this command:
+
+```bash
+docker run --shm-size 14g --gpus all \
+  -v /storage/models/awq/MythoMax-L2-13B-AWQ:/models/MythoMax-L2-13B-AWQ \
+  -p 7000:7000 -p 8888:8888 \
+  -e API_SERVER_ARGS="--model /models/MythoMax-L2-13B-AWQ --quantization awq --max-num-batched-tokens 8192 --gpu-memory-utilization 1" \
+  vllm-cloud
+```

other/vllm/Docker/Dockerfile

@@ -1,45 +1,75 @@
 FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as build
 
 RUN apt-get update && apt-get install -y git python3-pip python3-venv wget unzip && rm -rf /var/lib/apt/lists/*
+RUN pip3 install --upgrade pip setuptools wheel
 
 RUN git clone https://git.evulid.cc/cyberes/local-llm-server.git /local-llm-server
 
 WORKDIR /local-llm-server
 
 RUN python3 -m venv /venv
-RUN /venv/bin/pip install -r requirements.txt
 RUN /venv/bin/pip install git+https://github.com/vllm-project/vllm
 
-ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache
+RUN python3 -m venv /jupyterlab
-
+RUN /jupyterlab/bin/pip install jupyterlab
-RUN git fetch; git reset --hard origin/master
+RUN /jupyterlab/bin/jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
 
 RUN mkdir -p /app
-COPY ./rathole-client.toml /app/client.toml
 RUN wget https://github.com/rapiz1/rathole/releases/download/v0.4.8/rathole-x86_64-unknown-linux-gnu.zip -O /tmp/rathole.zip
 RUN unzip -j /tmp/rathole.zip -d /tmp
 RUN rm /tmp/rathole.zip
 RUN cp /tmp/rathole /app
 
+# The local local-llm-server repo may be cached, so we will fetch and reset to the remote every time.
+# Also, make sure there weren't any pip deps added.
+ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache
+RUN git fetch; git reset --hard origin/master
+RUN /venv/bin/pip install -r requirements.txt
+
 FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as runtime
 
 RUN apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/*
 
+RUN useradd -ms /bin/bash apiserver
+RUN usermod -s /bin/bash root
+
+RUN apt-get update && apt-get install -y python3 python3-pip wget aria2 git-lfs git openssh-server openssh-client
+RUN pip3 install --upgrade pip setuptools wheel
+RUN pip3 install glances
+
+# Enable root SSH login
+RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+
+# Disable password SSH login
+RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config
+
+# Create the necessary directory for SSH
+RUN mkdir /var/run/sshd
+
+ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache
+
 COPY --from=build /local-llm-server /local-llm-server
 COPY --from=build /venv /venv
 COPY --from=build /app /app
+COPY --from=build /jupyterlab /jupyterlab
 RUN cp /local-llm-server/other/vllm/Docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
 
-RUN useradd -ms /bin/bash apiserver
+COPY ./rathole-client.toml /app/client.toml
-RUN chown -R apiserver:apiserver /local-llm-server
+COPY ./jupyter /app/jupyter
+COPY ./ssh /root/.ssh
 
-RUN apt-get update && apt-get install -y python3 python3-pip wget aria2
+RUN chown -R apiserver:apiserver /local-llm-server && \
-RUN pip3 install --upgrade pip setuptools wheel
+    chown -R apiserver:apiserver /app
-RUN pip install jupyterlab
 
+ENV SHELL="/bin/bash"
+
+# SSH
+EXPOSE 22
+
+# VLLM
 EXPOSE 7000
-EXPOSE 8888 # jupyter
-CMD ["bash", "-c", "source /etc/bash.bashrc && /usr/bin/supervisord"]
 
-# To test your config, uncomment this and comment out the other CMD.
+# Jupyter
-#CMD ["/app/rathole", "-c", "/app/client.toml"]
+EXPOSE 8888
+
+CMD /usr/bin/supervisord

other/vllm/Docker/supervisord.conf

@@ -24,4 +24,8 @@ user=apiserver
 environment=HOME="/home/apiserver",USER="apiserver"
 
 [program:jupyter]
-command=bash -c "source /etc/bash.bashrcPIP_DISABLE_PIP_VERSION_CHECK=1 jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True"
+command=/jupyterlab/bin/jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True
+environment=JUPYTER_CONFIG_DIR="/app/jupyter",SHELL="/bin/bash"
+
+[program:ssh]
+command=/usr/sbin/sshd -D