diff --git a/other/vllm/Docker/DOCKER.md b/other/vllm/Docker/DOCKER.md index cadb8b9..fcb3529 100644 --- a/other/vllm/Docker/DOCKER.md +++ b/other/vllm/Docker/DOCKER.md @@ -1 +1,15 @@ -`docker run --shm-size 14g --gpus all -v /storage/models/awq/MythoMax-L2-13B-AWQ:/models/MythoMax-L2-13B-AWQ -e ENV_API_SERVER_ARGS="--model /models/MythoMax-L2-13B-AWQ --quantization awq --host 0.0.0.0 --port 7000 --max-num-batched-tokens 8192 --gpu-memory-utilization 1" -d cyberes_vllm_cloud` \ No newline at end of file +**A Docker container for running VLLM on Paperspace Gradient notebooks.** + +1. Run `jupyter server --generate-config` and `jupyter server password` on your local machine, then copy Jupyter's config directory to `./jupyter` +2. Place your Rathole client config at `./rathole-client.toml` +3. `docker build . -t "paperspace-vllm"` + +To test on your local machine, run this command: + +```bash +docker run --shm-size 14g --gpus all \ + -v /storage/models/awq/MythoMax-L2-13B-AWQ:/models/MythoMax-L2-13B-AWQ \ + -p 7000:7000 -p 8888:8888 \ + -e API_SERVER_ARGS="--model /models/MythoMax-L2-13B-AWQ --quantization awq --max-num-batched-tokens 8192 --gpu-memory-utilization 1" \ + vllm-cloud +``` \ No newline at end of file diff --git a/other/vllm/Docker/Dockerfile b/other/vllm/Docker/Dockerfile index 7755917..176f292 100644 --- a/other/vllm/Docker/Dockerfile +++ b/other/vllm/Docker/Dockerfile @@ -1,45 +1,75 @@ FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as build RUN apt-get update && apt-get install -y git python3-pip python3-venv wget unzip && rm -rf /var/lib/apt/lists/* +RUN pip3 install --upgrade pip setuptools wheel RUN git clone https://git.evulid.cc/cyberes/local-llm-server.git /local-llm-server WORKDIR /local-llm-server RUN python3 -m venv /venv -RUN /venv/bin/pip install -r requirements.txt RUN /venv/bin/pip install git+https://github.com/vllm-project/vllm -ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache - -RUN git fetch; git reset --hard origin/master +RUN python3 -m venv /jupyterlab +RUN /jupyterlab/bin/pip install jupyterlab +RUN /jupyterlab/bin/jupyter labextension disable "@jupyterlab/apputils-extension:announcements" RUN mkdir -p /app -COPY ./rathole-client.toml /app/client.toml RUN wget https://github.com/rapiz1/rathole/releases/download/v0.4.8/rathole-x86_64-unknown-linux-gnu.zip -O /tmp/rathole.zip RUN unzip -j /tmp/rathole.zip -d /tmp RUN rm /tmp/rathole.zip RUN cp /tmp/rathole /app +# The local local-llm-server repo may be cached, so we will fetch and reset to the remote every time. +# Also, make sure there weren't any pip deps added. +ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache +RUN git fetch; git reset --hard origin/master +RUN /venv/bin/pip install -r requirements.txt + FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as runtime RUN apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/* +RUN useradd -ms /bin/bash apiserver +RUN usermod -s /bin/bash root + +RUN apt-get update && apt-get install -y python3 python3-pip wget aria2 git-lfs git openssh-server openssh-client +RUN pip3 install --upgrade pip setuptools wheel +RUN pip3 install glances + +# Enable root SSH login +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config + +# Disable password SSH login +RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config + +# Create the necessary directory for SSH +RUN mkdir /var/run/sshd + +ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache + COPY --from=build /local-llm-server /local-llm-server COPY --from=build /venv /venv COPY --from=build /app /app +COPY --from=build /jupyterlab /jupyterlab RUN cp /local-llm-server/other/vllm/Docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf -RUN useradd -ms /bin/bash apiserver -RUN chown -R apiserver:apiserver /local-llm-server +COPY ./rathole-client.toml /app/client.toml +COPY ./jupyter /app/jupyter +COPY ./ssh /root/.ssh -RUN apt-get update && apt-get install -y python3 python3-pip wget aria2 -RUN pip3 install --upgrade pip setuptools wheel -RUN pip install jupyterlab +RUN chown -R apiserver:apiserver /local-llm-server && \ + chown -R apiserver:apiserver /app +ENV SHELL="/bin/bash" + +# SSH +EXPOSE 22 + +# VLLM EXPOSE 7000 -EXPOSE 8888 # jupyter -CMD ["bash", "-c", "source /etc/bash.bashrc && /usr/bin/supervisord"] -# To test your config, uncomment this and comment out the other CMD. -#CMD ["/app/rathole", "-c", "/app/client.toml"] \ No newline at end of file +# Jupyter +EXPOSE 8888 + +CMD /usr/bin/supervisord \ No newline at end of file diff --git a/other/vllm/Docker/supervisord.conf b/other/vllm/Docker/supervisord.conf index 05a62ae..182c1cb 100644 --- a/other/vllm/Docker/supervisord.conf +++ b/other/vllm/Docker/supervisord.conf @@ -24,4 +24,8 @@ user=apiserver environment=HOME="/home/apiserver",USER="apiserver" [program:jupyter] -command=bash -c "source /etc/bash.bashrcPIP_DISABLE_PIP_VERSION_CHECK=1 jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True" \ No newline at end of file +command=/jupyterlab/bin/jupyter lab --allow-root --ip=0.0.0.0 --no-browser --ServerApp.trust_xheaders=True --ServerApp.disable_check_xsrf=False --ServerApp.allow_remote_access=True --ServerApp.allow_origin='*' --ServerApp.allow_credentials=True +environment=JUPYTER_CONFIG_DIR="/app/jupyter",SHELL="/bin/bash" + +[program:ssh] +command=/usr/sbin/sshd -D \ No newline at end of file