FROM nvidia/cuda:11.8.0-devel-ubuntu22.04 as build RUN apt-get update && \ apt-get install -y git python3-pip python3-venv wget unzip && \ rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade pip setuptools wheel RUN git clone https://git.evulid.cc/cyberes/local-llm-server.git /local-llm-server RUN python3 -m venv /jupyterlab RUN /jupyterlab/bin/pip install jupyterlab RUN /jupyterlab/bin/jupyter labextension disable "@jupyterlab/apputils-extension:announcements" RUN mkdir -p /app RUN wget https://github.com/rapiz1/rathole/releases/download/v0.4.8/rathole-x86_64-unknown-linux-gnu.zip -O /tmp/rathole.zip RUN unzip -j /tmp/rathole.zip -d /tmp RUN rm /tmp/rathole.zip RUN cp /tmp/rathole /app RUN python3 -m venv /venv RUN /venv/bin/pip install torch --index-url https://download.pytorch.org/whl/cu118 WORKDIR /local-llm-server # We don't need to rebuild VLLM every time we build the container. But if we need # to, uncomment the following line. # ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache RUN /venv/bin/pip install git+https://github.com/vllm-project/vllm FROM nvidia/cuda:11.8.0-base-ubuntu22.04 as runtime RUN apt-get update && apt-get install -y supervisor && rm -rf /var/lib/apt/lists/* RUN useradd -ms /bin/bash apiserver RUN usermod -s /bin/bash root RUN apt-get update && \ apt-get install -y python3 python3-pip wget aria2 git-lfs git openssh-server openssh-client nano tmux file && \ rm -rf /var/lib/apt/lists/* RUN pip3 install --upgrade pip setuptools wheel RUN pip3 install glances # Enable root SSH login RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config # Disable password SSH login RUN sed -i 's/#PasswordAuthentication yes/PasswordAuthentication no/' /etc/ssh/sshd_config # Create the necessary directory for SSH RUN mkdir /var/run/sshd ADD "https://www.random.org/cgi-bin/randbyte?nbytes=10&format=h" skipcache COPY --from=build /local-llm-server /local-llm-server COPY --from=build /venv /venv COPY --from=build /app /app COPY --from=build /jupyterlab /jupyterlab COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf COPY start-vllm.sh /app/start-vllm.sh COPY start-container.sh /app/start.sh RUN mkdir -p /var/log/app/ RUN chown -R apiserver:apiserver /local-llm-server && \ chown -R apiserver:apiserver /app && \ chown -R apiserver:apiserver /var/log/app/ RUN git config --global --add safe.directory /local-llm-server RUN chmod +x /app/start.sh ENV SHELL="/bin/bash" # SSH EXPOSE 22 # VLLM EXPOSE 7000 # Jupyter EXPOSE 8888 CMD /app/start.sh