Add Gitlab CI and self-hosting instructions (khanon/oai-reverse-proxy!61)
This commit is contained in:
khanon
parent
81f1e2bc37
commit
f5e7195cc9
23
README.md
23
README.md
|
|
@ -6,10 +6,10 @@ Reverse proxy server for various LLM APIs.
|
|||
- [What is this?](#what-is-this)
|
||||
- [Features](#features)
|
||||
- [Usage Instructions](#usage-instructions)
|
||||
- [Self-hosting (locally or without Docker)](#self-hosting-locally-or-without-docker)
|
||||
- [Self hosting (with Docker)](#self-hosting-with-docker)
|
||||
- [Huggingface (not advised)](#huggingface-not-advised)
|
||||
- [Render (not advised)](#render-not-advised)
|
||||
- [Self-hosting](#self-hosting)
|
||||
- [Alternatives](#alternatives)
|
||||
- [Huggingface (outdated, not advised)](#huggingface-outdated-not-advised)
|
||||
- [Render (outdated, not advised)](#render-outdated-not-advised)
|
||||
- [Local Development](#local-development)
|
||||
|
||||
## What is this?
|
||||
|
|
@ -36,23 +36,18 @@ This project allows you to run a reverse proxy server for various LLM APIs.
|
|||
## Usage Instructions
|
||||
If you'd like to run your own instance of this server, you'll need to deploy it somewhere and configure it with your API keys. A few easy options are provided below, though you can also deploy it to any other service you'd like if you know what you're doing and the service supports Node.js.
|
||||
|
||||
### Self-hosting (locally or without Docker)
|
||||
Follow the "Local Development" instructions below to set up prerequisites and start the server. Then you can use a service like [ngrok](https://ngrok.com/) or [trycloudflare.com](https://trycloudflare.com/) to securely expose your server to the internet, or you can use a more traditional reverse proxy/WAF like [Cloudflare](https://www.cloudflare.com/) or [Nginx](https://www.nginx.com/).
|
||||
|
||||
**Ensure you set the `TRUSTED_PROXIES` environment variable according to your deployment.** Refer to [.env.example](./.env.example) and [config.ts](./src/config.ts) for more information.
|
||||
|
||||
### Self hosting (with Docker)
|
||||
If you have a Docker-capable VPS or server, use the Huggingface Dockerfile ([./docker/huggingface/Dockerfile](./docker/huggingface/Dockerfile)) to build an image and run it on your server.
|
||||
### Self-hosting
|
||||
[See here for instructions on how to self-host the application on your own VPS or local machine.](./docs/self-hosting.md)
|
||||
|
||||
**Ensure you set the `TRUSTED_PROXIES` environment variable according to your deployment.** Refer to [.env.example](./.env.example) and [config.ts](./src/config.ts) for more information.
|
||||
|
||||
### Alternatives
|
||||
Fiz and Sekrit are working on some alternative ways to deploy this conveniently. While I'm not directly involved in writing code or scripts for that project, I'm providing some advice and will include links to their work here when it's ready.
|
||||
Fiz and Sekrit are working on some alternative ways to deploy this conveniently. While I'm not involved in this effort beyond providing technical advice regarding my code, I'll link to their work here for convenience: [Sekrit's rentry](https://rentry.org/sekrit)
|
||||
|
||||
### Huggingface (not advised)
|
||||
### Huggingface (outdated, not advised)
|
||||
[See here for instructions on how to deploy to a Huggingface Space.](./docs/deploy-huggingface.md)
|
||||
|
||||
### Render (not advised)
|
||||
### Render (outdated, not advised)
|
||||
[See here for instructions on how to deploy to Render.com.](./docs/deploy-render.md)
|
||||
|
||||
## Local Development
|
||||
|
|
|
|||
|
|
@ -0,0 +1,22 @@
|
|||
stages:
|
||||
- build
|
||||
|
||||
build_image:
|
||||
stage: build
|
||||
image:
|
||||
name: gcr.io/kaniko-project/executor:debug
|
||||
entrypoint: [""]
|
||||
script:
|
||||
- |
|
||||
if [ "$CI_COMMIT_REF_NAME" = "main" ]; then
|
||||
TAG="latest"
|
||||
else
|
||||
TAG=$CI_COMMIT_REF_NAME
|
||||
fi
|
||||
- echo "Building image with tag $TAG"
|
||||
- BASE64_AUTH=$(echo -n "$DOCKER_HUB_USERNAME:$DOCKER_HUB_ACCESS_TOKEN" | base64)
|
||||
- echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"auth\":\"$BASE64_AUTH\"}}}" > /kaniko/.docker/config.json
|
||||
- /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/docker/ci/Dockerfile --destination docker.io/khanonci/oai-reverse-proxy:$CI_COMMIT_REF_NAME --build-arg CI_COMMIT_REF_NAME=$CI_COMMIT_REF_NAME --build-arg CI_COMMIT_SHA=$CI_COMMIT_SHA --build-arg CI_PROJECT_PATH=$CI_PROJECT_PATH
|
||||
only:
|
||||
- main
|
||||
- gitgud-ci
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
FROM node:18-bullseye-slim
|
||||
|
||||
WORKDIR /app
|
||||
COPY . .
|
||||
|
||||
RUN npm ci
|
||||
RUN npm run build
|
||||
RUN npm prune --production
|
||||
|
||||
EXPOSE 7860
|
||||
ENV PORT=7860
|
||||
ENV NODE_ENV=production
|
||||
|
||||
ARG CI_COMMIT_REF_NAME
|
||||
ARG CI_COMMIT_SHA
|
||||
ARG CI_PROJECT_PATH
|
||||
|
||||
ENV GITGUD_BRANCH=$CI_COMMIT_REF_NAME
|
||||
ENV GITGUD_COMMIT=$CI_COMMIT_SHA
|
||||
ENV GITGUD_PROJECT=$CI_PROJECT_PATH
|
||||
|
||||
CMD [ "npm", "start" ]
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
# Before running this, create a .env and greeting.md file.
|
||||
# Refer to .env.example for the required environment variables.
|
||||
# User-generated content is stored in the data directory.
|
||||
# When self-hosting, it's recommended to run this behind a reverse proxy like
|
||||
# nginx or Caddy to handle SSL/TLS and rate limiting. Refer to
|
||||
# docs/self-hosting.md for more information and an example nginx config.
|
||||
version: '3.8'
|
||||
services:
|
||||
oai-reverse-proxy:
|
||||
image: khanonci/oai-reverse-proxy:latest
|
||||
ports:
|
||||
- "127.0.0.1:7860:7860"
|
||||
env_file:
|
||||
- ./.env
|
||||
volumes:
|
||||
- ./greeting.md:/app/greeting.md
|
||||
- ./data:/app/data
|
||||
|
|
@ -1,5 +1,7 @@
|
|||
# Deploy to Huggingface Space
|
||||
|
||||
**⚠️ This method is no longer recommended. Please use the [self-hosting instructions](./self-hosting.md) instead.**
|
||||
|
||||
This repository can be deployed to a [Huggingface Space](https://huggingface.co/spaces). This is a free service that allows you to run a simple server in the cloud. You can use it to safely share your OpenAI API key with a friend.
|
||||
|
||||
### 1. Get an API key
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
# Deploy to Render.com
|
||||
|
||||
**⚠️ This method is no longer recommended. Please use the [self-hosting instructions](./self-hosting.md) instead.**
|
||||
|
||||
Render.com offers a free tier that includes 750 hours of compute time per month. This is enough to run a single proxy instance 24/7. Instances shut down after 15 minutes without traffic but start up again automatically when a request is received. You can use something like https://app.checklyhq.com/ to ping your proxy every 15 minutes to keep it alive.
|
||||
|
||||
### 1. Create account
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
# Quick self-hosting guide
|
||||
|
||||
Temporary guide for self-hosting. This will be improved in the future to provide more robust instructions and options. Provided commands are for Ubuntu.
|
||||
|
||||
This uses prebuilt Docker images for convenience. If you want to make adjustments to the code you can instead clone the repo and follow the Local Development guide in the [README](../README.md).
|
||||
|
||||
## Table of Contents
|
||||
- [Requirements](#requirements)
|
||||
- [Running the application](#running-the-application)
|
||||
- [Setting up a reverse proxy](#setting-up-a-reverse-proxy)
|
||||
- [trycloudflare](#trycloudflare)
|
||||
- [nginx](#nginx)
|
||||
- [Example basic nginx configuration (no SSL)](#example-basic-nginx-configuration-no-ssl)
|
||||
- [Example with Cloudflare SSL](#example-with-cloudflare-ssl)
|
||||
- [Updating/Restarting the application](#updatingrestarting-the-application)
|
||||
|
||||
## Requirements
|
||||
|
||||
- Docker
|
||||
- Docker Compose
|
||||
- A VPS with at least 512MB of RAM (1GB recommended)
|
||||
- A domain name
|
||||
|
||||
If you don't have a VPS and domain name you can use TryCloudflare to set up a temporary URL that you can share with others. See [trycloudflare](#trycloudflare) for more information.
|
||||
|
||||
## Running the application
|
||||
|
||||
- Install Docker and Docker Compose
|
||||
- Create a new directory for the application
|
||||
- This will contain your .env file, greeting file, and any user-generated files
|
||||
- Execute the following commands:
|
||||
- ```
|
||||
touch .env
|
||||
touch greeting.md
|
||||
echo "OPENAI_KEY=your-openai-key" >> .env
|
||||
curl https://gitgud.io/khanon/oai-reverse-proxy/-/raw/main/docker/docker-compose-selfhost.yml -o docker-compose.yml
|
||||
```
|
||||
- You can set further environment variables and keys in the `.env` file. See [.env.example](../.env.example) for a list of available options.
|
||||
- You can set a custom greeting in `greeting.md`. This will be displayed on the homepage.
|
||||
- Run `docker compose up -d`
|
||||
|
||||
You can check logs with `docker compose logs -n 100 -f`.
|
||||
|
||||
The provided docker-compose file listens on port 7860 but binds to localhost only. You should use a reverse proxy to expose the application to the internet as described in the next section.
|
||||
|
||||
## Setting up a reverse proxy
|
||||
|
||||
Rather than exposing the application directly to the internet, it is recommended to set up a reverse proxy. This will allow you to use HTTPS and add additional security measures.
|
||||
|
||||
### trycloudflare
|
||||
|
||||
This will give you a temporary (72 hours) URL that you can use to let others connect to your instance securely, without having to set up a reverse proxy. If you are running the server on your home network, this is probably the best option.
|
||||
- Install `cloudflared` following the instructions at [try.cloudflare.com](https://try.cloudflare.com/).
|
||||
- Run `cloudflared tunnel --url http://localhost:7860`
|
||||
- You will be given a temporary URL that you can share with others.
|
||||
|
||||
If you have a VPS, you should use a proper reverse proxy like nginx instead for a more permanent solution which will allow you to use your own domain name, handle SSL, and add additional security/anti-abuse measures.
|
||||
|
||||
### nginx
|
||||
|
||||
First, install nginx.
|
||||
- `sudo apt update && sudo apt install nginx`
|
||||
|
||||
#### Example basic nginx configuration (no SSL)
|
||||
|
||||
- `sudo nano /etc/nginx/sites-available/oai.conf`
|
||||
- ```
|
||||
server {
|
||||
listen 80;
|
||||
server_name example.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:7860;
|
||||
}
|
||||
}
|
||||
```
|
||||
- Replace `example.com` with your domain name.
|
||||
- Ctrl+X to exit, Y to save, Enter to confirm.
|
||||
- `sudo ln -s /etc/nginx/sites-available/oai.conf /etc/nginx/sites-enabled`
|
||||
- `sudo nginx -t`
|
||||
- This will check the configuration file for errors.
|
||||
- `sudo systemctl restart nginx`
|
||||
- This will restart nginx and apply the new configuration.
|
||||
|
||||
#### Example with Cloudflare SSL
|
||||
|
||||
This allows you to use a self-signed certificate on the server, and have Cloudflare handle client SSL. You need to have a Cloudflare account and have your domain set up with Cloudflare already, pointing to your server's IP address.
|
||||
|
||||
- Set Cloudflare to use Full SSL mode. Since we are using a self-signed certificate, don't use Full (strict) mode.
|
||||
- Create a self-signed certificate:
|
||||
- `openssl req -x509 -nodes -days 365 -newkey rsa:2048 -keyout /etc/ssl/private/nginx-selfsigned.key -out /etc/ssl/certs/nginx-selfsigned.crt`
|
||||
- `sudo nano /etc/nginx/sites-available/oai.conf`
|
||||
- ```
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name yourdomain.com www.yourdomain.com;
|
||||
|
||||
ssl_certificate /etc/ssl/certs/nginx-selfsigned.crt;
|
||||
ssl_certificate_key /etc/ssl/private/nginx-selfsigned.key;
|
||||
|
||||
# Only allow inbound traffic from Cloudflare
|
||||
allow 173.245.48.0/20;
|
||||
allow 103.21.244.0/22;
|
||||
allow 103.22.200.0/22;
|
||||
allow 103.31.4.0/22;
|
||||
allow 141.101.64.0/18;
|
||||
allow 108.162.192.0/18;
|
||||
allow 190.93.240.0/20;
|
||||
allow 188.114.96.0/20;
|
||||
allow 197.234.240.0/22;
|
||||
allow 198.41.128.0/17;
|
||||
allow 162.158.0.0/15;
|
||||
allow 104.16.0.0/13;
|
||||
allow 104.24.0.0/14;
|
||||
allow 172.64.0.0/13;
|
||||
allow 131.0.72.0/22;
|
||||
deny all;
|
||||
|
||||
location / {
|
||||
proxy_pass http://localhost:7860;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header Connection 'upgrade';
|
||||
proxy_set_header Host $host;
|
||||
proxy_cache_bypass $http_upgrade;
|
||||
}
|
||||
|
||||
ssl_protocols TLSv1.2 TLSv1.3;
|
||||
ssl_ciphers 'ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256';
|
||||
ssl_prefer_server_ciphers on;
|
||||
ssl_session_cache shared:SSL:10m;
|
||||
}
|
||||
```
|
||||
- Replace `yourdomain.com` with your domain name.
|
||||
- Ctrl+X to exit, Y to save, Enter to confirm.
|
||||
- `sudo ln -s /etc/nginx/sites-available/oai.conf /etc/nginx/sites-enabled`
|
||||
|
||||
## Updating/Restarting the application
|
||||
|
||||
After making an .env change, you need to restart the application for it to take effect.
|
||||
|
||||
- `docker compose down`
|
||||
- `docker compose up -d`
|
||||
|
||||
To update the application to the latest version:
|
||||
|
||||
- `docker compose pull`
|
||||
- `docker compose down`
|
||||
- `docker compose up -d`
|
||||
- `docker image prune -f`
|
||||
|
|
@ -161,7 +161,18 @@ function registerUncaughtExceptionHandler() {
|
|||
* didn't set it to something misleading.
|
||||
*/
|
||||
async function setBuildInfo() {
|
||||
// Render .dockerignore's the .git directory but provides info in the env
|
||||
// For CI builds, use the env vars set during the build process
|
||||
if (process.env.GITGUD_BRANCH) {
|
||||
const sha = process.env.GITGUD_COMMIT?.slice(0, 7) || "unknown SHA";
|
||||
const branch = process.env.GITGUD_BRANCH;
|
||||
const repo = process.env.GITGUD_PROJECT;
|
||||
const buildInfo = `[ci] ${sha} (${branch}@${repo})`;
|
||||
process.env.BUILD_INFO = buildInfo;
|
||||
logger.info({ build: buildInfo }, "Using build info from CI image.");
|
||||
return;
|
||||
}
|
||||
|
||||
// For render, the git directory is dockerignore'd so we use env vars
|
||||
if (process.env.RENDER) {
|
||||
const sha = process.env.RENDER_GIT_COMMIT?.slice(0, 7) || "unknown SHA";
|
||||
const branch = process.env.RENDER_GIT_BRANCH || "unknown branch";
|
||||
|
|
@ -172,10 +183,11 @@ async function setBuildInfo() {
|
|||
return;
|
||||
}
|
||||
|
||||
|
||||
// For huggingface and bare metal deployments, we can get the info from git
|
||||
try {
|
||||
// Ignore git's complaints about dubious directory ownership on Huggingface
|
||||
// (which evidently runs dockerized Spaces on Windows with weird NTFS perms)
|
||||
if (process.env.SPACE_ID) {
|
||||
// TODO: may not be necessary anymore with adjusted Huggingface dockerfile
|
||||
childProcess.execSync("git config --global --add safe.directory /app");
|
||||
}
|
||||
|
||||
|
|
@ -195,7 +207,7 @@ async function setBuildInfo() {
|
|||
|
||||
let [sha, branch, remote, status] = await Promise.all(promises);
|
||||
|
||||
remote = remote.match(/.*[\/:]([\w-]+)\/([\w\-\.]+?)(?:\.git)?$/) || [];
|
||||
remote = remote.match(/.*[\/:]([\w-]+)\/([\w\-.]+?)(?:\.git)?$/) || [];
|
||||
const repo = remote.slice(-2).join("/");
|
||||
status = status
|
||||
// ignore Dockerfile changes since that's how the user deploys the app
|
||||
|
|
|
|||
Loading…
Reference in New Issue