Mirrors
/
oai-reverse-proxy
mirror of https://gitgud.io/khanon/oai-reverse-proxy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

adds Sonnet 3.5v2 AWS model ID and adjusts AWS model assignment to raise error on no match

This commit is contained in:
nai-degen 2024-10-23 13:39:34 -05:00
parent 4190d5fef6
commit 49c578f4dc
3 changed files with 40 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -40,8 +40,6 @@ This project allows you to run a reverse proxy server for various LLM APIs.
- [x] IP blacklists
- [x] Proof-of-work challenge for access by anonymous users
---
## Usage Instructions
If you'd like to run your own instance of this server, you'll need to deploy it somewhere and configure it with your API keys. A few easy options are provided below, though you can also deploy it to any other service you'd like if you know what you're doing and the service supports Node.js.

6
src/proxy/anthropic.ts
View File

@ -169,12 +169,12 @@ export function transformAnthropicChatResponseToOpenAI(
/**
* If a client using the OpenAI compatibility endpoint requests an actual OpenAI
* model, reassigns it to Claude 3 Sonnet.
* model, reassigns it to Sonnet.
*/
function maybeReassignModel(req: Request) {
const model = req.body.model;
if (!model.startsWith("gpt-")) return;
req.body.model = "claude-3-sonnet-20240229";
if (model.includes("claude")) return; // use whatever model the user requested
req.body.model = "claude-3-5-sonnet-latest";
}
/**

49
src/proxy/aws-claude.ts
View File

@ -177,19 +177,17 @@ function maybeReassignModel(req: Request) {
// Anthropic model names can look like:
// - claude-v1
// - claude-2.1
// - claude-3-5-sonnet-20240620-v1:0
// - claude-3-5-sonnet-20240620
// - claude-3-opus-latest
const pattern =
/^(claude-)?(instant-)?(v)?(\d+)([.-](\d))?(-\d+k)?(-sonnet-|-opus-|-haiku-)?(\d*)/i;
/^(claude-)?(instant-)?(v)?(\d+)([.-](\d))?(-\d+k)?(-sonnet-|-opus-|-haiku-)?(latest|\d*)/i;
const match = model.match(pattern);
// If there's no match, fallback to Claude v2 as it is most likely to be
// available on AWS.
if (!match) {
req.body.model = `anthropic.claude-v2:1`;
return;
throw new Error(`Provided model name (${model}) doesn't resemble a Claude model ID.`);
}
const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, _rev] = match;
const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, rev] = match;
if (instant) {
req.body.model = "anthropic.claude-instant-v1";
@ -208,22 +206,41 @@ function maybeReassignModel(req: Request) {
return;
case "3":
case "3.0":
if (name.includes("opus")) {
req.body.model = "anthropic.claude-3-opus-20240229-v1:0";
} else if (name.includes("haiku")) {
req.body.model = "anthropic.claude-3-haiku-20240307-v1:0";
} else {
// there is only one snapshot for all Claude 3 models so there is no need
// to check the revision
switch (name) {
case "sonnet":
req.body.model = "anthropic.claude-3-sonnet-20240229-v1:0";
}
return;
case "haiku":
req.body.model = "anthropic.claude-3-haiku-20240307-v1:0";
return;
case "opus":
req.body.model = "anthropic.claude-3-opus-20240229-v1:0";
return;
}
break;
case "3.5":
switch (name) {
case "sonnet":
switch (rev) {
case "20241022":
case "latest":
req.body.model = "anthropic.claude-3-5-sonnet-20241022-v2:0";
return;
case "20240620":
req.body.model = "anthropic.claude-3-5-sonnet-20240620-v1:0";
return;
}
break;
case "haiku":
case "opus":
// Add after model ids are announced late 2024
break;
}
}
// Fallback to Claude 2.1
req.body.model = `anthropic.claude-v2:1`;
return;
throw new Error(`Provided model name (${model}) could not be mapped to a known AWS Claude model ID.`);
}
export const awsClaude = awsClaudeRouter;