diff --git a/README.md b/README.md index 59b8be5..3a14ea6 100644 --- a/README.md +++ b/README.md @@ -40,8 +40,6 @@ This project allows you to run a reverse proxy server for various LLM APIs. - [x] IP blacklists - [x] Proof-of-work challenge for access by anonymous users ---- - ## Usage Instructions If you'd like to run your own instance of this server, you'll need to deploy it somewhere and configure it with your API keys. A few easy options are provided below, though you can also deploy it to any other service you'd like if you know what you're doing and the service supports Node.js. diff --git a/src/proxy/anthropic.ts b/src/proxy/anthropic.ts index 78be31c..7a77569 100644 --- a/src/proxy/anthropic.ts +++ b/src/proxy/anthropic.ts @@ -169,12 +169,12 @@ export function transformAnthropicChatResponseToOpenAI( /** * If a client using the OpenAI compatibility endpoint requests an actual OpenAI - * model, reassigns it to Claude 3 Sonnet. + * model, reassigns it to Sonnet. */ function maybeReassignModel(req: Request) { const model = req.body.model; - if (!model.startsWith("gpt-")) return; - req.body.model = "claude-3-sonnet-20240229"; + if (model.includes("claude")) return; // use whatever model the user requested + req.body.model = "claude-3-5-sonnet-latest"; } /** diff --git a/src/proxy/aws-claude.ts b/src/proxy/aws-claude.ts index 7648b1b..5eaedc6 100644 --- a/src/proxy/aws-claude.ts +++ b/src/proxy/aws-claude.ts @@ -80,7 +80,7 @@ const awsClaudeProxy = createQueuedProxyMiddleware({ if (!signedRequest) throw new Error("Must sign request before proxying"); return `${signedRequest.protocol}//${signedRequest.hostname}`; }, - mutations: [signAwsRequest,finalizeSignedRequest], + mutations: [signAwsRequest, finalizeSignedRequest], blockingResponseHandler: awsBlockingResponseHandler, }); @@ -177,19 +177,17 @@ function maybeReassignModel(req: Request) { // Anthropic model names can look like: // - claude-v1 // - claude-2.1 - // - claude-3-5-sonnet-20240620-v1:0 + // - claude-3-5-sonnet-20240620 + // - claude-3-opus-latest const pattern = - /^(claude-)?(instant-)?(v)?(\d+)([.-](\d))?(-\d+k)?(-sonnet-|-opus-|-haiku-)?(\d*)/i; + /^(claude-)?(instant-)?(v)?(\d+)([.-](\d))?(-\d+k)?(-sonnet-|-opus-|-haiku-)?(latest|\d*)/i; const match = model.match(pattern); - // If there's no match, fallback to Claude v2 as it is most likely to be - // available on AWS. if (!match) { - req.body.model = `anthropic.claude-v2:1`; - return; + throw new Error(`Provided model name (${model}) doesn't resemble a Claude model ID.`); } - const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, _rev] = match; + const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, rev] = match; if (instant) { req.body.model = "anthropic.claude-instant-v1"; @@ -208,22 +206,41 @@ function maybeReassignModel(req: Request) { return; case "3": case "3.0": - if (name.includes("opus")) { - req.body.model = "anthropic.claude-3-opus-20240229-v1:0"; - } else if (name.includes("haiku")) { - req.body.model = "anthropic.claude-3-haiku-20240307-v1:0"; - } else { - req.body.model = "anthropic.claude-3-sonnet-20240229-v1:0"; + // there is only one snapshot for all Claude 3 models so there is no need + // to check the revision + switch (name) { + case "sonnet": + req.body.model = "anthropic.claude-3-sonnet-20240229-v1:0"; + return; + case "haiku": + req.body.model = "anthropic.claude-3-haiku-20240307-v1:0"; + return; + case "opus": + req.body.model = "anthropic.claude-3-opus-20240229-v1:0"; + return; } - return; + break; case "3.5": - req.body.model = "anthropic.claude-3-5-sonnet-20240620-v1:0"; - return; + switch (name) { + case "sonnet": + switch (rev) { + case "20241022": + case "latest": + req.body.model = "anthropic.claude-3-5-sonnet-20241022-v2:0"; + return; + case "20240620": + req.body.model = "anthropic.claude-3-5-sonnet-20240620-v1:0"; + return; + } + break; + case "haiku": + case "opus": + // Add after model ids are announced late 2024 + break; + } } - // Fallback to Claude 2.1 - req.body.model = `anthropic.claude-v2:1`; - return; + throw new Error(`Provided model name (${model}) could not be mapped to a known AWS Claude model ID.`); } export const awsClaude = awsClaudeRouter;