From eb700d3da6221a3b45353dfc968624e0fe631397 Mon Sep 17 00:00:00 2001 From: nai-degen Date: Thu, 20 Jun 2024 10:34:48 -0500 Subject: [PATCH] adds untested claude 3.5 model ids and model assignment --- src/proxy/anthropic.ts | 1 + src/proxy/aws.ts | 72 ++++++++++++----------- src/shared/key-management/aws/checker.ts | 5 +- src/shared/key-management/aws/provider.ts | 10 ++++ 4 files changed, 52 insertions(+), 36 deletions(-) diff --git a/src/proxy/anthropic.ts b/src/proxy/anthropic.ts index 095cd50..e0dcf8c 100644 --- a/src/proxy/anthropic.ts +++ b/src/proxy/anthropic.ts @@ -46,6 +46,7 @@ const getModelsResponse = () => { "claude-3-haiku-20240307", "claude-3-opus-20240229", "claude-3-sonnet-20240229", + "claude-3-5-sonnet-20240628" ]; const models = claudeVariants.map((id) => ({ diff --git a/src/proxy/aws.ts b/src/proxy/aws.ts index 89f6510..c02ff0e 100644 --- a/src/proxy/aws.ts +++ b/src/proxy/aws.ts @@ -37,6 +37,7 @@ const getModelsResponse = () => { "anthropic.claude-v2:1", "anthropic.claude-3-haiku-20240307-v1:0", "anthropic.claude-3-sonnet-20240229-v1:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0", "anthropic.claude-3-opus-20240229-v1:0", ]; @@ -234,65 +235,66 @@ awsRouter.post( * - frontends sending Anthropic model names that AWS doesn't recognize * - frontends sending OpenAI model names because they expect the proxy to * translate them + * + * If client sends AWS model ID it will be used verbatim. Otherwise, various + * strategies are used to try to map a non-AWS model name to AWS model ID. */ function maybeReassignModel(req: Request) { const model = req.body.model; - // If client already specified an AWS Claude model ID, use it + // If it looks like an AWS model, use it as-is if (model.includes("anthropic.claude")) { return; } + // Anthropic model names can look like: + // - claude-v1 + // - claude-2.1 + // - claude-3-5-sonnet-20240620-v1:0 const pattern = - /^(claude-)?(instant-)?(v)?(\d+)(\.(\d+))?(-\d+k)?(-sonnet-?|-opus-?|-haiku-?)(\d*)/i; + /^(claude-)?(instant-)?(v)?(\d+)([.-](\d{1}))?(-\d+k)?(-sonnet-|-opus-|-haiku-)?(\d*)/i; const match = model.match(pattern); - // If there's no match, return the latest v2 model + // If there's no match, fallback to Claude v2 as it is most likely to be + // available on AWS. if (!match) { req.body.model = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`; return; } - const instant = match[2]; - const major = match[4]; - const minor = match[6]; + const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, _rev] = match; if (instant) { req.body.model = "anthropic.claude-instant-v1"; return; } - - // There's only one v1 model - if (major === "1") { - req.body.model = "anthropic.claude-v1"; - return; - } - - // Try to map Anthropic API v2 models to AWS v2 models - if (major === "2") { - if (minor === "0") { + + const ver = minor ? `${major}.${minor}` : major; + switch (ver) { + case "1": + case "1.0": + req.body.model = "anthropic.claude-v1"; + return; + case "2": + case "2.0": req.body.model = "anthropic.claude-v2"; return; - } - req.body.model = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`; - return; + case "3": + case "3.0": + if (name.includes("opus")) { + req.body.model = "anthropic.claude-3-opus-20240229-v1:0"; + } else if (name.includes("haiku")) { + req.body.model = "anthropic.claude-3-haiku-20240307-v1:0"; + } else { + req.body.model = "anthropic.claude-3-sonnet-20240229-v1:0"; + } + return; + case "3.5": + req.body.model = "anthropic.claude-3-5-sonnet-20240620-v1:0"; + return; } - // AWS currently only supports one v3 model. - const variant = match[8]; // sonnet, opus, or haiku - const variantVersion = match[9]; - if (major === "3") { - if (variant.includes("opus")) { - req.body.model = "anthropic.claude-3-opus-20240229-v1:0"; - } else if (variant.includes("haiku")) { - req.body.model = "anthropic.claude-3-haiku-20240307-v1:0"; - } else { - req.body.model = "anthropic.claude-3-sonnet-20240229-v1:0"; - } - return; - } - - // Fallback to latest v2 model + // Fallback to Claude 2.1 req.body.model = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`; return; } @@ -304,7 +306,7 @@ export function handleCompatibilityRequest( ) { const action = req.params.action; const alreadyInChatFormat = Boolean(req.body.messages); - const compatModel = "anthropic.claude-3-sonnet-20240229-v1:0"; + const compatModel = "anthropic.claude-3-5-sonnet-20240620-v1:0"; req.log.info( { inputModel: req.body.model, compatModel, alreadyInChatFormat }, "Handling AWS compatibility request" diff --git a/src/shared/key-management/aws/checker.ts b/src/shared/key-management/aws/checker.ts index e965770..30dbf60 100644 --- a/src/shared/key-management/aws/checker.ts +++ b/src/shared/key-management/aws/checker.ts @@ -57,11 +57,13 @@ export class AwsKeyChecker extends KeyCheckerBase { this.invokeModel("anthropic.claude-3-sonnet-20240229-v1:0", key), this.invokeModel("anthropic.claude-3-haiku-20240307-v1:0", key), this.invokeModel("anthropic.claude-3-opus-20240229-v1:0", key), + this.invokeModel("anthropic.claude-3-5-sonnet-20240620-v1:0", key), ]; } checks.unshift(this.checkLoggingConfiguration(key)); - const [_logging, claudeV2, sonnet, haiku, opus] = await Promise.all(checks); + const [_logging, claudeV2, sonnet, haiku, opus, sonnet35] = + await Promise.all(checks); if (isInitialCheck) { const families: AwsBedrockModelFamily[] = []; @@ -79,6 +81,7 @@ export class AwsKeyChecker extends KeyCheckerBase { this.updateKey(key.hash, { sonnetEnabled: sonnet, haikuEnabled: haiku, + sonnet35Enabled: sonnet35, modelFamilies: families, }); } diff --git a/src/shared/key-management/aws/provider.ts b/src/shared/key-management/aws/provider.ts index 38699d1..44f5f92 100644 --- a/src/shared/key-management/aws/provider.ts +++ b/src/shared/key-management/aws/provider.ts @@ -26,6 +26,7 @@ export interface AwsBedrockKey extends Key, AwsBedrockKeyUsage { awsLoggingStatus: "unknown" | "disabled" | "enabled"; sonnetEnabled: boolean; haikuEnabled: boolean; + sonnet35Enabled: boolean; } /** @@ -77,6 +78,7 @@ export class AwsBedrockKeyProvider implements KeyProvider { lastChecked: 0, sonnetEnabled: true, haikuEnabled: false, + sonnet35Enabled: false, ["aws-claudeTokens"]: 0, ["aws-claude-opusTokens"]: 0, }; @@ -100,15 +102,23 @@ export class AwsBedrockKeyProvider implements KeyProvider { const availableKeys = this.keys.filter((k) => { const isNotLogged = k.awsLoggingStatus !== "enabled"; const neededFamily = getAwsBedrockModelFamily(model); + + // this is a horrible mess + // each of these should be separate model families, but adding model + // families is not low enough friction for the rate at which aws claude + // model variants are added. const needsSonnet = model.includes("sonnet") && neededFamily === "aws-claude"; const needsHaiku = model.includes("haiku") && neededFamily === "aws-claude"; + const needsSonnet35 = + model.includes("claude-3-5-sonnet") && neededFamily === "aws-claude"; return ( !k.isDisabled && (isNotLogged || config.allowAwsLogging) && (k.sonnetEnabled || !needsSonnet) && // sonnet and haiku are both under aws-claude, while opus is not (k.haikuEnabled || !needsHaiku) && + (k.sonnet35Enabled || !needsSonnet35) && k.modelFamilies.includes(neededFamily) ); });