adds untested claude 3.5 model ids and model assignment
This commit is contained in:
parent
d706d4c59d
commit
eb700d3da6
|
@ -46,6 +46,7 @@ const getModelsResponse = () => {
|
|||
"claude-3-haiku-20240307",
|
||||
"claude-3-opus-20240229",
|
||||
"claude-3-sonnet-20240229",
|
||||
"claude-3-5-sonnet-20240628"
|
||||
];
|
||||
|
||||
const models = claudeVariants.map((id) => ({
|
||||
|
|
|
@ -37,6 +37,7 @@ const getModelsResponse = () => {
|
|||
"anthropic.claude-v2:1",
|
||||
"anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20240620-v1:0",
|
||||
"anthropic.claude-3-opus-20240229-v1:0",
|
||||
];
|
||||
|
||||
|
@ -234,65 +235,66 @@ awsRouter.post(
|
|||
* - frontends sending Anthropic model names that AWS doesn't recognize
|
||||
* - frontends sending OpenAI model names because they expect the proxy to
|
||||
* translate them
|
||||
*
|
||||
* If client sends AWS model ID it will be used verbatim. Otherwise, various
|
||||
* strategies are used to try to map a non-AWS model name to AWS model ID.
|
||||
*/
|
||||
function maybeReassignModel(req: Request) {
|
||||
const model = req.body.model;
|
||||
|
||||
// If client already specified an AWS Claude model ID, use it
|
||||
// If it looks like an AWS model, use it as-is
|
||||
if (model.includes("anthropic.claude")) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Anthropic model names can look like:
|
||||
// - claude-v1
|
||||
// - claude-2.1
|
||||
// - claude-3-5-sonnet-20240620-v1:0
|
||||
const pattern =
|
||||
/^(claude-)?(instant-)?(v)?(\d+)(\.(\d+))?(-\d+k)?(-sonnet-?|-opus-?|-haiku-?)(\d*)/i;
|
||||
/^(claude-)?(instant-)?(v)?(\d+)([.-](\d{1}))?(-\d+k)?(-sonnet-|-opus-|-haiku-)?(\d*)/i;
|
||||
const match = model.match(pattern);
|
||||
|
||||
// If there's no match, return the latest v2 model
|
||||
// If there's no match, fallback to Claude v2 as it is most likely to be
|
||||
// available on AWS.
|
||||
if (!match) {
|
||||
req.body.model = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`;
|
||||
return;
|
||||
}
|
||||
|
||||
const instant = match[2];
|
||||
const major = match[4];
|
||||
const minor = match[6];
|
||||
const [_, _cl, instant, _v, major, _sep, minor, _ctx, name, _rev] = match;
|
||||
|
||||
if (instant) {
|
||||
req.body.model = "anthropic.claude-instant-v1";
|
||||
return;
|
||||
}
|
||||
|
||||
// There's only one v1 model
|
||||
if (major === "1") {
|
||||
const ver = minor ? `${major}.${minor}` : major;
|
||||
switch (ver) {
|
||||
case "1":
|
||||
case "1.0":
|
||||
req.body.model = "anthropic.claude-v1";
|
||||
return;
|
||||
}
|
||||
|
||||
// Try to map Anthropic API v2 models to AWS v2 models
|
||||
if (major === "2") {
|
||||
if (minor === "0") {
|
||||
case "2":
|
||||
case "2.0":
|
||||
req.body.model = "anthropic.claude-v2";
|
||||
return;
|
||||
}
|
||||
req.body.model = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`;
|
||||
return;
|
||||
}
|
||||
|
||||
// AWS currently only supports one v3 model.
|
||||
const variant = match[8]; // sonnet, opus, or haiku
|
||||
const variantVersion = match[9];
|
||||
if (major === "3") {
|
||||
if (variant.includes("opus")) {
|
||||
case "3":
|
||||
case "3.0":
|
||||
if (name.includes("opus")) {
|
||||
req.body.model = "anthropic.claude-3-opus-20240229-v1:0";
|
||||
} else if (variant.includes("haiku")) {
|
||||
} else if (name.includes("haiku")) {
|
||||
req.body.model = "anthropic.claude-3-haiku-20240307-v1:0";
|
||||
} else {
|
||||
req.body.model = "anthropic.claude-3-sonnet-20240229-v1:0";
|
||||
}
|
||||
return;
|
||||
case "3.5":
|
||||
req.body.model = "anthropic.claude-3-5-sonnet-20240620-v1:0";
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback to latest v2 model
|
||||
// Fallback to Claude 2.1
|
||||
req.body.model = `anthropic.claude-v2:${LATEST_AWS_V2_MINOR_VERSION}`;
|
||||
return;
|
||||
}
|
||||
|
@ -304,7 +306,7 @@ export function handleCompatibilityRequest(
|
|||
) {
|
||||
const action = req.params.action;
|
||||
const alreadyInChatFormat = Boolean(req.body.messages);
|
||||
const compatModel = "anthropic.claude-3-sonnet-20240229-v1:0";
|
||||
const compatModel = "anthropic.claude-3-5-sonnet-20240620-v1:0";
|
||||
req.log.info(
|
||||
{ inputModel: req.body.model, compatModel, alreadyInChatFormat },
|
||||
"Handling AWS compatibility request"
|
||||
|
|
|
@ -57,11 +57,13 @@ export class AwsKeyChecker extends KeyCheckerBase<AwsBedrockKey> {
|
|||
this.invokeModel("anthropic.claude-3-sonnet-20240229-v1:0", key),
|
||||
this.invokeModel("anthropic.claude-3-haiku-20240307-v1:0", key),
|
||||
this.invokeModel("anthropic.claude-3-opus-20240229-v1:0", key),
|
||||
this.invokeModel("anthropic.claude-3-5-sonnet-20240620-v1:0", key),
|
||||
];
|
||||
}
|
||||
checks.unshift(this.checkLoggingConfiguration(key));
|
||||
|
||||
const [_logging, claudeV2, sonnet, haiku, opus] = await Promise.all(checks);
|
||||
const [_logging, claudeV2, sonnet, haiku, opus, sonnet35] =
|
||||
await Promise.all(checks);
|
||||
|
||||
if (isInitialCheck) {
|
||||
const families: AwsBedrockModelFamily[] = [];
|
||||
|
@ -79,6 +81,7 @@ export class AwsKeyChecker extends KeyCheckerBase<AwsBedrockKey> {
|
|||
this.updateKey(key.hash, {
|
||||
sonnetEnabled: sonnet,
|
||||
haikuEnabled: haiku,
|
||||
sonnet35Enabled: sonnet35,
|
||||
modelFamilies: families,
|
||||
});
|
||||
}
|
||||
|
|
|
@ -26,6 +26,7 @@ export interface AwsBedrockKey extends Key, AwsBedrockKeyUsage {
|
|||
awsLoggingStatus: "unknown" | "disabled" | "enabled";
|
||||
sonnetEnabled: boolean;
|
||||
haikuEnabled: boolean;
|
||||
sonnet35Enabled: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -77,6 +78,7 @@ export class AwsBedrockKeyProvider implements KeyProvider<AwsBedrockKey> {
|
|||
lastChecked: 0,
|
||||
sonnetEnabled: true,
|
||||
haikuEnabled: false,
|
||||
sonnet35Enabled: false,
|
||||
["aws-claudeTokens"]: 0,
|
||||
["aws-claude-opusTokens"]: 0,
|
||||
};
|
||||
|
@ -100,15 +102,23 @@ export class AwsBedrockKeyProvider implements KeyProvider<AwsBedrockKey> {
|
|||
const availableKeys = this.keys.filter((k) => {
|
||||
const isNotLogged = k.awsLoggingStatus !== "enabled";
|
||||
const neededFamily = getAwsBedrockModelFamily(model);
|
||||
|
||||
// this is a horrible mess
|
||||
// each of these should be separate model families, but adding model
|
||||
// families is not low enough friction for the rate at which aws claude
|
||||
// model variants are added.
|
||||
const needsSonnet =
|
||||
model.includes("sonnet") && neededFamily === "aws-claude";
|
||||
const needsHaiku =
|
||||
model.includes("haiku") && neededFamily === "aws-claude";
|
||||
const needsSonnet35 =
|
||||
model.includes("claude-3-5-sonnet") && neededFamily === "aws-claude";
|
||||
return (
|
||||
!k.isDisabled &&
|
||||
(isNotLogged || config.allowAwsLogging) &&
|
||||
(k.sonnetEnabled || !needsSonnet) && // sonnet and haiku are both under aws-claude, while opus is not
|
||||
(k.haikuEnabled || !needsHaiku) &&
|
||||
(k.sonnet35Enabled || !needsSonnet35) &&
|
||||
k.modelFamilies.includes(neededFamily)
|
||||
);
|
||||
});
|
||||
|
|
Loading…
Reference in New Issue