Mirrors
/
oai-reverse-proxy
mirror of https://gitgud.io/khanon/oai-reverse-proxy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

improves model detection for AWS Sydney region

This commit is contained in:
nai-degen 2024-06-20 12:15:18 -05:00
parent 84b917f726
commit ff38eda066
1 changed files with 12 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/shared/key-management/aws/checker.ts
View File

@ -57,25 +57,10 @@ export class AwsKeyChecker extends KeyCheckerBase<AwsBedrockKey> {
this.invokeModel("anthropic.claude-3-sonnet-20240229-v1:0", key),
this.invokeModel("anthropic.claude-3-haiku-20240307-v1:0", key),
this.invokeModel("anthropic.claude-3-opus-20240229-v1:0", key),
this.invokeModel("anthropic.claude-3-5-sonnet-20240620-v1:0", key),
];
}
// Sonnet 3.5 is being gradually rolled out and some AWS keys/regions throw
// a ResourceNotFoundException when trying to invoke it which will fail the
// entire key As a temporary measure we will trap thrown errors for this
// particular check and ignore them.
checks.push(
this.invokeModel("anthropic.claude-3-5-sonnet-20240620-v1:0", key).catch(
({ response }) => {
this.log.debug(
{ key: key.hash, error: response.data, status: response.status },
"AWS Sonnet 3.5 model is not accessible."
);
return false;
}
)
);
checks.unshift(this.checkLoggingConfiguration(key));
const [_logging, claudeV2, sonnet, haiku, opus, sonnet35] =
@ -199,7 +184,7 @@ export class AwsKeyChecker extends KeyCheckerBase<AwsBedrockKey> {
method: "POST",
url: POST_INVOKE_MODEL_URL(creds.region, model),
data: payload,
validateStatus: (status) => status === 400 || status === 403,
validateStatus: (status) => [400, 403, 404].includes(status),
};
config.headers = new AxiosHeaders({
"content-type": "application/json",
@ -218,6 +203,16 @@ export class AwsKeyChecker extends KeyCheckerBase<AwsBedrockKey> {
) {
return false;
}
// ResourceNotFound typically indicates that the tested model cannot be used
// on the configured region for this set of credentials.
if (status === 404) {
this.log.debug(
{ region: creds.region, model, key: key.hash },
"Model not supported in this AWS region."
);
return false;
}
// We're looking for a specific error type and message here
// "ValidationException"