improves aws sonnet key detection and no keys available error messaging

src/proxy/aws.ts

@@ -211,7 +211,7 @@ awsRouter.post("/v1/claude-3/messages", (req, res) => {
     options: {
       title: "Proxy error (wrong endpoint)",
       message:
-        "Your client is attempting to use the /anthropic/v1/claude-3 compatibility endpoint, but supports the new API format and should use the normal /anthropic/v1 endpoint instead.",
+        "Your client is attempting to use the /aws/claude/claude-3 compatibility endpoint, but supports the new API format and should use the normal /aws/claude endpoint instead.",
       format: "unknown",
       statusCode: 404,
       reqId: req.id,

src/proxy/middleware/common.ts

@@ -5,6 +5,7 @@ import { generateErrorMessage } from "zod-error";
 import { assertNever } from "../../shared/utils";
 import { QuotaExceededError } from "./request/preprocessors/apply-quota-limits";
 import { buildSpoofedSSE, sendErrorToClient } from "./response/error-generator";
+import { HttpError } from "../../shared/errors";
 
 const OPENAI_CHAT_COMPLETION_ENDPOINT = "/v1/chat/completions";
 const OPENAI_TEXT_COMPLETION_ENDPOINT = "/v1/completions";
@@ -111,6 +112,15 @@ function classifyError(err: Error): {
   };
 
   switch (err.constructor.name) {
+    case "HttpError":
+      if ((err as HttpError).status === 402) {
+        return {
+          statusCode: 402,
+          statusMessage: "No Keys Available",
+          userMessage: err.message,
+          type: "proxy_no_keys_available",
+        };
+      } else return defaultError;
     case "ZodError":
       const userMessage = generateErrorMessage((err as ZodError).issues, {
         prefix: "Request validation failed. ",

src/proxy/middleware/response/error-generator.ts

@@ -40,10 +40,8 @@ function getMessageContent({
   }
   ```
    */
-
-  const friendlyMessage = obj?.proxy_note
-    ? `${message}\n\n***\n\n*${obj.proxy_note}*`
-    : message;
+  const note = obj?.proxy_note || obj?.error?.message || "";
+  const friendlyMessage = note ? `${message}\n\n***\n\n*${note}*` : message;
   const details = JSON.parse(JSON.stringify(obj ?? {}));
   let stack = "";
   if (details.stack) {

src/proxy/middleware/response/index.ts

@@ -330,9 +330,8 @@ const handleUpstreamErrors: ProxyResHandlerWithBody = async (
         errorPayload.proxy_note = `API key is invalid or revoked. ${tryAgainMessage}`;
         break;
       case "AccessDeniedException":
-        const isModelAccessError = errorPayload.error?.message?.includes(
-          `access to the model with the specified model ID`
-        );
+        const isModelAccessError =
+          errorPayload.error?.message?.includes(`specified model ID`);
         if (!isModelAccessError) {
           req.log.error(
             { key: req.key?.hash, model: req.body?.model },
@@ -451,7 +450,7 @@ async function handleAnthropicBadRequestError(
     return;
   }
 
-  errorPayload.proxy_note = `Unrecognized 400 Bad Request error from the API.`;
+  errorPayload.proxy_note = `Unrecognized error from the API. (${error?.message})`;
 }
 
 async function handleAnthropicRateLimitError(

src/proxy/middleware/response/streaming/sse-stream-adapter.ts

@@ -21,6 +21,7 @@ type SSEStreamAdapterOptions = TransformOptions & {
 export class SSEStreamAdapter extends Transform {
   private readonly isAwsStream;
   private readonly isGoogleStream;
+  private api: APIFormat;
   private partialMessage = "";
   private textDecoder = new TextDecoder("utf8");
   private log: pino.Logger;
@@ -30,6 +31,7 @@ export class SSEStreamAdapter extends Transform {
     this.isAwsStream =
       options?.contentType === "application/vnd.amazon.eventstream";
     this.isGoogleStream = options?.api === "google-ai";
+    this.api = options.api;
     this.log = options.logger.child({ module: "sse-stream-adapter" });
   }
 
@@ -51,13 +53,10 @@ export class SSEStreamAdapter extends Transform {
           const event = Buffer.from(bytes, "base64").toString("utf8");
           const eventObj = JSON.parse(event);
 
-          if ('completion' in eventObj) {
+          if ("completion" in eventObj) {
             return ["event: completion", `data: ${event}`].join(`\n`);
           } else {
-            return [
-              `event: ${eventObj.type}`,
-              `data: ${event}`,
-            ].join(`\n`);
+            return [`event: ${eventObj.type}`, `data: ${event}`].join(`\n`);
           }
         }
       // Intentional fallthrough, as non-JSON events may as well be errors
@@ -75,15 +74,10 @@ export class SSEStreamAdapter extends Transform {
             throw new RetryableError("AWS request throttled mid-stream");
           default:
             this.log.error({ message, type }, "Received bad AWS stream event");
-            return buildSpoofedSSE({
-              format: "anthropic-text",
-              title: "Proxy stream error",
-              message:
-                "The proxy received an unrecognized error from AWS while streaming.",
-              obj: message,
-              reqId: "proxy-sse-adapter-message",
-              model: "",
-            });
+            const error: any = new Error(`Got mysterious error chunk: ${type}`);
+            error.lastEvent = message;
+            this.emit("error", error);
+            return null;
         }
       default:
         // Amazon says this can't ever happen...

src/shared/errors.ts

@@ -1,6 +1,7 @@
 export class HttpError extends Error {
   constructor(public status: number, message: string) {
     super(message);
+    this.name = "HttpError";
   }
 }
 

src/shared/key-management/anthropic/provider.ts

@@ -4,6 +4,7 @@ import { config } from "../../../config";
 import { logger } from "../../../logger";
 import { AnthropicModelFamily, getClaudeModelFamily } from "../../models";
 import { AnthropicKeyChecker } from "./checker";
+import { HttpError } from "../../errors";
 
 // https://docs.anthropic.com/claude/reference/selecting-a-model
 export type AnthropicModel =
@@ -130,7 +131,7 @@ export class AnthropicKeyProvider implements KeyProvider<AnthropicKey> {
     // certainly change when they move out of beta later this year.
     const availableKeys = this.keys.filter((k) => !k.isDisabled);
     if (availableKeys.length === 0) {
-      throw new Error("No Anthropic keys available.");
+      throw new HttpError(402, "No Anthropic keys available.");
     }
 
     // (largely copied from the OpenAI provider, without trial key support)

src/shared/key-management/aws/provider.ts

@@ -4,6 +4,7 @@ import { config } from "../../../config";
 import { logger } from "../../../logger";
 import type { AwsBedrockModelFamily } from "../../models";
 import { AwsKeyChecker } from "./checker";
+import { HttpError } from "../../errors";
 
 // https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html
 export type AwsBedrockModel =
@@ -109,7 +110,8 @@ export class AwsBedrockKeyProvider implements KeyProvider<AwsBedrockKey> {
       );
     });
     if (availableKeys.length === 0) {
-      throw new Error(
+      throw new HttpError(
+        402,
         "No keys available for this model. If you are requesting Sonnet, use Claude-2 instead."
       );
     }

src/shared/key-management/azure/provider.ts

@@ -6,6 +6,7 @@ import type { AzureOpenAIModelFamily } from "../../models";
 import { getAzureOpenAIModelFamily } from "../../models";
 import { OpenAIModel } from "../openai/provider";
 import { AzureOpenAIKeyChecker } from "./checker";
+import { HttpError } from "../../errors";
 
 export type AzureOpenAIModel = Exclude<OpenAIModel, "dall-e">;
 
@@ -100,7 +101,10 @@ export class AzureOpenAIKeyProvider implements KeyProvider<AzureOpenAIKey> {
       (k) => !k.isDisabled && k.modelFamilies.includes(neededFamily)
     );
     if (availableKeys.length === 0) {
-      throw new Error(`No keys available for model family '${neededFamily}'.`);
+      throw new HttpError(
+        402,
+        `No keys available for model family '${neededFamily}'.`
+      );
     }
 
     // (largely copied from the OpenAI provider, without trial key support)

src/shared/key-management/google-ai/provider.ts

@@ -3,6 +3,7 @@ import { Key, KeyProvider } from "..";
 import { config } from "../../../config";
 import { logger } from "../../../logger";
 import type { GoogleAIModelFamily } from "../../models";
+import { HttpError } from "../../errors";
 
 // Note that Google AI is not the same as Vertex AI, both are provided by Google
 // but Vertex is the GCP product for enterprise. while Google AI is the
@@ -95,7 +96,7 @@ export class GoogleAIKeyProvider implements KeyProvider<GoogleAIKey> {
   public get(_model: GoogleAIModel) {
     const availableKeys = this.keys.filter((k) => !k.isDisabled);
     if (availableKeys.length === 0) {
-      throw new Error("No Google AI keys available");
+      throw new HttpError(402, "No Google AI keys available");
     }
 
     // (largely copied from the OpenAI provider, without trial key support)

src/shared/key-management/mistral-ai/provider.ts

@@ -4,6 +4,7 @@ import { config } from "../../../config";
 import { logger } from "../../../logger";
 import { MistralAIModelFamily, getMistralAIModelFamily } from "../../models";
 import { MistralAIKeyChecker } from "./checker";
+import { HttpError } from "../../errors";
 
 type MistralAIKeyUsage = {
   [K in MistralAIModelFamily as `${K}Tokens`]: number;
@@ -94,7 +95,7 @@ export class MistralAIKeyProvider implements KeyProvider<MistralAIKey> {
   public get(_model: Model) {
     const availableKeys = this.keys.filter((k) => !k.isDisabled);
     if (availableKeys.length === 0) {
-      throw new Error("No Mistral AI keys available");
+      throw new HttpError(402, "No Mistral AI keys available");
     }
 
     // (largely copied from the OpenAI provider, without trial key support)

src/shared/key-management/openai/provider.ts

@@ -8,6 +8,7 @@ import { config } from "../../../config";
 import { logger } from "../../../logger";
 import { OpenAIKeyChecker } from "./checker";
 import { getOpenAIModelFamily, OpenAIModelFamily } from "../../models";
+import { HttpError } from "../../errors";
 
 export type OpenAIModel =
   | "gpt-3.5-turbo"
@@ -17,7 +18,7 @@ export type OpenAIModel =
   | "gpt-4-1106"
   | "text-embedding-ada-002"
   | "dall-e-2"
-  | "dall-e-3"
+  | "dall-e-3";
 
 // Flattening model families instead of using a nested object for easier
 // cloning.
@@ -167,7 +168,10 @@ export class OpenAIKeyProvider implements KeyProvider<OpenAIKey> {
     );
 
     if (availableKeys.length === 0) {
-      throw new Error(`No keys available for model family '${neededFamily}'.`);
+      throw new HttpError(
+        402,
+        `No keys available for model family '${neededFamily}'.`
+      );
     }
 
     // Select a key, from highest priority to lowest priority: