detects and removes over-quota claude keys from keypool

src/proxy/middleware/response/index.ts

@@ -431,9 +431,11 @@ async function handleAnthropicBadRequestError(
     throw new RetryableError("Claude request re-enqueued to add preamble.");
   }
 
-  // Only affects Anthropic keys
   // {"type":"error","error":{"type":"invalid_request_error","message":"Usage blocked until 2024-03-01T00:00:00+00:00 due to user specified spend limits."}}
-  const isOverQuota = error?.message?.match(/usage blocked until/i);
+  // {"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low to access the Claude API. Please go to Plans & Billing to upgrade or purchase credits."}}
+  const isOverQuota =
+    error?.message?.match(/usage blocked until/i) ||
+    error?.message?.match(/credit balance is too low/i);
   if (isOverQuota) {
     req.log.warn(
       { key: req.key?.hash, message: error?.message },

src/shared/key-management/anthropic/checker.ts

@@ -11,7 +11,7 @@ const POZZED_RESPONSES = [
   /please answer ethically/i,
   /respond as helpfully/i,
   /be very careful to ensure/i,
-  /song lyrics, sections of books, or long excerpts/i
+  /song lyrics, sections of books, or long excerpts/i,
 ];
 
 type CompleteResponse = {
@@ -44,23 +44,28 @@ export class AnthropicKeyChecker extends KeyCheckerBase<AnthropicKey> {
     const [{ pozzed }] = await Promise.all([this.testLiveness(key)]);
     const updates = { isPozzed: pozzed };
     this.updateKey(key.hash, updates);
-    this.log.info(
-      { key: key.hash, models: key.modelFamilies },
-      "Checked key."
-    );
+    this.log.info({ key: key.hash, models: key.modelFamilies }, "Checked key.");
   }
 
   protected handleAxiosError(key: AnthropicKey, error: AxiosError) {
     if (error.response && AnthropicKeyChecker.errorIsAnthropicAPIError(error)) {
       const { status, data } = error.response;
-      if (status === 401 || status === 403) {
+      const isOverQuota =
+        data.error?.message?.match(/usage blocked until/i) ||
+        data.error?.message?.match(/credit balance is too low/i);
+      if (status === 400 && isOverQuota) {
+        this.log.warn(
+          { key: key.hash, error: data },
+          "Key is over quota. Disabling key."
+        );
+        this.updateKey(key.hash, { isDisabled: true, isOverQuota: true });
+      } else if (status === 401 || status === 403) {
         this.log.warn(
           { key: key.hash, error: data },
           "Key is invalid or revoked. Disabling key."
         );
         this.updateKey(key.hash, { isDisabled: true, isRevoked: true });
-      }
-      else if (status === 429) {
+      } else if (status === 429) {
         switch (data.error.type) {
           case "rate_limit_error":
             this.log.warn(
@@ -111,7 +116,7 @@ export class AnthropicKeyChecker extends KeyCheckerBase<AnthropicKey> {
       { headers: AnthropicKeyChecker.getHeaders(key) }
     );
     this.log.debug({ data }, "Response from Anthropic");
-    if (POZZED_RESPONSES.some(re => re.test(data.completion))) {
+    if (POZZED_RESPONSES.some((re) => re.test(data.completion))) {
       this.log.debug(
         { key: key.hash, response: data.completion },
         "Key is pozzed."

src/shared/streaming.ts

@@ -94,6 +94,13 @@ export function makeCompletionSSE({
         log_id: "proxy-req-" + id,
       };
       break;
+    case "anthropic-chat":
+      event = {
+        type: "content_block_delta",
+        index: 0,
+        delta: { type: "text_delta", text: content },
+      };
+      break;
     case "google-ai":
       return JSON.stringify({
         candidates: [
@@ -106,7 +113,6 @@ export function makeCompletionSSE({
           },
         ],
       });
-    case "anthropic-chat":
     case "openai-image":
       throw new Error(`SSE not supported for ${format} requests`);
     default:
@@ -120,5 +126,52 @@ export function makeCompletionSSE({
     );
   }
 
+  // ugh.
+  if (format === "anthropic-chat") {
+    return (
+      [
+        [
+          "event: message_start",
+          `data: ${JSON.stringify({
+            type: "message_start",
+            message: {
+              id: "error-" + id,
+              type: "message",
+              role: "assistant",
+              content: [],
+              model,
+            },
+          })}`,
+        ].join("\n"),
+        [
+          "event: content_block_start",
+          `data: ${JSON.stringify({
+            type: "content_block_start",
+            index: 0,
+            content_block: { type: "text", text: "" },
+          })}`,
+        ].join("\n"),
+        ["event: content_block_delta", `data: ${JSON.stringify(event)}`].join(
+          "\n"
+        ),
+        [
+          "event: content_block_stop",
+          `data: ${JSON.stringify({ type: "content_block_stop", index: 0 })}`,
+        ].join("\n"),
+        [
+          "event: message_delta",
+          `data: ${JSON.stringify({
+            type: "message_delta",
+            delta: { stop_reason: title, stop_sequence: null, usage: null },
+          })}`,
+        ],
+        [
+          "event: message_stop",
+          `data: ${JSON.stringify({ type: "message_stop" })}`,
+        ].join("\n"),
+      ].join("\n\n") + "\n\n"
+    );
+  }
+
   return `data: ${JSON.stringify(event)}\n\n`;
 }