diff --git a/src/proxy/middleware/response/index.ts b/src/proxy/middleware/response/index.ts index 4add890..3b8ca22 100644 --- a/src/proxy/middleware/response/index.ts +++ b/src/proxy/middleware/response/index.ts @@ -431,9 +431,11 @@ async function handleAnthropicBadRequestError( throw new RetryableError("Claude request re-enqueued to add preamble."); } - // Only affects Anthropic keys // {"type":"error","error":{"type":"invalid_request_error","message":"Usage blocked until 2024-03-01T00:00:00+00:00 due to user specified spend limits."}} - const isOverQuota = error?.message?.match(/usage blocked until/i); + // {"type":"error","error":{"type":"invalid_request_error","message":"Your credit balance is too low to access the Claude API. Please go to Plans & Billing to upgrade or purchase credits."}} + const isOverQuota = + error?.message?.match(/usage blocked until/i) || + error?.message?.match(/credit balance is too low/i); if (isOverQuota) { req.log.warn( { key: req.key?.hash, message: error?.message }, diff --git a/src/shared/key-management/anthropic/checker.ts b/src/shared/key-management/anthropic/checker.ts index d853303..388b23b 100644 --- a/src/shared/key-management/anthropic/checker.ts +++ b/src/shared/key-management/anthropic/checker.ts @@ -11,7 +11,7 @@ const POZZED_RESPONSES = [ /please answer ethically/i, /respond as helpfully/i, /be very careful to ensure/i, - /song lyrics, sections of books, or long excerpts/i + /song lyrics, sections of books, or long excerpts/i, ]; type CompleteResponse = { @@ -44,23 +44,28 @@ export class AnthropicKeyChecker extends KeyCheckerBase { const [{ pozzed }] = await Promise.all([this.testLiveness(key)]); const updates = { isPozzed: pozzed }; this.updateKey(key.hash, updates); - this.log.info( - { key: key.hash, models: key.modelFamilies }, - "Checked key." - ); + this.log.info({ key: key.hash, models: key.modelFamilies }, "Checked key."); } protected handleAxiosError(key: AnthropicKey, error: AxiosError) { if (error.response && AnthropicKeyChecker.errorIsAnthropicAPIError(error)) { const { status, data } = error.response; - if (status === 401 || status === 403) { + const isOverQuota = + data.error?.message?.match(/usage blocked until/i) || + data.error?.message?.match(/credit balance is too low/i); + if (status === 400 && isOverQuota) { + this.log.warn( + { key: key.hash, error: data }, + "Key is over quota. Disabling key." + ); + this.updateKey(key.hash, { isDisabled: true, isOverQuota: true }); + } else if (status === 401 || status === 403) { this.log.warn( { key: key.hash, error: data }, "Key is invalid or revoked. Disabling key." ); this.updateKey(key.hash, { isDisabled: true, isRevoked: true }); - } - else if (status === 429) { + } else if (status === 429) { switch (data.error.type) { case "rate_limit_error": this.log.warn( @@ -111,7 +116,7 @@ export class AnthropicKeyChecker extends KeyCheckerBase { { headers: AnthropicKeyChecker.getHeaders(key) } ); this.log.debug({ data }, "Response from Anthropic"); - if (POZZED_RESPONSES.some(re => re.test(data.completion))) { + if (POZZED_RESPONSES.some((re) => re.test(data.completion))) { this.log.debug( { key: key.hash, response: data.completion }, "Key is pozzed." diff --git a/src/shared/streaming.ts b/src/shared/streaming.ts index 65eb957..d04a009 100644 --- a/src/shared/streaming.ts +++ b/src/shared/streaming.ts @@ -94,6 +94,13 @@ export function makeCompletionSSE({ log_id: "proxy-req-" + id, }; break; + case "anthropic-chat": + event = { + type: "content_block_delta", + index: 0, + delta: { type: "text_delta", text: content }, + }; + break; case "google-ai": return JSON.stringify({ candidates: [ @@ -106,7 +113,6 @@ export function makeCompletionSSE({ }, ], }); - case "anthropic-chat": case "openai-image": throw new Error(`SSE not supported for ${format} requests`); default: @@ -120,5 +126,52 @@ export function makeCompletionSSE({ ); } + // ugh. + if (format === "anthropic-chat") { + return ( + [ + [ + "event: message_start", + `data: ${JSON.stringify({ + type: "message_start", + message: { + id: "error-" + id, + type: "message", + role: "assistant", + content: [], + model, + }, + })}`, + ].join("\n"), + [ + "event: content_block_start", + `data: ${JSON.stringify({ + type: "content_block_start", + index: 0, + content_block: { type: "text", text: "" }, + })}`, + ].join("\n"), + ["event: content_block_delta", `data: ${JSON.stringify(event)}`].join( + "\n" + ), + [ + "event: content_block_stop", + `data: ${JSON.stringify({ type: "content_block_stop", index: 0 })}`, + ].join("\n"), + [ + "event: message_delta", + `data: ${JSON.stringify({ + type: "message_delta", + delta: { stop_reason: title, stop_sequence: null, usage: null }, + })}`, + ], + [ + "event: message_stop", + `data: ${JSON.stringify({ type: "message_stop" })}`, + ].join("\n"), + ].join("\n\n") + "\n\n" + ); + } + return `data: ${JSON.stringify(event)}\n\n`; }