adds ?debug=true query param to have proxy respond with transformed prompt

src/proxy/anthropic.ts

@@ -79,31 +79,23 @@ const anthropicResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
-  }
-
+  let newBody = body;
   switch (`${req.inboundApi}<-${req.outboundApi}`) {
     case "openai<-anthropic-text":
       req.log.info("Transforming Anthropic Text back to OpenAI format");
-      body = transformAnthropicTextResponseToOpenAI(body, req);
+      newBody = transformAnthropicTextResponseToOpenAI(body, req);
       break;
     case "openai<-anthropic-chat":
       req.log.info("Transforming Anthropic Chat back to OpenAI format");
-      body = transformAnthropicChatResponseToOpenAI(body);
+      newBody = transformAnthropicChatResponseToOpenAI(body);
       break;
     case "anthropic-text<-anthropic-chat":
       req.log.info("Transforming Anthropic Chat back to Anthropic chat format");
-      body = transformAnthropicChatResponseToAnthropicText(body);
+      newBody = transformAnthropicChatResponseToAnthropicText(body);
       break;
   }
 
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  res.status(200).json(body);
+  res.status(200).json({ ...newBody, proxy: body.proxy });
 };
 
 function flattenChatResponse(

src/proxy/aws.ts

@@ -70,32 +70,26 @@ const awsResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
+  let newBody = body;
+  switch (`${req.inboundApi}<-${req.outboundApi}`) {
+    case "openai<-anthropic-text":
+      req.log.info("Transforming Anthropic Text back to OpenAI format");
+      newBody = transformAwsTextResponseToOpenAI(body, req);
+      break;
+    // case "openai<-anthropic-chat":
+    // todo: implement this
+    case "anthropic-text<-anthropic-chat":
+      req.log.info("Transforming AWS Anthropic Chat back to Text format");
+      newBody = transformAnthropicChatResponseToAnthropicText(body);
+      break;
   }
 
-  if (req.inboundApi === "openai") {
-    req.log.info("Transforming AWS Claude response to OpenAI format");
-    body = transformAwsTextResponseToOpenAI(body, req);
+  // AWS does not always confirm the model in the response, so we have to add it
+  if (!newBody.model && req.body.model) {
+    newBody.model = req.body.model;
   }
 
-  if (
-    req.inboundApi === "anthropic-text" &&
-    req.outboundApi === "anthropic-chat"
-  ) {
-    req.log.info("Transforming AWS Claude chat response to Text format");
-    body = transformAnthropicChatResponseToAnthropicText(body);
-  }
-
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  // AWS does not confirm the model in the response, so we have to add it
-  body.model = req.body.model;
-
-  res.status(200).json(body);
+  res.status(200).json({ ...newBody, proxy: body.proxy });
 };
 
 /**

src/proxy/azure.ts

@@ -3,9 +3,9 @@ import { createProxyMiddleware } from "http-proxy-middleware";
 import { config } from "../config";
 import { keyPool } from "../shared/key-management";
 import {
-  ModelFamily,
   AzureOpenAIModelFamily,
   getAzureOpenAIModelFamily,
+  ModelFamily,
 } from "../shared/models";
 import { logger } from "../logger";
 import { KNOWN_OPENAI_MODELS } from "./openai";
@@ -80,16 +80,7 @@ const azureOpenaiResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
-  }
-
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  res.status(200).json(body);
+  res.status(200).json({ ...body, proxy: body.proxy });
 };
 
 const azureOpenAIProxy = createQueueMiddleware({

src/proxy/google-ai.ts

@@ -63,21 +63,13 @@ const googleAIResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
-  }
-
+  let newBody = body;
   if (req.inboundApi === "openai") {
     req.log.info("Transforming Google AI response to OpenAI format");
-    body = transformGoogleAIResponse(body, req);
+    newBody = transformGoogleAIResponse(body, req);
   }
 
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  res.status(200).json(body);
+  res.status(200).json({ ...newBody, proxy: body.proxy });
 };
 
 function transformGoogleAIResponse(

src/proxy/middleware/common.ts

@@ -56,10 +56,6 @@ export function sendProxyError(
       ? `The proxy encountered an error while trying to process your prompt.`
       : `The proxy encountered an error while trying to send your prompt to the upstream service.`;
 
-  if (req.tokenizerInfo && typeof errorPayload.error === "object") {
-    errorPayload.error.proxy_tokenizer = req.tokenizerInfo;
-  }
-
   sendErrorToClient({
     options: {
       format: req.inboundApi,

src/proxy/middleware/response/error-generator.ts

@@ -26,15 +26,6 @@ function getMessageContent({
    "error": {
      "type": "not_found_error",
      "message": "model: some-invalid-model-id",
-     "proxy_tokenizer": {
-       "tokenizer": "@anthropic-ai/tokenizer",
-       "token_count": 6104,
-       "tokenization_duration_ms": 4.0765,
-       "prompt_tokens": 6104,
-       "completion_tokens": 30,
-       "max_model_tokens": 200000,
-       "max_proxy_tokens": 9007199254740991
-      }
     },
    "proxy_note": "The requested Claude model might not exist, or the key might not be provisioned for it."
   }

src/proxy/middleware/response/index.ts

@@ -23,6 +23,7 @@ import {
 import { handleStreamedResponse } from "./handle-streamed-response";
 import { logPrompt } from "./log-prompt";
 import { saveImage } from "./save-image";
+import { config } from "../../../config";
 
 const DECODER_MAP = {
   gzip: util.promisify(zlib.gunzip),
@@ -105,6 +106,7 @@ export const createOnProxyResHandler = (apiMiddleware: ProxyResMiddleware) => {
       } else {
         middlewareStack.push(
           trackRateLimit,
+          addProxyInfo,
           handleUpstreamErrors,
           countResponseTokens,
           incrementUsage,
@@ -706,6 +708,38 @@ const copyHttpHeaders: ProxyResHandlerWithBody = async (
   });
 };
 
+/**
+ * Injects metadata into the response, such as the tokenizer used, logging
+ * status, upstream API endpoint used, and whether the input prompt was modified
+ * or transformed.
+ * Only used for non-streaming requests.
+ */
+const addProxyInfo: ProxyResHandlerWithBody = async (
+  _proxyRes,
+  req,
+  res,
+  body
+) => {
+  const { service, inboundApi, outboundApi, tokenizerInfo } = req;
+  const native = inboundApi === outboundApi;
+  const info: any = {
+    logged: config.promptLogging,
+    tokens: tokenizerInfo,
+    service,
+    in_api: inboundApi,
+    out_api: outboundApi,
+    prompt_transformed: !native,
+  };
+
+  if (req.query?.debug?.length) {
+    info.final_request_body = req.signedRequest?.body || req.body;
+  }
+
+  if (typeof body === "object") {
+    body.proxy = info;
+  }
+};
+
 function getAwsErrorType(header: string | string[] | undefined) {
   const val = String(header).match(/^(\w+):?/)?.[1];
   return val || String(header);

src/proxy/mistral-ai.ts

@@ -89,16 +89,7 @@ const mistralAIResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
-  }
-
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  res.status(200).json(body);
+  res.status(200).json({ ...body, proxy: body.proxy });
 };
 
 const mistralAIProxy = createQueueMiddleware({

src/proxy/openai-image.ts

@@ -16,9 +16,7 @@ import {
   ProxyResHandlerWithBody,
 } from "./middleware/response";
 import { generateModelList } from "./openai";
-import {
-  OpenAIImageGenerationResult,
-} from "../shared/file-storage/mirror-generated-image";
+import { OpenAIImageGenerationResult } from "../shared/file-storage/mirror-generated-image";
 
 const KNOWN_MODELS = ["dall-e-2", "dall-e-3"];
 
@@ -44,21 +42,16 @@ const openaiImagesResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
-  }
-
+  let newBody = body;
   if (req.inboundApi === "openai") {
     req.log.info("Transforming OpenAI image response to OpenAI chat format");
-    body = transformResponseForChat(body as OpenAIImageGenerationResult, req);
+    newBody = transformResponseForChat(
+      body as OpenAIImageGenerationResult,
+      req
+    );
   }
 
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  res.status(200).json(body);
+  res.status(200).json({ ...newBody, proxy: body.proxy });
 };
 
 /**

src/proxy/openai.ts

@@ -138,21 +138,13 @@ const openaiResponseHandler: ProxyResHandlerWithBody = async (
     throw new Error("Expected body to be an object");
   }
 
-  if (config.promptLogging) {
-    const host = req.get("host");
-    body.proxy_note = `Prompts are logged on this proxy instance. See ${host} for more information.`;
-  }
-
+  let newBody = body;
   if (req.outboundApi === "openai-text" && req.inboundApi === "openai") {
     req.log.info("Transforming Turbo-Instruct response to Chat format");
-    body = transformTurboInstructResponse(body);
+    newBody = transformTurboInstructResponse(body);
   }
 
-  if (req.tokenizerInfo) {
-    body.proxy_tokenizer = req.tokenizerInfo;
-  }
-
-  res.status(200).json(body);
+  res.status(200).json({ ...newBody, proxy: body.proxy });
 };
 
 /** Only used for non-streaming responses. */