spoofs response for SillyTavern test messages

src/proxy/middleware/request/preprocessor-factory.ts

@@ -1,4 +1,5 @@
 import { RequestHandler } from "express";
+import { ZodIssue } from "zod";
 import { initializeSseStream } from "../../../shared/streaming";
 import { classifyErrorAndSend } from "../common";
 import {
@@ -9,7 +10,6 @@ import {
   transformOutboundPayload,
   languageFilter,
 } from ".";
-import { ZodIssue } from "zod";
 
 type RequestPreprocessorOptions = {
   /**
@@ -71,6 +71,9 @@ async function executePreprocessors(
   preprocessors: RequestPreprocessor[],
   [req, res, next]: Parameters<RequestHandler>
 ) {
+  handleTestMessage(req, res, next);
+  if (res.headersSent) return;
+
   try {
     for (const preprocessor of preprocessors) {
       await preprocessor(req);
@@ -99,3 +102,49 @@ async function executePreprocessors(
     classifyErrorAndSend(error as Error, req, res);
   }
 }
+
+/**
+ * Bypasses the API call and returns a test message response if the request body
+ * is a known test message from SillyTavern. Otherwise these messages just waste
+ * API request quota and confuse users when the proxy is busy, because ST always
+ * makes them with `stream: false` (which is not allowed when the proxy is busy)
+ */
+const handleTestMessage: RequestHandler = (req, res) => {
+  const { method, body } = req;
+  if (method !== "POST") {
+    return;
+  }
+  
+  if (isTestMessage(body)) {
+    req.log.info({ body }, "Received test message. Skipping API call.");
+    res.json({
+      id: "test-message",
+      object: "chat.completion",
+      created: Date.now(),
+      model: body.model,
+      choices: [
+        {
+          message: { role: "assistant", content: "Hello!" },
+          finish_reason: "stop",
+          index: 0,
+        },
+      ],
+      proxy_note:
+        "This response was generated by the proxy's test message handler and did not go to the API.",
+    });
+  }
+};
+
+function isTestMessage(body: any) {
+  const { messages, prompt } = body;
+
+  if (messages) {
+    return (
+      messages.length === 1 &&
+      messages[0].role === "user" &&
+      messages[0].content === "Hi"
+    );
+  } else {
+    return prompt?.trim() === "Human: Hi\n\nAssistant:";
+  }
+}

src/proxy/routes.ts

@@ -45,6 +45,11 @@ proxyRouter.get("*", (req, res, next) => {
     next();
   }
 });
+// Handle 404s.
+proxyRouter.use((_req, res) => {
+  res.status(404).json({ error: "Not found" });
+});
+
 export { proxyRouter as proxyRouter };
 
 function addV1(req: Request, res: Response, next: NextFunction) {