diff --git a/src/proxy/middleware/request/transform-outbound-payload.ts b/src/proxy/middleware/request/transform-outbound-payload.ts index 042ff83..887f39d 100644 --- a/src/proxy/middleware/request/transform-outbound-payload.ts +++ b/src/proxy/middleware/request/transform-outbound-payload.ts @@ -57,9 +57,9 @@ const OpenAIV1ChatCompletionSchema = z.object({ max_tokens: z.coerce .number() .int() - .optional() + .nullish() .default(16) - .transform((v) => Math.min(v, OPENAI_OUTPUT_MAX)), + .transform((v) => Math.min(v ?? OPENAI_OUTPUT_MAX, OPENAI_OUTPUT_MAX)), frequency_penalty: z.number().optional().default(0), presence_penalty: z.number().optional().default(0), logit_bias: z.any().optional(), diff --git a/src/proxy/queue.ts b/src/proxy/queue.ts index 3d8e811..91eb4a2 100644 --- a/src/proxy/queue.ts +++ b/src/proxy/queue.ts @@ -98,7 +98,7 @@ export function enqueue(req: Request) { } req.heartbeatInterval = setInterval(() => { if (process.env.NODE_ENV === "production") { - req.res!.write(": queue heartbeat\n\n"); + if (!req.query.badSseParser) req.res!.write(": queue heartbeat\n\n"); } else { req.log.info(`Sending heartbeat to request in queue.`); const partition = getPartitionForRequest(req); @@ -340,6 +340,14 @@ function initStreaming(req: Request) { res.setHeader("Connection", "keep-alive"); res.setHeader("X-Accel-Buffering", "no"); // nginx-specific fix res.flushHeaders(); + + if (req.query.badSseParser) { + // Some clients have a broken SSE parser that doesn't handle comments + // correctly. These clients can pass ?badSseParser=true to + // disable comments in the SSE stream. + return; + } + res.write("\n"); res.write(": joining queue\n\n"); }