Mirrors
/
oai-reverse-proxy
mirror of https://gitgud.io/khanon/oai-reverse-proxy.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

attempts to improve compatibility with BetterGPT frontend

This commit is contained in:
nai-degen 2023-09-16 11:04:40 -05:00
parent 201f71a989
commit 5e57dbb8f1
2 changed files with 11 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/proxy/middleware/request/transform-outbound-payload.ts
View File

@ -57,9 +57,9 @@ const OpenAIV1ChatCompletionSchema = z.object({
max_tokens: z.coerce
.number()
.int()
.optional()
.nullish()
.default(16)
.transform((v) => Math.min(v, OPENAI_OUTPUT_MAX)),
.transform((v) => Math.min(v ?? OPENAI_OUTPUT_MAX, OPENAI_OUTPUT_MAX)),
frequency_penalty: z.number().optional().default(0),
presence_penalty: z.number().optional().default(0),
logit_bias: z.any().optional(),

10
src/proxy/queue.ts
View File

@ -98,7 +98,7 @@ export function enqueue(req: Request) {
}
req.heartbeatInterval = setInterval(() => {
if (process.env.NODE_ENV === "production") {
req.res!.write(": queue heartbeat\n\n");
if (!req.query.badSseParser) req.res!.write(": queue heartbeat\n\n");
} else {
req.log.info(`Sending heartbeat to request in queue.`);
const partition = getPartitionForRequest(req);
@ -340,6 +340,14 @@ function initStreaming(req: Request) {
res.setHeader("Connection", "keep-alive");
res.setHeader("X-Accel-Buffering", "no"); // nginx-specific fix
res.flushHeaders();
if (req.query.badSseParser) {
// Some clients have a broken SSE parser that doesn't handle comments
// correctly. These clients can pass ?badSseParser=true to
// disable comments in the SSE stream.
return;
}
res.write("\n");
res.write(": joining queue\n\n");
}