138 lines
4.2 KiB
TypeScript
138 lines
4.2 KiB
TypeScript
import { Key, OpenAIKey, keyPool } from "../../../shared/key-management";
|
|
import { isCompletionRequest, isEmbeddingsRequest } from "../common";
|
|
import { ProxyRequestMiddleware } from ".";
|
|
import { assertNever } from "../../../shared/utils";
|
|
|
|
/** Add a key that can service this request to the request object. */
|
|
export const addKey: ProxyRequestMiddleware = (proxyReq, req) => {
|
|
let assignedKey: Key;
|
|
|
|
if (!isCompletionRequest(req)) {
|
|
// Horrible, horrible hack to stop the proxy from complaining about clients
|
|
// not sending a model when they are requesting the list of models (which
|
|
// requires a key, but obviously not a model).
|
|
|
|
// I don't think this is needed anymore since models requests are no longer
|
|
// proxied to the upstream API. Everything going through this is either a
|
|
// completion request or a special case like OpenAI embeddings.
|
|
req.log.warn({ path: req.path }, "addKey called on non-completion request");
|
|
req.body.model = "gpt-3.5-turbo";
|
|
}
|
|
|
|
if (!req.inboundApi || !req.outboundApi) {
|
|
const err = new Error(
|
|
"Request API format missing. Did you forget to add the request preprocessor to your router?"
|
|
);
|
|
req.log.error(
|
|
{ in: req.inboundApi, out: req.outboundApi, path: req.path },
|
|
err.message
|
|
);
|
|
throw err;
|
|
}
|
|
|
|
if (!req.body?.model) {
|
|
throw new Error("You must specify a model with your request.");
|
|
}
|
|
|
|
// TODO: use separate middleware to deal with stream flags
|
|
req.isStreaming = req.body.stream === true || req.body.stream === "true";
|
|
req.body.stream = req.isStreaming;
|
|
|
|
if (req.inboundApi === req.outboundApi) {
|
|
assignedKey = keyPool.get(req.body.model);
|
|
} else {
|
|
switch (req.outboundApi) {
|
|
// If we are translating between API formats we may need to select a model
|
|
// for the user, because the provided model is for the inbound API.
|
|
case "anthropic":
|
|
assignedKey = keyPool.get("claude-v1");
|
|
break;
|
|
case "google-palm":
|
|
assignedKey = keyPool.get("text-bison-001");
|
|
delete req.body.stream;
|
|
break;
|
|
case "openai-text":
|
|
assignedKey = keyPool.get("gpt-3.5-turbo-instruct");
|
|
break;
|
|
case "openai":
|
|
throw new Error(
|
|
"OpenAI Chat as an API translation target is not supported"
|
|
);
|
|
default:
|
|
assertNever(req.outboundApi);
|
|
}
|
|
}
|
|
|
|
req.key = assignedKey;
|
|
req.log.info(
|
|
{
|
|
key: assignedKey.hash,
|
|
model: req.body?.model,
|
|
fromApi: req.inboundApi,
|
|
toApi: req.outboundApi,
|
|
},
|
|
"Assigned key to request"
|
|
);
|
|
|
|
// TODO: KeyProvider should assemble all necessary headers
|
|
switch (assignedKey.service) {
|
|
case "anthropic":
|
|
proxyReq.setHeader("X-API-Key", assignedKey.key);
|
|
break;
|
|
case "openai":
|
|
const key: OpenAIKey = assignedKey as OpenAIKey;
|
|
if (key.organizationId) {
|
|
proxyReq.setHeader("OpenAI-Organization", key.organizationId);
|
|
}
|
|
proxyReq.setHeader("Authorization", `Bearer ${assignedKey.key}`);
|
|
break;
|
|
case "google-palm":
|
|
const originalPath = proxyReq.path;
|
|
proxyReq.path = originalPath.replace(
|
|
/(\?.*)?$/,
|
|
`?key=${assignedKey.key}`
|
|
);
|
|
break;
|
|
case "aws":
|
|
throw new Error(
|
|
"add-key should not be used for AWS security credentials. Use sign-aws-request instead."
|
|
);
|
|
default:
|
|
assertNever(assignedKey.service);
|
|
}
|
|
};
|
|
|
|
/**
|
|
* Special case for embeddings requests which don't go through the normal
|
|
* request pipeline.
|
|
*/
|
|
export const addKeyForEmbeddingsRequest: ProxyRequestMiddleware = (
|
|
proxyReq,
|
|
req
|
|
) => {
|
|
if (!isEmbeddingsRequest(req)) {
|
|
throw new Error(
|
|
"addKeyForEmbeddingsRequest called on non-embeddings request"
|
|
);
|
|
}
|
|
|
|
if (req.inboundApi !== "openai") {
|
|
throw new Error("Embeddings requests must be from OpenAI");
|
|
}
|
|
|
|
req.body.model = "text-embedding-ada-002";
|
|
|
|
const key = keyPool.get("text-embedding-ada-002") as OpenAIKey;
|
|
|
|
req.key = key;
|
|
req.log.info(
|
|
{ key: key.hash, toApi: req.outboundApi },
|
|
"Assigned Turbo key to embeddings request"
|
|
);
|
|
|
|
proxyReq.setHeader("Authorization", `Bearer ${key.key}`);
|
|
if (key.organizationId) {
|
|
proxyReq.setHeader("OpenAI-Organization", key.organizationId);
|
|
}
|
|
};
|