From 6cf029112e97f1c179134bed6f6b55e40743bb6c Mon Sep 17 00:00:00 2001 From: nai-degen Date: Wed, 13 Mar 2024 20:47:57 -0500 Subject: [PATCH] adds Anthropic's SOTA Haiku model; misc code cleanup --- package-lock.json | 281 ++++++++---------- package.json | 4 +- src/proxy/anthropic.ts | 1 + src/proxy/aws.ts | 1 + src/proxy/middleware/common.ts | 37 ++- .../middleware/request/onproxyreq/add-key.ts | 46 ++- src/service-info.ts | 9 +- src/shared/api-schemas/anthropic.ts | 5 +- src/shared/errors.ts | 6 + .../key-management/anthropic/provider.ts | 17 +- src/shared/key-management/aws/checker.ts | 45 +-- src/shared/key-management/aws/provider.ts | 21 +- src/shared/key-management/azure/provider.ts | 10 +- .../key-management/google-ai/provider.ts | 8 +- src/shared/key-management/index.ts | 13 +- src/shared/key-management/key-pool.ts | 13 +- .../key-management/mistral-ai/provider.ts | 4 +- src/shared/key-management/openai/checker.ts | 2 +- src/shared/key-management/openai/provider.ts | 24 +- src/user/web/views/user_lookup.ejs | 16 +- 20 files changed, 267 insertions(+), 296 deletions(-) diff --git a/package-lock.json b/package-lock.json index 1e2d645..8d3d4bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,7 +35,7 @@ "node-schedule": "^2.1.1", "pino": "^8.11.0", "pino-http": "^8.3.3", - "sanitize-html": "^2.11.0", + "sanitize-html": "2.12.1", "sharp": "^0.32.6", "showdown": "^2.1.0", "source-map-support": "^0.5.21", @@ -65,7 +65,7 @@ "pino-pretty": "^10.2.3", "prettier": "^3.0.3", "ts-node": "^10.9.1", - "typescript": "^5.1.3" + "typescript": "^5.4.2" }, "engines": { "node": ">=18.0.0" @@ -154,9 +154,9 @@ } }, "node_modules/@babel/parser": { - "version": "7.22.7", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.22.7.tgz", - "integrity": "sha512-7NF8pOkHP5o2vpmGgNGcfAeCvOYhGLyA3Z4eBQkT1RJlWu47n63bCs93QfJ2hIAFCil7L5P2IWhs1oToVgrL0Q==", + "version": "7.24.0", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.0.tgz", + "integrity": "sha512-QuP/FxEAzMSjXygs8v4N9dvdXzEHN4W1oF3PxuWAtPo08UdM17u89RDMgjLn/mlc56iM0HlLmVkO/wgR+rDgHg==", "optional": true, "bin": { "parser": "bin/babel-parser.js" @@ -611,15 +611,15 @@ } }, "node_modules/@google-cloud/firestore": { - "version": "6.6.1", - "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-6.6.1.tgz", - "integrity": "sha512-Z41j2h0mrgBH9qNIVmbRLqGKc6XmdJtWipeKwdnGa/bPTP1gn2SGTrYyWnpfsLMEtzKSYieHPSkAFp5kduF2RA==", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/@google-cloud/firestore/-/firestore-6.8.0.tgz", + "integrity": "sha512-JRpk06SmZXLGz0pNx1x7yU3YhkUXheKgH5hbDZ4kMsdhtfV5qPLJLRI4wv69K0cZorIk+zTMOwptue7hizo0eA==", "optional": true, "dependencies": { "fast-deep-equal": "^3.1.1", "functional-red-black-tree": "^1.0.1", "google-gax": "^3.5.7", - "protobufjs": "^7.0.0" + "protobufjs": "^7.2.5" }, "engines": { "node": ">=12.0.0" @@ -706,9 +706,9 @@ } }, "node_modules/@grpc/grpc-js": { - "version": "1.8.17", - "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.17.tgz", - "integrity": "sha512-DGuSbtMFbaRsyffMf+VEkVu8HkSXEUfO3UyGJNtqxW9ABdtTIA+2UXAJpwbJS+xfQxuwqLUeELmL6FuZkOqPxw==", + "version": "1.8.21", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.8.21.tgz", + "integrity": "sha512-KeyQeZpxeEBSqFVTi3q2K7PiPXmgBfECc4updA1ejCLjYmoAlvvM3ZMp5ztTDUCUQmoY3CpDxvchjO1+rFkoHg==", "optional": true, "dependencies": { "@grpc/proto-loader": "^0.7.0", @@ -719,15 +719,14 @@ } }, "node_modules/@grpc/proto-loader": { - "version": "0.7.7", - "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.7.tgz", - "integrity": "sha512-1TIeXOi8TuSCQprPItwoMymZXxWT0CPxUhkrkeCUH+D8U7QDwQ6b7SUz2MaLuWM2llT+J/TVFLmQI5KtML3BhQ==", + "version": "0.7.10", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.7.10.tgz", + "integrity": "sha512-CAqDfoaQ8ykFd9zqBDn4k6iWT9loLAlc2ETmDFS9JCD70gDcnA4L3AFEo2iV7KyAtAAHFW9ftq1Fz+Vsgq80RQ==", "optional": true, "dependencies": { - "@types/long": "^4.0.1", "lodash.camelcase": "^4.3.0", - "long": "^4.0.0", - "protobufjs": "^7.0.0", + "long": "^5.0.0", + "protobufjs": "^7.2.4", "yargs": "^17.7.2" }, "bin": { @@ -763,9 +762,9 @@ } }, "node_modules/@jsdoc/salty": { - "version": "0.2.5", - "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.5.tgz", - "integrity": "sha512-TfRP53RqunNe2HBobVBJ0VLhK1HbfvBYeTC1ahnN64PWvyYyGebmMiPkuwvD9fpw2ZbkoPb8Q7mwy0aR8Z9rvw==", + "version": "0.2.7", + "resolved": "https://registry.npmjs.org/@jsdoc/salty/-/salty-0.2.7.tgz", + "integrity": "sha512-mh8LbS9d4Jq84KLw8pzho7XC2q2/IJGiJss3xwRoLD1A+EE16SjN4PfaG4jRCzKegTFLlN0Zd8SdUPE6XdoPFg==", "optional": true, "dependencies": { "lodash": "^4.17.21" @@ -1110,9 +1109,9 @@ } }, "node_modules/@types/linkify-it": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.2.tgz", - "integrity": "sha512-HZQYqbiFVWufzCwexrvh694SOim8z2d+xJl5UNamcvQFejLY/2YUtzXHYi3cHdI7PMlS8ejH2slRAOJQ32aNbA==", + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/@types/linkify-it/-/linkify-it-3.0.5.tgz", + "integrity": "sha512-yg6E+u0/+Zjva+buc3EIb+29XEg4wltq7cSmd4Uc2EE/1nUVmxyzpX6gUXD0V8jIrG0r7YeOGVIbYRkxeooCtw==", "optional": true }, "node_modules/@types/long": { @@ -1132,9 +1131,9 @@ } }, "node_modules/@types/mdurl": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.2.tgz", - "integrity": "sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@types/mdurl/-/mdurl-1.0.5.tgz", + "integrity": "sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==", "optional": true }, "node_modules/@types/mime": { @@ -2470,61 +2469,10 @@ "node": ">=4.0" } }, - "node_modules/escodegen/node_modules/levn": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", - "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", - "optional": true, - "dependencies": { - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/escodegen/node_modules/optionator": { - "version": "0.8.3", - "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", - "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "optional": true, - "dependencies": { - "deep-is": "~0.1.3", - "fast-levenshtein": "~2.0.6", - "levn": "~0.3.0", - "prelude-ls": "~1.1.2", - "type-check": "~0.3.2", - "word-wrap": "~1.2.3" - }, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/escodegen/node_modules/prelude-ls": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", - "optional": true, - "engines": { - "node": ">= 0.8.0" - } - }, - "node_modules/escodegen/node_modules/type-check": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", - "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", - "optional": true, - "dependencies": { - "prelude-ls": "~1.1.2" - }, - "engines": { - "node": ">= 0.8.0" - } - }, "node_modules/eslint-visitor-keys": { - "version": "3.4.1", - "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.1.tgz", - "integrity": "sha512-pZnmmLwYzf+kWaM/Qgrvpen51upAktaaiI01nsJD/Yr3lMOdNtq0cxkrrg16w64VtisN6okbs7Q8AfGqj4c9fA==", + "version": "3.4.3", + "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz", + "integrity": "sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag==", "optional": true, "engines": { "node": "^12.22.0 || ^14.17.0 || >=16.0.0" @@ -2534,9 +2482,9 @@ } }, "node_modules/espree": { - "version": "9.6.0", - "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.0.tgz", - "integrity": "sha512-1FH/IiruXZ84tpUlm0aCUEwMl2Ho5ilqVh0VvQXw+byAz/4SAciyHLlfmL5WYqsvD38oymdUwBss0LtK8m4s/A==", + "version": "9.6.1", + "resolved": "https://registry.npmjs.org/espree/-/espree-9.6.1.tgz", + "integrity": "sha512-oruZaFkjorTpF32kDSI5/75ViwGeZginGGy2NoOSg3Q9bnwlnmDm4HLnkl0RE3n+njDXR037aY1+x58Z/zFdwQ==", "optional": true, "dependencies": { "acorn": "^8.9.0", @@ -2799,9 +2747,9 @@ } }, "node_modules/firebase-admin": { - "version": "11.10.1", - "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-11.10.1.tgz", - "integrity": "sha512-atv1E6GbuvcvWaD3eHwrjeP5dAVs+EaHEJhu9CThMzPY6In8QYDiUR6tq5SwGl4SdA/GcAU0nhwWc/FSJsAzfQ==", + "version": "11.11.1", + "resolved": "https://registry.npmjs.org/firebase-admin/-/firebase-admin-11.11.1.tgz", + "integrity": "sha512-UyEbq+3u6jWzCYbUntv/HuJiTixwh36G1R9j0v71mSvGAx/YZEWEW7uSGLYxBYE6ckVRQoKMr40PYUEzrm/4dg==", "dependencies": { "@fastify/busboy": "^1.2.1", "@firebase/database-compat": "^0.3.4", @@ -2816,7 +2764,7 @@ "node": ">=14" }, "optionalDependencies": { - "@google-cloud/firestore": "^6.6.0", + "@google-cloud/firestore": "^6.8.0", "@google-cloud/storage": "^6.9.5" } }, @@ -3056,6 +3004,30 @@ "node": ">=12" } }, + "node_modules/google-gax/node_modules/protobufjs": { + "version": "7.2.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", + "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "hasInstallScript": true, + "optional": true, + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/google-p12-pem": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", @@ -3696,6 +3668,19 @@ "graceful-fs": "^4.1.9" } }, + "node_modules/levn": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", + "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", + "optional": true, + "dependencies": { + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/limiter": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/limiter/-/limiter-1.1.5.tgz", @@ -3727,9 +3712,9 @@ "integrity": "sha512-H5ZhCF25riFd9uB5UCkVKo61m3S/xZk1x4wA6yp/L3RFP6Z/eHH1ymQcGLo7J3GMPfm0V/7m1tryHuGVxpqEBQ==" }, "node_modules/long": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/long/-/long-4.0.0.tgz", - "integrity": "sha512-XsP+KhQif4bjX1kbuSiySJFNAehNxgLb6hPRGJ9QsUr8ajHkuXGdrHmFUTUUXhDwVX2R5bY4JNZEwbUiMhV+MA==", + "version": "5.2.3", + "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", + "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", "optional": true }, "node_modules/long-timeout": { @@ -4258,6 +4243,23 @@ "wrappy": "1" } }, + "node_modules/optionator": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", + "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", + "optional": true, + "dependencies": { + "deep-is": "~0.1.3", + "fast-levenshtein": "~2.0.6", + "levn": "~0.3.0", + "prelude-ls": "~1.1.2", + "type-check": "~0.3.2", + "word-wrap": "~1.2.3" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/p-limit": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", @@ -4477,6 +4479,15 @@ "node": ">=6" } }, + "node_modules/prelude-ls": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", + "optional": true, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/prettier": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.0.3.tgz", @@ -4523,9 +4534,9 @@ } }, "node_modules/protobufjs": { - "version": "7.2.4", - "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.4.tgz", - "integrity": "sha512-AT+RJgD2sH8phPmCf7OUZR8xGdcJRga4+1cOaXJ64hvcSkVhNcRHOwIxUatPH15+nj59WAGTDv3LSGZPEQbJaQ==", + "version": "7.2.6", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.2.6.tgz", + "integrity": "sha512-dgJaEDDL6x8ASUZ1YqWciTRrdOuYNzoOf27oHNfdyvKqHr5i0FV7FSLU+aIeFjyFgVxrpTOtQUi0BLLBymZaBw==", "hasInstallScript": true, "optional": true, "dependencies": { @@ -4574,12 +4585,6 @@ "protobufjs": "^7.0.0" } }, - "node_modules/protobufjs/node_modules/long": { - "version": "5.2.3", - "resolved": "https://registry.npmjs.org/long/-/long-5.2.3.tgz", - "integrity": "sha512-lcHwpNoggQTObv5apGNCTdJrO69eHOZMi4BNC+rTLER8iHAqGrUVeLh/irVIM7zTw2bOXA8T6uNPeujwOLg/2Q==", - "optional": true - }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -4794,41 +4799,6 @@ "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==", "optional": true }, - "node_modules/rimraf": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", - "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", - "optional": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/rimraf/node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "optional": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, "node_modules/rxjs": { "version": "7.8.0", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.0.tgz", @@ -4871,9 +4841,9 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, "node_modules/sanitize-html": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.11.0.tgz", - "integrity": "sha512-BG68EDHRaGKqlsNjJ2xUB7gpInPA8gVx/mvjO743hZaeMCZ2DwzW7xvsqZ+KNU4QKwj86HJ3uu2liISf2qBBUA==", + "version": "2.12.1", + "resolved": "https://registry.npmjs.org/sanitize-html/-/sanitize-html-2.12.1.tgz", + "integrity": "sha512-Plh+JAn0UVDpBRP/xEjsk+xDCoOvMBwQUf/K+/cBAVuTbtX8bj2VB7S1sL1dssVpykqp0/KPSesHrqXtokVBpA==", "dependencies": { "deepmerge": "^4.2.2", "escape-string-regexp": "^4.0.0", @@ -5341,15 +5311,12 @@ "integrity": "sha512-gF8ndTCNu7WcRFbl1UUWaFIB4CTXmHzS3tRYdyUYF7x3C6YR6Evoao4zhKDmWIwv2PzNbzoQMV8Pxt+17lEDbA==" }, "node_modules/tmp": { - "version": "0.2.1", - "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.1.tgz", - "integrity": "sha512-76SUhtfqR2Ijn+xllcI5P1oyannHNHByD80W1q447gU3mp9G9PSpGdWmjUOHRDPiHYacIk66W7ubDTuPF3BEtQ==", + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", "optional": true, - "dependencies": { - "rimraf": "^3.0.0" - }, "engines": { - "node": ">=8.17.0" + "node": ">=14.14" } }, "node_modules/to-regex-range": { @@ -5456,6 +5423,18 @@ "node": "*" } }, + "node_modules/type-check": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", + "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", + "optional": true, + "dependencies": { + "prelude-ls": "~1.1.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/type-is": { "version": "1.6.18", "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", @@ -5474,9 +5453,9 @@ "integrity": "sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==" }, "node_modules/typescript": { - "version": "5.1.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.3.tgz", - "integrity": "sha512-XH627E9vkeqhlZFQuL+UsyAXEnibT0kWR2FWONlr4sTjvxyJYnyefgrkyECLzM5NenmKzRAy2rR/OlYLA1HkZw==", + "version": "5.4.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.2.tgz", + "integrity": "sha512-+2/g0Fds1ERlP6JsakQQDXjZdZMM+rqpamFZJEKh4kwTIn3iDkgKtby0CeNd5ATNZ4Ry1ax15TMx0W2V+miizQ==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -5619,9 +5598,9 @@ } }, "node_modules/word-wrap": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.4.tgz", - "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", "optional": true, "engines": { "node": ">=0.10.0" diff --git a/package.json b/package.json index 1c24127..532952c 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "node-schedule": "^2.1.1", "pino": "^8.11.0", "pino-http": "^8.3.3", - "sanitize-html": "^2.11.0", + "sanitize-html": "2.12.1", "sharp": "^0.32.6", "showdown": "^2.1.0", "source-map-support": "^0.5.21", @@ -73,7 +73,7 @@ "pino-pretty": "^10.2.3", "prettier": "^3.0.3", "ts-node": "^10.9.1", - "typescript": "^5.1.3" + "typescript": "^5.4.2" }, "overrides": { "google-gax": "^3.6.1", diff --git a/src/proxy/anthropic.ts b/src/proxy/anthropic.ts index 12c30e6..926cd8c 100644 --- a/src/proxy/anthropic.ts +++ b/src/proxy/anthropic.ts @@ -43,6 +43,7 @@ const getModelsResponse = () => { "claude-2", "claude-2.0", "claude-2.1", + "claude-3-haiku-20240307", "claude-3-opus-20240229", "claude-3-sonnet-20240229", ]; diff --git a/src/proxy/aws.ts b/src/proxy/aws.ts index 41c8878..bc70a2f 100644 --- a/src/proxy/aws.ts +++ b/src/proxy/aws.ts @@ -35,6 +35,7 @@ const getModelsResponse = () => { const variants = [ "anthropic.claude-v2", "anthropic.claude-v2:1", + "anthropic.claude-3-haiku-20240307-v1:0", "anthropic.claude-3-sonnet-20240229-v1:0", ]; diff --git a/src/proxy/middleware/common.ts b/src/proxy/middleware/common.ts index bf478ec..918af61 100644 --- a/src/proxy/middleware/common.ts +++ b/src/proxy/middleware/common.ts @@ -1,4 +1,5 @@ import { Request, Response } from "express"; +import http from "http"; import httpProxy from "http-proxy"; import { ZodError } from "zod"; import { generateErrorMessage } from "zod-error"; @@ -115,14 +116,34 @@ function classifyError(err: Error): { switch (err.constructor.name) { case "HttpError": - if ((err as HttpError).status === 402) { - return { - statusCode: 402, - statusMessage: "No Keys Available", - userMessage: err.message, - type: "proxy_no_keys_available", - }; - } else return defaultError; + const statusCode = (err as HttpError).status; + return { + statusCode, + statusMessage: `HTTP ${statusCode} ${http.STATUS_CODES[statusCode]}`, + userMessage: `Reverse proxy error: ${err.message}`, + type: "proxy_http_error", + }; + case "BadRequestError": + return { + statusCode: 400, + statusMessage: "Bad Request", + userMessage: `Request is not valid. (${err.message})`, + type: "proxy_bad_request", + }; + case "NotFoundError": + return { + statusCode: 404, + statusMessage: "Not Found", + userMessage: `Requested resource not found. (${err.message})`, + type: "proxy_not_found", + }; + case "PaymentRequiredError": + return { + statusCode: 402, + statusMessage: "No Keys Available", + userMessage: err.message, + type: "proxy_no_keys_available", + }; case "ZodError": const userMessage = generateErrorMessage((err as ZodError).issues, { prefix: "Request validation failed. ", diff --git a/src/proxy/middleware/request/onproxyreq/add-key.ts b/src/proxy/middleware/request/onproxyreq/add-key.ts index fd0b714..5aacf69 100644 --- a/src/proxy/middleware/request/onproxyreq/add-key.ts +++ b/src/proxy/middleware/request/onproxyreq/add-key.ts @@ -3,62 +3,54 @@ import { isEmbeddingsRequest } from "../../common"; import { HPMRequestCallback } from "../index"; import { assertNever } from "../../../../shared/utils"; -/** Add a key that can service this request to the request object. */ export const addKey: HPMRequestCallback = (proxyReq, req) => { let assignedKey: Key; + const { service, inboundApi, outboundApi, body } = req; - if (!req.inboundApi || !req.outboundApi) { + if (!inboundApi || !outboundApi) { const err = new Error( "Request API format missing. Did you forget to add the request preprocessor to your router?" ); - req.log.error( - { in: req.inboundApi, out: req.outboundApi, path: req.path }, - err.message - ); + req.log.error({ inboundApi, outboundApi, path: req.path }, err.message); throw err; } - if (!req.body?.model) { + if (!body?.model) { throw new Error("You must specify a model with your request."); } - if (req.inboundApi === req.outboundApi) { - assignedKey = keyPool.get(req.body.model, req.service); + if (inboundApi === outboundApi) { + assignedKey = keyPool.get(body.model, service); } else { - switch (req.outboundApi) { + switch (outboundApi) { // If we are translating between API formats we may need to select a model // for the user, because the provided model is for the inbound API. + // TODO: This whole else condition is probably no longer needed since API + // translation now reassigns the model earlier in the request pipeline. case "anthropic-chat": case "anthropic-text": - assignedKey = keyPool.get("claude-v1", req.service); + assignedKey = keyPool.get("claude-v1", service); break; case "openai-text": - assignedKey = keyPool.get("gpt-3.5-turbo-instruct", req.service); + assignedKey = keyPool.get("gpt-3.5-turbo-instruct", service); + break; + case "openai-image": + assignedKey = keyPool.get("dall-e-3", service); break; case "openai": - throw new Error( - "OpenAI Chat as an API translation target is not supported" - ); case "google-ai": - throw new Error("add-key should not be used for this model."); case "mistral-ai": - throw new Error("Mistral AI should never be translated"); - case "openai-image": - assignedKey = keyPool.get("dall-e-3", req.service); - break; + throw new Error( + `add-key should not be called for outbound API ${outboundApi}` + ); default: - assertNever(req.outboundApi); + assertNever(outboundApi); } } req.key = assignedKey; req.log.info( - { - key: assignedKey.hash, - model: req.body?.model, - fromApi: req.inboundApi, - toApi: req.outboundApi, - }, + { key: assignedKey.hash, model: body.model, inboundApi, outboundApi }, "Assigned key to request" ); diff --git a/src/service-info.ts b/src/service-info.ts index 7e8d04c..cd0b572 100644 --- a/src/service-info.ts +++ b/src/service-info.ts @@ -52,6 +52,7 @@ type ModelAggregates = { pozzed?: number; awsLogged?: number; awsSonnet?: number; + awsHaiku?: number; queued: number; queueTime: string; tokens: number; @@ -82,7 +83,11 @@ type AnthropicInfo = BaseFamilyInfo & { prefilledKeys?: number; overQuotaKeys?: number; }; -type AwsInfo = BaseFamilyInfo & { privacy?: string; sonnetKeys?: number }; +type AwsInfo = BaseFamilyInfo & { + privacy?: string; + sonnetKeys?: number; + haikuKeys?: number; +}; // prettier-ignore export type ServiceInfo = { @@ -387,6 +392,7 @@ function addKeyToAggregates(k: KeyPoolKey) { increment(modelStats, `${family}__revoked`, k.isRevoked ? 1 : 0); increment(modelStats, `${family}__tokens`, k["aws-claudeTokens"]); increment(modelStats, `${family}__awsSonnet`, k.sonnetEnabled ? 1 : 0); + increment(modelStats, `${family}__awsHaiku`, k.haikuEnabled ? 1 : 0); // Ignore revoked keys for aws logging stats, but include keys where the // logging status is unknown. @@ -435,6 +441,7 @@ function getInfoForFamily(family: ModelFamily): BaseFamilyInfo { break; case "aws": info.sonnetKeys = modelStats.get(`${family}__awsSonnet`) || 0; + info.haikuKeys = modelStats.get(`${family}__awsHaiku`) || 0; const logged = modelStats.get(`${family}__awsLogged`) || 0; if (logged > 0) { info.privacy = config.allowAwsLogging diff --git a/src/shared/api-schemas/anthropic.ts b/src/shared/api-schemas/anthropic.ts index dd56fc1..318ab35 100644 --- a/src/shared/api-schemas/anthropic.ts +++ b/src/shared/api-schemas/anthropic.ts @@ -1,5 +1,6 @@ import { z } from "zod"; import { config } from "../../config"; +import { BadRequestError } from "../errors"; import { flattenOpenAIMessageContent, OpenAIChatMessage, @@ -240,7 +241,7 @@ export const transformAnthropicTextToAnthropicChat: APIFormatTransformer< function validateAnthropicTextPrompt(prompt: string) { if (!prompt.includes("\n\nHuman:") || !prompt.includes("\n\nAssistant:")) { - throw new Error( + throw new BadRequestError( "Prompt must contain at least one human and one assistant message." ); } @@ -248,7 +249,7 @@ function validateAnthropicTextPrompt(prompt: string) { const firstHuman = prompt.indexOf("\n\nHuman:"); const firstAssistant = prompt.indexOf("\n\nAssistant:"); if (firstAssistant < firstHuman) { - throw new Error( + throw new BadRequestError( "First Assistant message must come after the first Human message." ); } diff --git a/src/shared/errors.ts b/src/shared/errors.ts index d9a1a46..7e14ea8 100644 --- a/src/shared/errors.ts +++ b/src/shared/errors.ts @@ -11,6 +11,12 @@ export class BadRequestError extends HttpError { } } +export class PaymentRequiredError extends HttpError { + constructor(message: string) { + super(402, message); + } +} + export class ForbiddenError extends HttpError { constructor(message: string) { super(403, message); diff --git a/src/shared/key-management/anthropic/provider.ts b/src/shared/key-management/anthropic/provider.ts index 96d2c2f..020304a 100644 --- a/src/shared/key-management/anthropic/provider.ts +++ b/src/shared/key-management/anthropic/provider.ts @@ -4,18 +4,7 @@ import { config } from "../../../config"; import { logger } from "../../../logger"; import { AnthropicModelFamily, getClaudeModelFamily } from "../../models"; import { AnthropicKeyChecker } from "./checker"; -import { HttpError } from "../../errors"; - -// https://docs.anthropic.com/claude/reference/selecting-a-model -export type AnthropicModel = - | "claude-instant-v1" - | "claude-instant-v1-100k" - | "claude-v1" - | "claude-v1-100k" - | "claude-2" - | "claude-2.1" - | "claude-3-opus-20240229" // new expensive model - | "claude-3-sonnet-20240229"; // new cheap claude2 sidegrade +import { HttpError, PaymentRequiredError } from "../../errors"; export type AnthropicKeyUpdate = Omit< Partial, @@ -126,12 +115,12 @@ export class AnthropicKeyProvider implements KeyProvider { return this.keys.map((k) => Object.freeze({ ...k, key: undefined })); } - public get(_model: AnthropicModel) { + public get(_model: string) { // Currently, all Anthropic keys have access to all models. This will almost // certainly change when they move out of beta later this year. const availableKeys = this.keys.filter((k) => !k.isDisabled); if (availableKeys.length === 0) { - throw new HttpError(402, "No Anthropic keys available."); + throw new PaymentRequiredError("No Anthropic keys available."); } // (largely copied from the OpenAI provider, without trial key support) diff --git a/src/shared/key-management/aws/checker.ts b/src/shared/key-management/aws/checker.ts index 5175845..16d78b5 100644 --- a/src/shared/key-management/aws/checker.ts +++ b/src/shared/key-management/aws/checker.ts @@ -47,24 +47,22 @@ export class AwsKeyChecker extends KeyCheckerBase { protected async testKeyOrFail(key: AwsBedrockKey) { // Only check models on startup. For now all models must be available to // the proxy because we don't route requests to different keys. - const modelChecks: Promise[] = []; + let checks: Promise[] = []; const isInitialCheck = !key.lastChecked; if (isInitialCheck) { - modelChecks.push(this.invokeModel("anthropic.claude-v2:1", key)); - modelChecks.push( - this.invokeModel("anthropic.claude-3-sonnet-20240229-v1:0", key) - ); + checks = [ + this.invokeModel("anthropic.claude-v2", key), + this.invokeModel("anthropic.claude-3-sonnet-20240229-v1:0", key), + this.invokeModel("anthropic.claude-3-haiku-20240307-v1:0", key), + this.checkLoggingConfiguration(key), + ]; } - await Promise.all(modelChecks); - await this.checkLoggingConfiguration(key); + const [_claudeV2, sonnet, haiku, _logging] = await Promise.all(checks); + this.updateKey(key.hash, { sonnetEnabled: sonnet, haikuEnabled: haiku }); this.log.info( - { - key: key.hash, - models: key.modelFamilies, - logged: key.awsLoggingStatus, - }, + { key: key.hash, sonnet, haiku, logged: key.awsLoggingStatus }, "Checked key." ); } @@ -129,6 +127,11 @@ export class AwsKeyChecker extends KeyCheckerBase { this.updateKey(key.hash, { lastChecked: next }); } + /** + * Attempt to invoke the given model with the given key. Returns true if the + * key has access to the model, false if it does not. Throws an error if the + * key is disabled. + */ private async invokeModel(model: string, key: AwsBedrockKey) { const creds = AwsKeyChecker.getCredentialsFromKey(key); // This is not a valid invocation payload, but a 400 response indicates that @@ -157,13 +160,11 @@ export class AwsKeyChecker extends KeyCheckerBase { const errorMessage = data?.message; // We only allow one type of 403 error, and we only allow it for one model. - if (status === 403 && errorMessage?.match(/access to the model with the specified model ID/)) { - this.log.warn( - { key: key.hash, errorType, data, status, model }, - "Key does not have access to Claude 3 Sonnet." - ); - this.updateKey(key.hash, { sonnetEnabled: false }); - return; + if ( + status === 403 && + errorMessage?.match(/access to the model with the specified model ID/) + ) { + return false; } // We're looking for a specific error type and message here @@ -181,9 +182,10 @@ export class AwsKeyChecker extends KeyCheckerBase { } this.log.debug( - { key: key.hash, errorType, data, status, model }, - "Liveness test complete." + { key: key.hash, model, errorType, data, status }, + "AWS InvokeModel test successful." ); + return true; } private async checkLoggingConfiguration(key: AwsBedrockKey) { @@ -217,6 +219,7 @@ export class AwsKeyChecker extends KeyCheckerBase { } this.updateKey(key.hash, { awsLoggingStatus: result }); + return !!result; } static errorIsAwsError(error: AxiosError): error is AxiosError { diff --git a/src/shared/key-management/aws/provider.ts b/src/shared/key-management/aws/provider.ts index 0359453..6b2a2f2 100644 --- a/src/shared/key-management/aws/provider.ts +++ b/src/shared/key-management/aws/provider.ts @@ -4,13 +4,7 @@ import { config } from "../../../config"; import { logger } from "../../../logger"; import type { AwsBedrockModelFamily } from "../../models"; import { AwsKeyChecker } from "./checker"; -import { HttpError } from "../../errors"; - -// https://docs.aws.amazon.com/bedrock/latest/userguide/model-ids-arns.html -export type AwsBedrockModel = - | "anthropic.claude-v1" - | "anthropic.claude-v2" - | "anthropic.claude-instant-v1"; +import { PaymentRequiredError } from "../../errors"; type AwsBedrockKeyUsage = { [K in AwsBedrockModelFamily as `${K}Tokens`]: number; @@ -31,6 +25,7 @@ export interface AwsBedrockKey extends Key, AwsBedrockKeyUsage { */ awsLoggingStatus: "unknown" | "disabled" | "enabled"; sonnetEnabled: boolean; + haikuEnabled: boolean; } /** @@ -81,6 +76,7 @@ export class AwsBedrockKeyProvider implements KeyProvider { .slice(0, 8)}`, lastChecked: 0, sonnetEnabled: true, + haikuEnabled: false, ["aws-claudeTokens"]: 0, }; this.keys.push(newKey); @@ -99,20 +95,21 @@ export class AwsBedrockKeyProvider implements KeyProvider { return this.keys.map((k) => Object.freeze({ ...k, key: undefined })); } - public get(model: AwsBedrockModel) { + public get(model: string) { const availableKeys = this.keys.filter((k) => { const isNotLogged = k.awsLoggingStatus === "disabled"; const needsSonnet = model.includes("sonnet"); + const needsHaiku = model.includes("haiku"); return ( !k.isDisabled && (isNotLogged || config.allowAwsLogging) && - (k.sonnetEnabled || !needsSonnet) + (k.sonnetEnabled || !needsSonnet) && + (k.haikuEnabled || !needsHaiku) ); }); if (availableKeys.length === 0) { - throw new HttpError( - 402, - "No keys available for this model. This proxy might not have Claude 3 Sonnet keys available." + throw new PaymentRequiredError( + `No AWS Bedrock keys available for model ${model}` ); } diff --git a/src/shared/key-management/azure/provider.ts b/src/shared/key-management/azure/provider.ts index 4f9d484..8babd08 100644 --- a/src/shared/key-management/azure/provider.ts +++ b/src/shared/key-management/azure/provider.ts @@ -1,15 +1,12 @@ import crypto from "crypto"; import { Key, KeyProvider } from ".."; import { config } from "../../../config"; -import { HttpError } from "../../errors"; +import { PaymentRequiredError } from "../../errors"; import { logger } from "../../../logger"; import type { AzureOpenAIModelFamily } from "../../models"; import { getAzureOpenAIModelFamily } from "../../models"; -import { OpenAIModel } from "../openai/provider"; import { AzureOpenAIKeyChecker } from "./checker"; -export type AzureOpenAIModel = OpenAIModel; - type AzureOpenAIKeyUsage = { [K in AzureOpenAIModelFamily as `${K}Tokens`]: number; }; @@ -96,14 +93,13 @@ export class AzureOpenAIKeyProvider implements KeyProvider { return this.keys.map((k) => Object.freeze({ ...k, key: undefined })); } - public get(model: AzureOpenAIModel) { + public get(model: string) { const neededFamily = getAzureOpenAIModelFamily(model); const availableKeys = this.keys.filter( (k) => !k.isDisabled && k.modelFamilies.includes(neededFamily) ); if (availableKeys.length === 0) { - throw new HttpError( - 402, + throw new PaymentRequiredError( `No keys available for model family '${neededFamily}'.` ); } diff --git a/src/shared/key-management/google-ai/provider.ts b/src/shared/key-management/google-ai/provider.ts index 97bbd07..20f67fb 100644 --- a/src/shared/key-management/google-ai/provider.ts +++ b/src/shared/key-management/google-ai/provider.ts @@ -3,15 +3,13 @@ import { Key, KeyProvider } from ".."; import { config } from "../../../config"; import { logger } from "../../../logger"; import type { GoogleAIModelFamily } from "../../models"; -import { HttpError } from "../../errors"; +import { HttpError, PaymentRequiredError } from "../../errors"; // Note that Google AI is not the same as Vertex AI, both are provided by Google // but Vertex is the GCP product for enterprise. while Google AI is the // consumer-ish product. The API is different, and keys are not compatible. // https://ai.google.dev/docs/migrate_to_cloud -export type GoogleAIModel = "gemini-pro"; - export type GoogleAIKeyUpdate = Omit< Partial, | "key" @@ -93,10 +91,10 @@ export class GoogleAIKeyProvider implements KeyProvider { return this.keys.map((k) => Object.freeze({ ...k, key: undefined })); } - public get(_model: GoogleAIModel) { + public get(_model: string) { const availableKeys = this.keys.filter((k) => !k.isDisabled); if (availableKeys.length === 0) { - throw new HttpError(402, "No Google AI keys available"); + throw new PaymentRequiredError("No Google AI keys available"); } // (largely copied from the OpenAI provider, without trial key support) diff --git a/src/shared/key-management/index.ts b/src/shared/key-management/index.ts index 2b7cd73..5e43e57 100644 --- a/src/shared/key-management/index.ts +++ b/src/shared/key-management/index.ts @@ -1,9 +1,4 @@ import type { LLMService, ModelFamily } from "../models"; -import { OpenAIModel } from "./openai/provider"; -import { AnthropicModel } from "./anthropic/provider"; -import { GoogleAIModel } from "./google-ai/provider"; -import { AwsBedrockModel } from "./aws/provider"; -import { AzureOpenAIModel } from "./azure/provider"; import { KeyPool } from "./key-pool"; /** The request and response format used by a model's API. */ @@ -15,12 +10,6 @@ export type APIFormat = | "anthropic-text" // Legacy flat string prompt format | "google-ai" | "mistral-ai"; -export type Model = - | OpenAIModel - | AnthropicModel - | GoogleAIModel - | AwsBedrockModel - | AzureOpenAIModel; export interface Key { /** The API key itself. Never log this, use `hash` instead. */ @@ -58,7 +47,7 @@ for service-agnostic functionality. export interface KeyProvider { readonly service: LLMService; init(): void; - get(model: Model): T; + get(model: string): T; list(): Omit[]; disable(key: T): void; update(hash: string, update: Partial): void; diff --git a/src/shared/key-management/key-pool.ts b/src/shared/key-management/key-pool.ts index db0b566..29f975b 100644 --- a/src/shared/key-management/key-pool.ts +++ b/src/shared/key-management/key-pool.ts @@ -5,7 +5,7 @@ import schedule from "node-schedule"; import { config } from "../../config"; import { logger } from "../../logger"; import { LLMService, MODEL_FAMILY_SERVICE, ModelFamily } from "../models"; -import { Key, Model, KeyProvider } from "./index"; +import { Key, KeyProvider } from "./index"; import { AnthropicKeyProvider, AnthropicKeyUpdate } from "./anthropic/provider"; import { OpenAIKeyProvider, OpenAIKeyUpdate } from "./openai/provider"; import { GoogleAIKeyProvider } from "./google-ai/provider"; @@ -41,7 +41,7 @@ export class KeyPool { this.scheduleRecheck(); } - public get(model: Model, service?: LLMService): Key { + public get(model: string, service?: LLMService): Key { const queryService = service || this.getServiceForModel(model); return this.getKeyProvider(queryService).get(model); } @@ -59,7 +59,10 @@ export class KeyPool { const service = this.getKeyProvider(key.service); service.disable(key); service.update(key.hash, { isRevoked: reason === "revoked" }); - if (service instanceof OpenAIKeyProvider || service instanceof AnthropicKeyProvider) { + if ( + service instanceof OpenAIKeyProvider || + service instanceof AnthropicKeyProvider + ) { service.update(key.hash, { isOverQuota: reason === "quota" }); } } @@ -69,7 +72,7 @@ export class KeyPool { service.update(key.hash, props); } - public available(model: Model | "all" = "all"): number { + public available(model: string | "all" = "all"): number { return this.keyProviders.reduce((sum, provider) => { const includeProvider = model === "all" || this.getServiceForModel(model) === provider.service; @@ -109,7 +112,7 @@ export class KeyPool { provider.recheck(); } - private getServiceForModel(model: Model): LLMService { + private getServiceForModel(model: string): LLMService { if ( model.startsWith("gpt") || model.startsWith("text-embedding-ada") || diff --git a/src/shared/key-management/mistral-ai/provider.ts b/src/shared/key-management/mistral-ai/provider.ts index f09f5e4..83785f8 100644 --- a/src/shared/key-management/mistral-ai/provider.ts +++ b/src/shared/key-management/mistral-ai/provider.ts @@ -1,5 +1,5 @@ import crypto from "crypto"; -import { Key, KeyProvider, Model } from ".."; +import { Key, KeyProvider } from ".."; import { config } from "../../../config"; import { logger } from "../../../logger"; import { MistralAIModelFamily, getMistralAIModelFamily } from "../../models"; @@ -92,7 +92,7 @@ export class MistralAIKeyProvider implements KeyProvider { return this.keys.map((k) => Object.freeze({ ...k, key: undefined })); } - public get(_model: Model) { + public get(_model: string) { const availableKeys = this.keys.filter((k) => !k.isDisabled); if (availableKeys.length === 0) { throw new HttpError(402, "No Mistral AI keys available"); diff --git a/src/shared/key-management/openai/checker.ts b/src/shared/key-management/openai/checker.ts index d49b347..481d0c4 100644 --- a/src/shared/key-management/openai/checker.ts +++ b/src/shared/key-management/openai/checker.ts @@ -1,4 +1,4 @@ -import axios, { AxiosError, AxiosResponse } from "axios"; +import axios, { AxiosError } from "axios"; import type { OpenAIModelFamily } from "../../models"; import { KeyCheckerBase } from "../key-checker-base"; import type { OpenAIKey, OpenAIKeyProvider } from "./provider"; diff --git a/src/shared/key-management/openai/provider.ts b/src/shared/key-management/openai/provider.ts index 0d06cb0..18485c7 100644 --- a/src/shared/key-management/openai/provider.ts +++ b/src/shared/key-management/openai/provider.ts @@ -1,25 +1,11 @@ -/* Manages OpenAI API keys. Tracks usage, disables expired keys, and provides -round-robin access to keys. Keys are stored in the OPENAI_KEY environment -variable as a comma-separated list of keys. */ import crypto from "crypto"; import http from "http"; -import { Key, KeyProvider, Model } from "../index"; +import { Key, KeyProvider } from "../index"; import { config } from "../../../config"; import { logger } from "../../../logger"; import { OpenAIKeyChecker } from "./checker"; import { getOpenAIModelFamily, OpenAIModelFamily } from "../../models"; -import { HttpError } from "../../errors"; - -export type OpenAIModel = - | "gpt-3.5-turbo" - | "gpt-3.5-turbo-instruct" - | "gpt-4" - | "gpt-4-32k" - | "gpt-4-1106" - | "text-embedding-ada-002" - | "dall-e-2" - | "dall-e-3" - | string; +import { PaymentRequiredError } from "../../errors"; // Flattening model families instead of using a nested object for easier // cloning. @@ -161,7 +147,7 @@ export class OpenAIKeyProvider implements KeyProvider { }); } - public get(requestModel: Model) { + public get(requestModel: string) { let model = requestModel; // Special case for GPT-4-32k. Some keys have access to only gpt4-32k-0314 @@ -185,7 +171,9 @@ export class OpenAIKeyProvider implements KeyProvider { ); if (availableKeys.length === 0) { - throw new HttpError(402, `No keys can fulfill request for ${model}`); + throw new PaymentRequiredError( + `No keys can fulfill request for ${model}` + ); } // Select a key, from highest priority to lowest priority: diff --git a/src/user/web/views/user_lookup.ejs b/src/user/web/views/user_lookup.ejs index 4b75c87..190d5fb 100644 --- a/src/user/web/views/user_lookup.ejs +++ b/src/user/web/views/user_lookup.ejs @@ -24,40 +24,40 @@ User Token - <%- "..." + user.token.slice(-5) %> + <%= "..." + user.token.slice(-5) %> Nickname - <%- user.nickname ?? "none" %> + <%= user.nickname ?? "none" %> ✏️ Type - <%- user.type %> + <%= user.type %> Prompts - <%- user.promptCount %> + <%= user.promptCount %> Created At - <%- user.createdAt %> + <%= user.createdAt %> Last Used At - <%- user.lastUsedAt || "never" %> + <%= user.lastUsedAt || "never" %> - IPs<%- ipLimit ? ` (max ${ipLimit})` : "" %> + IPs<%= ipLimit ? ` (max ${ipLimit})` : "" %> <%- include("partials/shared_user_ip_list", { user, shouldRedact: true }) %> <% if (user.type === "temporary") { %> Expires At - <%- user.expiresAt %> + <%= user.expiresAt %> <% } %>