minor adjustments to jsonl log backend to reduce filesize

This commit is contained in:
nai-degen 2024-04-26 15:06:12 -05:00
parent 32b623d6bc
commit b1062dc9b3
1 changed files with 11 additions and 2 deletions

View File

@ -7,7 +7,7 @@ import { logger } from "../../../logger";
import { LogBackend, PromptLogEntry } from "../index";
import { glob } from "glob";
const MAX_FILE_SIZE = 25 * 1024 * 1024;
const MAX_FILE_SIZE = 100 * 1024 * 1024;
let currentFileNumber = 0;
let currentFilePath = "";
@ -57,7 +57,16 @@ export const fileBackend: LogBackend = {
}
const batchString =
batch.map((entry) => JSON.stringify(entry)).join("\n") + "\n";
batch
.map((entry) =>
JSON.stringify({
endpoint: entry.endpoint,
model: entry.model,
prompt: entry.promptRaw,
response: entry.response,
})
)
.join("\n") + "\n";
const batchSizeBytes = Buffer.byteLength(batchString);
const batchLines = batch.length;
logger.debug(