import { afterEach, expect, it, vi } from "vitest"; import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { EVAL_EXPERIMENT_LIMITS, openExperimentIngestion } from "./experiment-ingestion.js"; const projectId = "11111111-1111-4111-8111-111111111111", experimentId = "22222222-2222-4222-8222-222222222222"; const roots: string[] = [], meters: Array>> = []; afterEach(async () => { await Promise.allSettled(meters.splice(0).map(meter => meter.close())); await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); async function fixture(transport: typeof fetch = async () => Response.json({ ok: true })) { const root = await mkdtemp(path.join(tmpdir(), "experiment-ingestion-")); roots.push(root); const policy = { projectId, campaignId: "campaign", deadlineAt: Date.now() + 60_000, receiptPath: path.join(root, "receipts.jsonl"), transport }; const meter = await openExperimentIngestion(policy); meters.push(meter); return { ...meter, policy, records: async () => (await readFile(policy.receiptPath, "utf8")).trim().split("\n").map(line => JSON.parse(line)), send: (payload: unknown, endpoint = "experiment") => meter.transport(`https://api.braintrust.dev/v1/${endpoint}`, { method: "POST", body: JSON.stringify(payload), headers: { authorization: "Bearer private-secret" }, }) }; } it("durably meters payloads and scores before dispatch while retaining body-free private evidence", async () => { const upstream = vi.fn(async (_url, init) => { expect((await f.records()).at(-1)).toMatchObject({ type: "admitted", bodyBytes: init?.body === undefined ? 0 : Buffer.byteLength(String(init.body)) }); expect(init!.redirect).toBe("error"); return Response.json({ ok: true }); }); const f = await fixture(upstream); const create = { project_id: projectId, name: "private-payload-canary" }; const insert = { events: [{ scores: { product_pass: 0, cleanup_pass: 0 }, input: "private-input-canary" }] }; await f.send(create); await f.send(insert, `experiment/${experimentId}/insert`); await f.transport(`https://api.braintrust.dev/v1/experiment/${experimentId}/summarize`, { method: "GET" }); await f.close(); const records = await f.records(); expect(records.at(-1)).toMatchObject({ type: "closed", admittedRequests: 3, admittedScores: 2, admittedBytes: Buffer.byteLength(JSON.stringify(create)) + Buffer.byteLength(JSON.stringify(insert)), unresolvedRequests: 0 }); const raw = await readFile(f.policy.receiptPath, "utf8"); expect(raw).not.toMatch(/private-secret|private-payload-canary|private-input-canary/); expect((await stat(f.policy.receiptPath)).mode & 0o777).toBe(0o600); await expect(openExperimentIngestion(f.policy)).rejects.toMatchObject({ code: "EEXIST" }); expect(await readFile(f.policy.receiptPath, "utf8")).toBe(raw); }); it("keeps failed forwarding charged to the finite allowance and records an unresolved outcome", async () => { const upstream = vi.fn(async () => { throw new Error("private-upstream-error"); }); const f = await fixture(upstream); for (let i = 0; i < EVAL_EXPERIMENT_LIMITS.maxRequests; i++) { await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed"); } await expect(f.send({ project_id: projectId })).rejects.toThrow("allowance exhausted"); expect(upstream).toHaveBeenCalledTimes(3); await f.close(); expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 3, responses: 0, unresolvedRequests: 3 }); }); it("retains an unresolved allowance when response headers arrive but the body disconnects", async () => { const f = await fixture(async () => new Response(new ReadableStream({ pull(controller) { controller.error(new Error("private-disconnect-error")); } }), { status: 200 })); await expect(f.send({ project_id: projectId })).rejects.toThrow("unconfirmed"); await f.close(); const rows = await f.records(); expect(rows[2]).toMatchObject({ type: "outcome", outcome: "unconfirmed" }); expect(rows.at(-1)).toMatchObject({ admittedRequests: 1, responses: 0, unresolvedRequests: 1 }); expect(JSON.stringify(rows)).not.toContain("private-disconnect-error"); }); it("keeps a pending response body unresolved until the whole response is read", async () => { let finish!: () => void; const f = await fixture(async () => new Response(new ReadableStream({ start(controller) { controller.enqueue(new TextEncoder().encode('{"ok":')); finish = () => { controller.enqueue(new TextEncoder().encode('true}')); controller.close(); }; } }))); const pending = f.send({ project_id: projectId }); await vi.waitFor(() => expect(finish).toBeTypeOf("function")); expect((await f.records()).at(-1)).toMatchObject({ type: "admitted" }); await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); finish(); expect(await (await pending).json()).toEqual({ ok: true }); await f.close(); expect((await f.records()).at(-1)).toMatchObject({ responses: 1, unresolvedRequests: 0 }); }); it("refuses excess bytes, scores, foreign project and unqualified endpoints before dispatch", async () => { const upstream = vi.fn(async () => Response.json({})); const f = await fixture(upstream); await expect(f.send({ project_id: experimentId })).rejects.toThrow("project mismatch"); await expect(f.send({ project_id: projectId, data: "x".repeat(EVAL_EXPERIMENT_LIMITS.maxTotalBytes) })).rejects.toThrow("byte limit"); await expect(f.send({ events: Array.from({ length: 9 }, () => ({ scores: { pass: 1 } })) }, `experiment/${experimentId}/insert`)) .rejects.toThrow("allowance exhausted"); await expect(f.send({}, "project")).rejects.toThrow("Unqualified"); expect(upstream).not.toHaveBeenCalled(); await f.close(); expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 0, admittedBytes: 0, admittedScores: 0 }); }); it("refuses concurrent requests and expired windows without resetting reservations", async () => { let release!: () => void; const upstream = vi.fn(async () => { await new Promise(resolve => { release = resolve; }); return Response.json({}); }); const f = await fixture(upstream); const pending = f.send({ project_id: projectId }); await vi.waitFor(() => expect(upstream).toHaveBeenCalledOnce()); await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); release(); await pending; const now = vi.spyOn(Date, "now").mockReturnValue(f.policy.deadlineAt); try { await expect(f.send({ project_id: projectId })).rejects.toThrow("unavailable"); } finally { now.mockRestore(); } expect(upstream).toHaveBeenCalledOnce(); });