diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index b99c85ec149..e193652ba13 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -399,12 +399,16 @@ This path currently supports the admitted local cells; remote transport remains unqualified. Scheduled experiment creation, grade insertion and summary retrieval share a -60-second window, three-request cap, 4 MiB total payload allowance and eight-score +60-second window, three-request cap, 4 MiB total payload allowance and fourteen-score cap. They run serially without automatic retry. Admission is persisted before dispatch in `.experiment-ingestion.jsonl`; uncertain responses retain their allowance and unresolved status. Reusing that file fails instead of resetting the budget. These limits cover the selected daily/weekly campaigns; the standalone experiment-import command does not use this transport budget. +Each retained attempt contributes two deterministic grades, including failed +grades. The cap supports seven attempts without dropping rows. Larger campaigns +still fail closed. Historical receipts with the eight-score policy remain +readable and keep their original limits and traffic counts. Private `ingestion-usage.json` binds all three receipt paths to the campaign's admitted attempts and ledger hash. Missing, malformed, unclosed or unresolved diff --git a/tests/runner-e2e/experiment-ingestion.test.ts b/tests/runner-e2e/experiment-ingestion.test.ts index 984bcb91a78..1e8fe2a2aec 100644 --- a/tests/runner-e2e/experiment-ingestion.test.ts +++ b/tests/runner-e2e/experiment-ingestion.test.ts @@ -52,6 +52,19 @@ it("keeps failed forwarding charged to the finite allowance and records an unres expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 3, responses: 0, unresolvedRequests: 3 }); }); +it.each([6, 7])("exports both deterministic grades for every retained case in a %i-case qualification", async cases => { + const upstream = vi.fn(async () => Response.json({ ok: true })); + const f = await fixture(upstream); + await f.send({ project_id: projectId }); + await f.send({ events: Array.from({ length: cases }, (_, index) => ({ + id: `attempt-${index}`, scores: { product_pass: Number(index !== 0), cleanup_pass: 1 }, + })) }, `experiment/${experimentId}/insert`); + await f.transport(`https://api.braintrust.dev/v1/experiment/${experimentId}/summarize`, { method: "GET" }); + await f.close(); + expect(upstream).toHaveBeenCalledTimes(3); + expect((await f.records()).at(-1)).toMatchObject({ admittedRequests: 3, admittedScores: cases * 2, unresolvedRequests: 0 }); +}); + it("retains an unresolved allowance when response headers arrive but the body disconnects", async () => { const f = await fixture(async () => new Response(new ReadableStream({ pull(controller) { controller.error(new Error("private-disconnect-error")); @@ -85,7 +98,7 @@ it("refuses excess bytes, scores, foreign project and unqualified endpoints befo const f = await fixture(upstream); await expect(f.send({ project_id: experimentId })).rejects.toThrow("project mismatch"); await expect(f.send({ project_id: projectId, data: "x".repeat(EVAL_EXPERIMENT_LIMITS.maxTotalBytes) })).rejects.toThrow("byte limit"); - await expect(f.send({ events: Array.from({ length: 9 }, () => ({ scores: { pass: 1 } })) }, `experiment/${experimentId}/insert`)) + await expect(f.send({ events: Array.from({ length: EVAL_EXPERIMENT_LIMITS.maxScores + 1 }, () => ({ scores: { pass: 1 } })) }, `experiment/${experimentId}/insert`)) .rejects.toThrow("allowance exhausted"); await expect(f.send({}, "project")).rejects.toThrow("Unqualified"); expect(upstream).not.toHaveBeenCalled(); diff --git a/tests/runner-e2e/experiment-ingestion.ts b/tests/runner-e2e/experiment-ingestion.ts index 7cb1c102bbe..1e221c9d934 100644 --- a/tests/runner-e2e/experiment-ingestion.ts +++ b/tests/runner-e2e/experiment-ingestion.ts @@ -3,7 +3,7 @@ import { mkdir, open } from "node:fs/promises"; import path from "node:path"; import { isEvalProjectId } from "./eval-otlp-payload.js"; -export const EVAL_EXPERIMENT_LIMITS = { maxRequests: 3, maxTotalBytes: 4 * 1024 * 1024, maxScores: 8 } as const; +export const EVAL_EXPERIMENT_LIMITS = { maxRequests: 3, maxTotalBytes: 4 * 1024 * 1024, maxScores: 14 } as const; export async function openExperimentIngestion(input: { receiptPath: string; campaignId: string; projectId: string; deadlineAt: number; transport?: typeof fetch; diff --git a/tests/runner-e2e/ingestion-usage.test.ts b/tests/runner-e2e/ingestion-usage.test.ts index 38e921b79f6..69633994a0e 100644 --- a/tests/runner-e2e/ingestion-usage.test.ts +++ b/tests/runner-e2e/ingestion-usage.test.ts @@ -90,6 +90,16 @@ it("keeps historical native receipts with the larger policy limits readable", as totalAdmittedPayloadBytes: f.expectedBytes, billedCostUsd: null }); }); +it.each([8, 14, 15])("validates an experiment receipt with a %i-score policy against the supported ceiling", async maxScores => { + const f = await fixture(), file = `${f.input.ledgerPath}.experiment-ingestion.jsonl`; + const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + rows[0].limits.maxScores = maxScores; + await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n"); + const usage = await collectScheduledIngestion(f.input); + expect(usage.status).toBe(maxScores <= 14 ? "complete" : "incomplete"); + expect(usage.receipts.at(-1)!.status).toBe(maxScores <= 14 ? "complete" : "unavailable"); +}); + it("keeps version 1 transport receipts readable without inventing expanded data measurements", async () => { const f = await fixture(), file = f.files[0]!; const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line));