diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 52650357b06..1dc7b6c3258 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -364,8 +364,10 @@ rejects ambiguous singular fields and routing values. Each relay allows at most 2,048 admitted exports, 8 MiB per request before and after decompression, 64 MiB of forwarded payloads, and four simultaneous exports. -It also limits each attempt to 16,384 records and 256 MiB of expanded source -JSON. This measurement includes the full decoded source plus each record in +The default allowance limits each attempt to 4,096 records and 16 MiB of +expanded source JSON. Historical receipts with up to 16,384 records and 256 MiB +remain readable; controller callers may still supply explicit limits up to +those maxima. This measurement includes the full decoded source plus each record in its own envelope with the shared resource and scope fields repeated. It counts span names, IDs, times, events, links and log bodies, not only routing attributes. Protobuf fields use the pinned OTLP v1.9 schema; unknown fields fail admission @@ -408,7 +410,7 @@ or an invoice. Vendor charges remain unreconciled. The current two-attempt daily maximum is 132.125 MiB and the four-attempt weekly maximum is 260.25 MiB, including finalization and the campaign's experiment payload allowance. These bounds do not include other projects, standalone imports or retained storage. -The corresponding expanded-source ceilings are 514 MiB daily and 1,028 MiB +The corresponding default expanded-source ceilings are 34 MiB daily and 68 MiB weekly before the separate 4-MiB experiment allowance. These source measurements support a priced forecast; they are not Braintrust's processed-data meter or a vendor-enforced dollar limit. Gateway auto-logging is a separate input to that @@ -463,7 +465,12 @@ the existing AWS/GitHub runner choice. Daily selects native Claude's marker and feedback-persistence cases with a shared $20 allowance. Weekly adds native Codex's two corresponding cases with a -shared $100 allowance. Model requests and declared operating overhead share +shared $100 allowance. Daily campaigns admit at most 30 model requests; weekly +campaigns admit at most 60. The October 23 qualification campaigns used 10 model +requests each. The largest observed native attempt exported 6,531,801 expanded +source bytes and 2,455 records. The defaults retain headroom for those measured +cases; exhausting a limit remains visible and never silently drops required +evidence. Model requests and declared operating overhead share this limit. Each campaign runs sequentially in one job with one journal, one attempt per execution, zero automatic retries, a 30-minute request deadline and a finite request count. Codex's worst-case request reservation exceeds diff --git a/tests/runner-e2e/eval-otlp-relay.test.ts b/tests/runner-e2e/eval-otlp-relay.test.ts index d241934242a..7b220dc0f45 100644 --- a/tests/runner-e2e/eval-otlp-relay.test.ts +++ b/tests/runner-e2e/eval-otlp-relay.test.ts @@ -239,7 +239,7 @@ it.each(["json", "protobuf"])("meters repeated resource/scope data and full reco it("accepts thousands of small native records while bounding their total count", async () => { const upstream = vi.fn(async () => Response.json({})); - const f = await fixture({ transport: upstream, limits: { ...EVAL_OTLP_LIMITS, maxRecords: 3000 } }); + const f = await fixture({ transport: upstream }); const body = JSON.stringify({ resource_logs: [{ scope_logs: [{ log_records: Array.from({ length: 2533 }, () => ({ body: { stringValue: "event" } })) }] }] }); expect((await f.send(body, {}, "/v1/logs")).status).toBe(200); expect((await f.send(body, {}, "/v1/logs")).status).toBe(429); @@ -247,6 +247,23 @@ it("accepts thousands of small native records while bounding their total count", expect((await f.records()).find(row => row.type === "admitted").recordCount).toBe(2533); }); +it("enforces the smaller default expanded-source allowance before export", async () => { + const upstream = vi.fn(async () => Response.json({})); + const f = await fixture({ transport: upstream }); + const body = JSON.stringify({ resourceLogs: [{ scopeLogs: [{ logRecords: [ + { body: { stringValue: "x".repeat(3 * 1024 * 1024) } }, + ] }] }] }); + expect((await f.send(body, {}, "/v1/logs")).status).toBe(200); + expect((await f.send(body, {}, "/v1/logs")).status).toBe(200); + expect((await f.send(body, {}, "/v1/logs")).status).toBe(429); + expect(upstream).toHaveBeenCalledTimes(2); + await f.relay.close(); + const rows = await f.records(); + expect(rows[0].limits.maxExpandedBytes).toBe(16 * 1024 * 1024); + expect(rows.at(-1).admittedExpandedBytes).toBeGreaterThan(12 * 1024 * 1024); + expect(rows.filter(row => row.type === "quota_exhausted")).toHaveLength(1); +}); + it("rejects unknown protobuf fields instead of excluding forwarded data from sizing", async () => { const upstream = vi.fn(async () => Response.json({})); const f = await fixture({ transport: upstream }); diff --git a/tests/runner-e2e/eval-otlp-relay.ts b/tests/runner-e2e/eval-otlp-relay.ts index 36e61d9dde4..d34efd814e5 100644 --- a/tests/runner-e2e/eval-otlp-relay.ts +++ b/tests/runner-e2e/eval-otlp-relay.ts @@ -9,12 +9,14 @@ import { evalOtlpRejected, evalOtlpResponse, isEvalProjectId, validateEvalOtlpPa export const EVAL_OTLP_LIMITS = { maxRequests: 2048, maxRequestBytes: 8 * 1024 * 1024, maxTotalBytes: 64 * 1024 * 1024, maxConcurrent: 4, maxExpandedBytes: 256 * 1024 * 1024, maxRecords: 16_384 } as const; +export const EVAL_OTLP_DEFAULT_LIMITS = { ...EVAL_OTLP_LIMITS, + maxExpandedBytes: 16 * 1024 * 1024, maxRecords: 4096 } as const; export async function startEvalOtlpRelay(input: { projectId: string; apiKey: string; campaignId: string; attemptId: string; receiptPath: string; deadlineAt: number; limits?: { [K in keyof typeof EVAL_OTLP_LIMITS]: number }; transport?: typeof fetch; }) { - const limits = input.limits ?? EVAL_OTLP_LIMITS; + const limits = input.limits ?? EVAL_OTLP_DEFAULT_LIMITS; if (!isEvalProjectId(input.projectId) || !input.apiKey.trim() || /[\r\n]/.test(input.apiKey) || ![input.campaignId, input.attemptId].every(value => /^[A-Za-z0-9_.:-]{1,512}$/.test(value)) || !path.isAbsolute(input.receiptPath) || !Number.isFinite(input.deadlineAt) diff --git a/tests/runner-e2e/ingestion-usage.test.ts b/tests/runner-e2e/ingestion-usage.test.ts index fdb327f4542..38e921b79f6 100644 --- a/tests/runner-e2e/ingestion-usage.test.ts +++ b/tests/runner-e2e/ingestion-usage.test.ts @@ -80,6 +80,16 @@ it("binds the expected receipt set to admitted attempts and retains unstarted se await expect(collectScheduledIngestion({ ...f.input, campaignId: "foreign" })).rejects.toThrow("campaign mismatch"); }); +it("keeps historical native receipts with the larger policy limits readable", async () => { + const f = await fixture(), file = f.files[0]!; + const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); + rows[0].limits.maxExpandedBytes = 256 * 1024 * 1024; + rows[0].limits.maxRecords = 16_384; + await writeFile(file, rows.map(row => JSON.stringify(row)).join("\n") + "\n"); + expect(await collectScheduledIngestion(f.input)).toMatchObject({ status: "complete", + totalAdmittedPayloadBytes: f.expectedBytes, billedCostUsd: null }); +}); + it("keeps version 1 transport receipts readable without inventing expanded data measurements", async () => { const f = await fixture(), file = f.files[0]!; const rows = (await readFile(file, "utf8")).trim().split("\n").map(line => JSON.parse(line)); diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts index a9856edd9cc..5fc31097dfe 100644 --- a/tests/runner-e2e/scheduled-operations.test.ts +++ b/tests/runner-e2e/scheduled-operations.test.ts @@ -19,7 +19,7 @@ it.each(["daily", "weekly"] as const)("runs the exact %s selection through one s const manifest = scheduledManifest({ ...input, cadence }); const selected = selectRunnerExecutions(parseRunnerSelectors(manifest.executionIds.flatMap(id => ["--id", id])), runnerMatrix); expect(selected.map(row => row.id).sort()).toEqual([...scheduledSelection(cadence)].sort()); - expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxConcurrent: 1, maxAttemptsPerExecution: 1 }); + expect(manifest.limits).toMatchObject({ maxUsd: cadence === "daily" ? "20" : "100", maxRequests: cadence === "daily" ? 30 : 60, maxConcurrent: 1, maxAttemptsPerExecution: 1 }); expect(Date.parse(manifest.limits.deadlineAt) - input.now).toBe(30 * 60_000); expect(scheduledMatrix(cadence).include).toHaveLength(1); expect(manifest.executionIds).toHaveLength(cadence === "daily" ? 2 : 4); diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts index f92c8ff135e..c2a5e80cfcb 100644 --- a/tests/runner-e2e/scheduled-operations.ts +++ b/tests/runner-e2e/scheduled-operations.ts @@ -64,7 +64,7 @@ export function scheduledManifest(input: { const { executionIds, executions, models } = validateScheduledBudget(input); const manifest: EvalOperationsManifest = { version: 1, limits: { campaignId: input.campaignId, maxUsd: budget(input.cadence), overhead: input.overhead, - deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 60 : 120, + deadlineAt: new Date(input.now + 30 * 60_000).toISOString(), maxRequests: input.cadence === "daily" ? 30 : 60, maxConcurrent: 1, maxAttemptsPerExecution: 1 }, ledgerPath: input.ledgerPath, executionIds, maxAttemptsPerExecution: 1, route: "braintrust", projectId: input.projectId, models,