diff --git a/tests/runner-e2e/OPERATIONS-COVERAGE.md b/tests/runner-e2e/OPERATIONS-COVERAGE.md index 4c6ba62aa58..1b75a563435 100644 --- a/tests/runner-e2e/OPERATIONS-COVERAGE.md +++ b/tests/runner-e2e/OPERATIONS-COVERAGE.md @@ -18,28 +18,40 @@ Both profiles use the exact model and limits in the reviewed pricecard. These associations identify partial coverage. They do not qualify every run lifecycle, the Commentary browser page, continuation, or remote execution. -The October 21 local pilot measured one successful sample per selected cell: -$0.427624800 and 72.757 summed attempt seconds for the daily selection, -$0.791539400 and 127.069 seconds for the weekly selection. These are historical -gateway estimates from the Linux ARM64 container. They include model setup -requests and attempt setup/teardown, but omit queue/build time and non-model -charges. Earlier failed and interrupted attempts remain in campaign spending. +The October 27 qualifications measured one successful attempt per selected cell: +$0.428650100 and 107.286 summed attempt seconds for the daily selection, +$0.793676500 and 158.093 seconds for the weekly selection. Claude ran on +GitHub-hosted Ubuntu; Codex ran in a local Linux ARM64 container. These are +historical gateway estimates from 19 final model request receipts. They include +setup, background and disconnected-client requests with terminal usage. Attempt +time includes setup and teardown, but omits workflow queue/build time and +non-model charges. Provider exports match usage and charges (Claude at one-cent +export precision); final monthly invoices remain pending. The hosted workflow +failed during report dependency setup after both cases passed. Its original +failure, and all earlier failed or interrupted attempts, remain retained. -`scheduled-measurements.json` retains the exact attempt, source revision, -definition, model, price version, result hash and journal hashes for each sample. -`forecast.json` includes this evidence and a snapshot hash. It separates the -historical successful selection from samples matching the current definition, -model, environment, runtime, route and pricing. A missing matching cell makes -the compatible selection total unknown. The feedback definition changed after -these four samples; its earlier results are not current qualification. +`scheduled-measurements.json` retains each attempt, source revision, definition, +model, price version, measurement host, final request count, result hash and +journal hashes. `forecast.json` includes that evidence and a snapshot hash. +Historical totals remain separate from samples matching the current definition, +model, environment, runtime, route and pricing. Hosts form separate cohorts; +measurements from different hosts are not pooled into one cell distribution. +The feedback definition's source fingerprint changed after these samples, so +current compatible selection totals remain unknown. No historical hash is +rewritten to match the new definition. + +`historicalHostedSelection` records the measured hosted subset only when every +selected cell has hosted evidence. Current `hostedModelUsd` and +`hostedAttemptMs` additionally require matching definitions. The four-case +weekly selection has no complete hosted measurement yet. Summed attempt time +is neither campaign wall time nor billed infrastructure time. Per-cell mean and median are descriptive. P95 stays unknown below 20 samples and then uses nearest rank; this is not a statistical confidence guarantee. -Do not sum per-cell percentiles into a campaign percentile. Hosted cost/time -and the selection's P95 remain unknown until separately measured. A passing -sample does not predict failure frequency or account for infrastructure, -storage, observability or subscriptions. Forecasts never reduce reservations -or the approved budget limits. +Do not sum per-cell percentiles into a campaign percentile. A passing sample +does not predict failure frequency or account for infrastructure, storage, +observability or subscriptions. Forecasts never reduce reservations, approved +budget limits, or qualification gates. At the October 21, 2026 catalog audit, 47 suites declared 738 cells. `--all` included 179; the other 559 required explicit selection. The bounded weekly diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 8ced2fc5b5d..5d45ece6627 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -670,10 +670,13 @@ preflight already exists. Review current official prices and context limits before updating `scheduled-prices.json`. `forecast.json` records the selected IDs, provenance-linked pilot measurements, compatibility gaps, uncertainty, reservations and price expiry. The historical daily/weekly successful model -subtotals are $0.427624800/$0.791539400; they are not campaign bills. Setup model +subtotals are $0.428650100/$0.793676500; they are not campaign bills. Setup model calls and attempt setup/teardown are included, while CI setup/build/queue and non-model charges are excluded. Current compatible totals stay unknown if any -selected cell lacks a matching measurement. Hosted timing is still unmeasured. +selected cell lacks a matching measurement. The historical hosted daily selection took 107.286 summed attempt seconds. +The weekly Codex samples ran locally; a complete hosted weekly total remains +unknown. Measurement hosts form separate cohorts. The original hosted workflow +failed during report dependency setup after both worker cases passed. Per-cell medians describe only the retained samples; P95 requires at least 20 matching samples and does not establish reliability. See [measurement provenance and limitations](OPERATIONS-COVERAGE.md#initial-cadence). diff --git a/tests/runner-e2e/scheduled-forecast.test.ts b/tests/runner-e2e/scheduled-forecast.test.ts index daf5b007a11..d6aa1efdca0 100644 --- a/tests/runner-e2e/scheduled-forecast.test.ts +++ b/tests/runner-e2e/scheduled-forecast.test.ts @@ -10,12 +10,14 @@ const models = prices.models as EvalOperationsManifest["models"]; const selected = (cadence: "daily" | "weekly") => runnerMatrix.filter(cell => scheduledSelection(cadence).includes(cell.id)); it.each([ - ["daily", "0.427624800", 72757], ["weekly", "0.791539400", 127069], -] as const)("retains exact historical %s measurements without inventing hosted or tail estimates", (cadence, modelUsd, summedAttemptMs) => { + ["daily", "0.428650100", 107286], ["weekly", "0.793676500", 158093], +] as const)("retains exact historical %s measurements without inventing tail estimates", (cadence, modelUsd, summedAttemptMs) => { const forecast = scheduledForecast(selected(cadence), models); expect(forecast.historicalSuccessfulSelection).toEqual({ modelUsd, summedAttemptMs }); expect(forecast.measurementSnapshotSha256).toMatch(/^[0-9a-f]{64}$/); expect(forecast).toMatchObject({ hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null, costStatus: "estimated" }); + expect(forecast.historicalHostedSelection).toEqual(cadence === "daily" ? { modelUsd, summedAttemptMs } : { modelUsd: null, summedAttemptMs: null }); + expect(forecast.compatibleSuccessfulSelection).toEqual({ modelUsd: null, summedAttemptMs: null }); for (const cell of forecast.cells) { expect(cell.historicalSuccessful.samples).toBe(1); expect(cell.historicalSuccessful.modelUsd.p95).toBeNull(); @@ -89,3 +91,33 @@ it("preserves nanodollar costs and genuine measured zero", () => { snapshot.samples[0]!.knownModelUsd = "0"; expect(scheduledForecast([cell], models, snapshot).compatibleSuccessfulSelection.modelUsd).toBe("0.000000000"); }); + +it("keeps hosts separate and requires hosted evidence for the whole selection", () => { + const snapshot = structuredClone(measurements); + const cell = selected("daily").find(cell => cell.id.startsWith("core-"))!; + const sample = snapshot.samples.find(sample => sample.executionId === cell.id)!; + snapshot.samples = [sample, { ...sample, attemptId: "different-host", measurementHost: "local-docker-linux-arm64" }]; + const forecast = scheduledForecast([cell], models, snapshot); + expect(forecast.cells[0]!.historicalCohorts).toHaveLength(2); + expect(forecast.cells[0]!.compatibleSuccessful.samples).toBe(0); + expect(forecast.cells[0]!.samples).toHaveLength(2); + expect(forecast.hostedModelUsd).toBe(sample.knownModelUsd); + expect(scheduledForecast(selected("weekly"), models).hostedModelUsd).toBeNull(); +}); + +it("uses current hosted totals only when every selected definition matches", () => { + const snapshot = structuredClone(measurements); + for (const cell of selected("daily")) snapshot.samples.find(sample => sample.executionId === cell.id)!.definitionHash = cell.suiteDefinitionHash; + const forecast = scheduledForecast(selected("daily"), models, snapshot); + expect(forecast.hostedModelUsd).toBe("0.428650100"); + expect(forecast.hostedAttemptMs).toBe(107286); +}); + +it.each([ + { measurementHost: undefined }, { measurementHost: "" }, + { finalModelRequests: 0 }, { finalModelRequests: 1.5 }, +])("rejects invalid host or final-receipt provenance %j", patch => { + const snapshot = structuredClone(measurements); + Object.assign(snapshot.samples[0]!, patch); + expect(() => scheduledForecast(selected("daily"), models, snapshot)).toThrow("provenance"); +}); diff --git a/tests/runner-e2e/scheduled-forecast.ts b/tests/runner-e2e/scheduled-forecast.ts index 7cc1acf23f8..5e0b1fe4f71 100644 --- a/tests/runner-e2e/scheduled-forecast.ts +++ b/tests/runner-e2e/scheduled-forecast.ts @@ -6,7 +6,7 @@ import type { EvalOperationsManifest } from "./operations.js"; type Sample = (typeof measurements.samples)[number]; const cohortKey = (sample: Sample) => JSON.stringify([sample.definitionHash, sample.model, sample.environmentId, - sample.runtimeMode, sample.route, sample.priceVersion]); + sample.runtimeMode, sample.route, sample.priceVersion, sample.measurementHost]); const usd = (units: bigint) => centsToUsd(unitsToCents(units)); const average = (values: bigint[]) => values.length ? (values.reduce((sum, value) => sum + value, 0n) + BigInt(Math.floor(values.length / 2))) / BigInt(values.length) : null; @@ -34,6 +34,10 @@ function summarize(samples: Sample[]) { }; } +function summarizeCohort(samples: Sample[]) { + return summarize(new Set(samples.map(cohortKey)).size === 1 ? samples : []); +} + function total(rows: ReturnType[]) { return { modelUsd: rows.length && rows.every(row => row.modelUsd.mean !== null) @@ -50,7 +54,8 @@ export function scheduledForecast( for (const sample of snapshot.samples) { if (seen.has(sample.attemptId)) throw new Error("Duplicate forecast attempt"); seen.add(sample.attemptId); - if (!Number.isSafeInteger(sample.attemptDurationMs) || sample.attemptDurationMs < 0 || usdToUnits(sample.knownModelUsd) < 0n + if (typeof sample.measurementHost !== "string" || !/^[a-z0-9][a-z0-9-]{0,63}$/.test(sample.measurementHost) || !Number.isSafeInteger(sample.finalModelRequests) || sample.finalModelRequests < 1 + || !Number.isSafeInteger(sample.attemptDurationMs) || sample.attemptDurationMs < 0 || usdToUnits(sample.knownModelUsd) < 0n || !/^[0-9a-f]{40}$/.test(sample.sourceRevision) || [sample.definitionHash, sample.resultSha256, sample.ledgerSha256, sample.ledgerFinalHash].some(hash => !/^[0-9a-f]{64}$/.test(hash))) { throw new Error("Invalid forecast measurement or provenance"); @@ -75,16 +80,22 @@ export function scheduledForecast( }); return { executionId: execution.id, currentDefinitionHash: execution.suiteDefinitionHash, historicalSuccessful: summarize(cohorts.length === 1 ? retained : []), historicalCohorts: cohorts, - compatibleSuccessful: summarize(samples.filter(sample => sample.mismatches.length === 0)), samples }; + compatibleSuccessful: summarizeCohort(samples.filter(sample => sample.mismatches.length === 0)), + historicalHosted: summarizeCohort(retained.filter(sample => sample.measurementHost === "github-hosted-ubuntu-latest")), + compatibleHosted: summarizeCohort(samples.filter(sample => sample.mismatches.length === 0 + && sample.measurementHost === "github-hosted-ubuntu-latest")), samples }; }); + const hosted = total(cells.map(cell => cell.compatibleHosted)); return { measurementSnapshotSha256: createHash("sha256").update(JSON.stringify(snapshot)).digest("hex"), measuredHost: snapshot.host, costStatus: snapshot.costStatus, sampleScope: snapshot.sampleScope, historicalSuccessfulSelection: total(cells.map(cell => cell.historicalSuccessful)), + historicalHostedSelection: total(cells.map(cell => cell.historicalHosted)), compatibleSuccessfulSelection: total(cells.map(cell => cell.compatibleSuccessful)), - hostedModelUsd: null, hostedAttemptMs: null, selectionP95: null, + hostedModelUsd: hosted.modelUsd, hostedAttemptMs: hosted.summedAttemptMs, selectionP95: null, cells, limitations: [...snapshot.limitations, - "Compatibility compares definition, model, environment, runtime, route and price version; source revisions remain explicit", + "Compatibility compares definition, model, environment, runtime, route and price version; measurement hosts form separate cohorts and source revisions remain explicit", + "Hosted totals require compatible GitHub-hosted measurements for every selected cell", "Per-cell P95 uses nearest rank only with at least 20 samples; it is descriptive, not a confidence bound", "Summed attempt time is serial attempt work, not campaign wall time, summed agent time or billed runner time", "Historical successful samples do not estimate failure frequency; every failure remains in campaign accounting"], diff --git a/tests/runner-e2e/scheduled-measurements.json b/tests/runner-e2e/scheduled-measurements.json index afd9b43aaf1..c3eba8ff0b0 100644 --- a/tests/runner-e2e/scheduled-measurements.json +++ b/tests/runner-e2e/scheduled-measurements.json @@ -1,84 +1,98 @@ { "version": 1, - "recordedAt": "2026-10-21T18:13:01.055Z", + "recordedAt": "2026-10-28T03:23:41.620Z", "costStatus": "estimated", - "sampleScope": "successful local native qualification attempts only", - "host": "linux-arm64-local-container", + "sampleScope": "successful Product attempts with complete terminal model receipts; original failed campaign outcomes retained", + "host": "mixed; see each sample measurementHost", "samples": [ - { - "executionId": "feedback-persistence.runner-codex.local.complaint-and-suggestion", - "attemptId": "96850eb44e6a9f9856fcf24a40fb4903", - "sourceRevision": "276b8ff966c177b8cb34af12479915d9c8f2c9a2", - "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92", - "knownModelUsd": "0.189173400", - "attemptDurationMs": 26770, - "model": "gpt-5.6-sol", - "environmentId": "local", - "runtimeMode": "native", - "route": "braintrust", - "priceVersion": "openai-standard-2026-10-20", - "measuredAt": "2026-10-21T16:32:44.767Z", - "resultSha256": "e1748f875784065f2d40ebd7cc83cb9c3f84b1ce8dd05629482a97e818608188", - "ledgerSha256": "b5f42a64e8a72c5ae23c4dbf33fda1c561a7e2833d499e86b63627166144f1ef", - "ledgerFinalHash": "f06b687f8063926b77c0378946f1d6a9bfb718e1d8b9ca7d6e2aa344187be2dc" - }, { "executionId": "core-compatibility.runner-acpx-claude.local.message-marker", - "attemptId": "8da10d462119f3cf6bb3a4669c43b6b5", - "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c", + "attemptId": "c718184ebbdde012b1523317a800f6fb", + "sourceRevision": "bf3ef12f3a78d0b15a32ad57f7e16520fbf6cd5e", "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa", - "knownModelUsd": "0.123679200", - "attemptDurationMs": 34495, + "knownModelUsd": "0.124193300", + "attemptDurationMs": 52545, "model": "claude-sonnet-5", "environmentId": "local", "runtimeMode": "native", "route": "braintrust", "priceVersion": "anthropic-standard-2026-10-20", - "measuredAt": "2026-10-21T16:54:26.516Z", - "resultSha256": "89aac65b9a703d812a5a3c83c51732031e9b6fcadc6bac3c14a86c419f169d7f", - "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756", - "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e" + "measurementHost": "github-hosted-ubuntu-latest", + "campaignOutcome": "failed_report_dependency_setup", + "finalModelRequests": 5, + "measuredAt": "2026-10-27T20:24:56.120Z", + "resultSha256": "34d2c8b0b2aa39c563397eb402e981a7617ea6adbd9fdc1d0c59f852a356390c", + "ledgerSha256": "b94a48f61471050559d5cd217ed9d3e2270e7d342b69ad4caa236ae1bbfc916f", + "ledgerFinalHash": "eb4a8aff44281c8f2ff420f137496e58513beb9afabee1cce2a560ebb056e2ac" }, { "executionId": "core-compatibility.runner-codex.local.message-marker", - "attemptId": "03720091468148fe7a0d2b1fa68d3966", - "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c", + "attemptId": "1021b4ba8fdd0aa68df04c023fc997c2", + "sourceRevision": "bf3ef12f3a78d0b15a32ad57f7e16520fbf6cd5e", "definitionHash": "19541f579a34848b2cab664296eb30773c883416bc5ea62cc34c71c6d415ccaa", - "knownModelUsd": "0.174741200", - "attemptDurationMs": 27542, + "knownModelUsd": "0.173863000", + "attemptDurationMs": 20216, "model": "gpt-5.6-sol", "environmentId": "local", "runtimeMode": "native", "route": "braintrust", "priceVersion": "openai-standard-2026-10-20", - "measuredAt": "2026-10-21T16:53:20.764Z", - "resultSha256": "188a0655a7aebe37ec21c582e6b4afef1633210fa8f93e1049b9db0078657c4d", - "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756", - "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e" + "measurementHost": "local-docker-linux-arm64", + "campaignOutcome": "passed", + "finalModelRequests": 4, + "measuredAt": "2026-10-27T20:59:06.423Z", + "resultSha256": "4f31f46c7ae78a47a8157e19657dcd4794bb96e651e5114ce57652eadece24e6", + "ledgerSha256": "5dc4529f3f3e4173615f0bbc31a65394c3fc94677f2debb32e06c0bb5bb48405", + "ledgerFinalHash": "481b0aa38bb45d5308ffdbc73b7bdbb2827ca427c1894430900a59395ce1e9a1" }, { "executionId": "feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion", - "attemptId": "3849bc83e59ccddfa964120053d3428c", - "sourceRevision": "d1cd2085989d389337a1f5c92cce5290947a444c", - "definitionHash": "f674d36b9d45e3521a29edf5018d3139770887dee13014c040eb84807ddd6b92", - "knownModelUsd": "0.303945600", - "attemptDurationMs": 38262, + "attemptId": "6d01c1d288747c84c0f2424e4e7a8042", + "sourceRevision": "bf3ef12f3a78d0b15a32ad57f7e16520fbf6cd5e", + "definitionHash": "df18788acadb987274bec4c1e4598147232bdee7db9b4848ceb7d7906ae31a4c", + "knownModelUsd": "0.304456800", + "attemptDurationMs": 54741, "model": "claude-sonnet-5", "environmentId": "local", "runtimeMode": "native", "route": "braintrust", "priceVersion": "anthropic-standard-2026-10-20", - "measuredAt": "2026-10-21T16:52:21.240Z", - "resultSha256": "c086648b3dfc67a6d6ee04bb66a1914b3db69f9d40457198e2ce53c3a752ac24", - "ledgerSha256": "a2d84fbcb15087f1ae8e77c8b9f1bbfcf5aab5f06b5ca97c33c5507f2a649756", - "ledgerFinalHash": "475da37e24a069604866e47757d70bcee465da746ea866366af79ff5208ac16e" + "measurementHost": "github-hosted-ubuntu-latest", + "campaignOutcome": "failed_report_dependency_setup", + "finalModelRequests": 5, + "measuredAt": "2026-10-27T20:23:36.857Z", + "resultSha256": "481ab702917e83f520c597a1e2b88ce69e6a34dcd21e1b3961d7aa728547f42c", + "ledgerSha256": "b94a48f61471050559d5cd217ed9d3e2270e7d342b69ad4caa236ae1bbfc916f", + "ledgerFinalHash": "eb4a8aff44281c8f2ff420f137496e58513beb9afabee1cce2a560ebb056e2ac" + }, + { + "executionId": "feedback-persistence.runner-codex.local.complaint-and-suggestion", + "attemptId": "f88c1b5843c4772ac7082781053611b2", + "sourceRevision": "bf3ef12f3a78d0b15a32ad57f7e16520fbf6cd5e", + "definitionHash": "df18788acadb987274bec4c1e4598147232bdee7db9b4848ceb7d7906ae31a4c", + "knownModelUsd": "0.191163400", + "attemptDurationMs": 30591, + "model": "gpt-5.6-sol", + "environmentId": "local", + "runtimeMode": "native", + "route": "braintrust", + "priceVersion": "openai-standard-2026-10-20", + "measurementHost": "local-docker-linux-arm64", + "campaignOutcome": "passed", + "finalModelRequests": 5, + "measuredAt": "2026-10-27T20:58:30.021Z", + "resultSha256": "d0520306ac87e321196a8ba4746ab9e674177a453093e23dd250ede853df1a5c", + "ledgerSha256": "5dc4529f3f3e4173615f0bbc31a65394c3fc94677f2debb32e06c0bb5bb48405", + "ledgerFinalHash": "481b0aa38bb45d5308ffdbc73b7bdbb2827ca427c1894430900a59395ce1e9a1" } ], "limitations": [ "One successful sample per cell; no measured slow-tail distribution or predicted failure rate", - "Gateway estimates include setup requests; historical phase labels may be inaccurate", - "Attempt time includes setup and teardown; it excludes queue, dependency installation and compilation", - "No hosted timing qualification or complete infrastructure, storage and subscription allocation", - "No change to the conservative per-request reservations or approved budgets" + "Gateway estimates include setup, background and disconnected-client requests with terminal usage", + "Attempt time includes setup, execution and teardown; it excludes workflow queue, installation and compilation", + "Claude samples ran on GitHub hosted Ubuntu; Codex samples ran in a local Linux ARM64 container", + "The hosted campaign failed only during report dependency setup; original workflow and grader outcomes remain retained", + "Provider usage exports match; dollars remain estimates pending final monthly invoices and non-model reconciliation", + "No change to conservative per-request reservations, approved budgets or recurrence gates" ] } diff --git a/tests/runner-e2e/scheduled-operations.test.ts b/tests/runner-e2e/scheduled-operations.test.ts index 5fc31097dfe..7157a0efc89 100644 --- a/tests/runner-e2e/scheduled-operations.test.ts +++ b/tests/runner-e2e/scheduled-operations.test.ts @@ -62,7 +62,8 @@ it("fails closed before paid work if disabled, missing project identity, or pref const prepared = await prepareScheduledCampaign(env, input.now); expect(prepared.manifest.limits.campaignId).toBe(prepared.campaignId); const forecast = JSON.parse(await readFile(path.join(path.dirname(prepared.manifestPath), "forecast.json"), "utf8")); - expect(forecast.measurements.historicalSuccessfulSelection).toEqual({ modelUsd: "0.427624800", summedAttemptMs: 72757 }); + expect(forecast.measurements.historicalSuccessfulSelection).toEqual({ modelUsd: "0.428650100", summedAttemptMs: 107286 }); + expect(forecast.measurements.historicalHostedSelection).toEqual({ modelUsd: "0.428650100", summedAttemptMs: 107286 }); expect(forecast.measurements.hostedAttemptMs).toBeNull(); expect(forecast.maxUsd).toBe("20"); await expect(prepareScheduledCampaign(env, input.now)).rejects.toThrow("EEXIST"); diff --git a/tests/runner-e2e/scheduled-operations.ts b/tests/runner-e2e/scheduled-operations.ts index c2a5e80cfcb..b5abf3fa504 100644 --- a/tests/runner-e2e/scheduled-operations.ts +++ b/tests/runner-e2e/scheduled-operations.ts @@ -94,8 +94,8 @@ export async function prepareScheduledCampaign(env: NodeJS.ProcessEnv, now = Dat const forecast = { cadence, executionIds: manifest.executionIds, maxUsd: manifest.limits.maxUsd, overhead: manifest.limits.overhead, budgetScope: "model requests plus declared operating overhead reservations", measurements: scheduledForecast(runnerMatrix.filter(execution => manifest.executionIds.includes(execution.id)), manifest.models), - basis: "Retained local pilot measurements; successful samples and matching configurations are reported separately", - uncertainty: "Incomplete provider receipts and no invoice reconciliation; estimates are not a guarantee", + basis: "Retained attempt measurements with per-sample hosts; historical and current-compatible totals remain separate", + uncertainty: "Sample coverage, definition drift and provider export precision limit the estimate; final invoices and operating reconciliation remain pending", maximumRequestReservations: manifest.models.map(model => ({ model: model.price.model, usd: evalRequestReservation({ ...model, apiKey: "forecast", mode: "braintrust" }) })), maximumMonthlyTotalAllowanceUsd: "1120", pricingExpiresAt: prices.expiresAt };