diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 7d0015d3a6a..1481b4451b1 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -223,7 +223,8 @@ stable key-sorted JSON hashing. The model configuration contains `provider`, `mode` (the manifest route), `price`, `maximumInputTokens`, `maximumOutputTokens`, and `reservation` (the nine-decimal value from `evalRequestReservation`). An operating configuration contains allowance -`reference`, `runner`, `verifiedAt`, `expiresAt`, and a `component` object with +`reference`, `runner`, `verifiedAt`, `expiresAt`, optional `accountingReadiness` +(only when present), and a `component` object with its `component`, `maxUsd`, and `reference`. Changing any of those values invalidates the approval hash. Evidence metadata itself is excluded from this configuration hash. @@ -897,3 +898,14 @@ an operations manifest. They use the same metered connection, prices, campaign journal, setup boundary, and durable attempt allowance as core compatibility. Collection and the independent saved-evidence oracle require no additional model call. The synthetic items need review as test data, not product incidents. + + +### Qualify model spend while operating reconciliation is pending + +Set `accountingReadiness: "model_requests"` on the reviewed allowance to use +complete model receipts as the scheduled readiness gate. This is an explicit +operator policy. The report retains the full-accounting status, operating +reservations and model-readiness status separately. It does not release any +historical hold or certify a full-lifecycle vendor cost bound. Missing model +receipts, conflicts, active requests, excess commitments, and the existing +product, cleanup, feedback and trace checks still fail qualification. diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.ts index cdf6800a9d9..85bdeedf1a1 100644 --- a/tests/runner-e2e/operating-overhead.test.ts +++ b/tests/runner-e2e/operating-overhead.test.ts @@ -3,9 +3,11 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { expect, it } from "vitest"; import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js"; -import { EVAL_OVERHEAD_COMPONENTS, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; +import { EVAL_OVERHEAD_COMPONENTS, operatingReservationBound, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; import { evalAccountingStatus, evalReadinessStatus, readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js"; +import { costBoundConfigurationSha256 } from "./cost-bound-evidence.js"; + const now = Date.parse("2026-10-21T00:00:00Z"); const allowance = (): EvalOperatingOverhead => ({ version: 1, reference: "fixture-only", runner: "aws-fleet", verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z", @@ -151,3 +153,17 @@ it("rejects unknown readiness policies instead of silently loosening accounting" expect(() => validateOperatingOverhead({ ...allowance(), accountingReadiness: "ignore_costs" } as never, "20")) .toThrow("Invalid accounting readiness policy"); }); + +it("binds optional readiness policy to reviewed operating evidence without changing legacy hash recipes", () => { + for (const accountingReadiness of [undefined, "model_requests"] as const) { + const value = { ...allowance(), ...(accountingReadiness ? { accountingReadiness } : {}) }; + const component = value.components[0]!; + const calculation = { reference: value.reference, runner: value.runner, verifiedAt: value.verifiedAt, + expiresAt: value.expiresAt, component: { ...component }, ...(accountingReadiness ? { accountingReadiness } : {}) }; + component.provisionalBound = { version: 1, scope: "full_lifecycle", evidenceSha256: "a".repeat(64), + configurationSha256: costBoundConfigurationSha256(calculation) }; + expect(operatingReservationBound(value, component)?.calculationSha256).toBe(component.provisionalBound.configurationSha256); + const changed = { ...value, accountingReadiness: accountingReadiness ? undefined : "model_requests" as const }; + expect(() => operatingReservationBound(changed, component)).toThrow("Reviewed cost bound configuration changed"); + } +}); diff --git a/tests/runner-e2e/scheduled-finalization.test.ts b/tests/runner-e2e/scheduled-finalization.test.ts index 8b90c9fd377..5f740c4c5b2 100644 --- a/tests/runner-e2e/scheduled-finalization.test.ts +++ b/tests/runner-e2e/scheduled-finalization.test.ts @@ -1,3 +1,9 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { openEvalCostLedger } from "./cost-ledger.js"; +import { readEvalOperationsReport } from "./operations-report.js"; +import { EVAL_OVERHEAD_COMPONENTS } from "./operating-overhead.js"; import { expect, it } from "vitest"; import { buildRunnerCampaign } from "./history.js"; import { runnerMatrix } from "./catalog.js"; @@ -52,3 +58,31 @@ it("keeps incomplete accounting and trace export visible independently of a pass campaign.feedbackReview.decisions[0]!.disposition = "needs_investigation"; expect(scheduledCampaignFindings(campaign)).toContain("feedback_needs_review"); }); + +it("qualifies scheduled model receipts while retaining operating holds, but rejects missing usage and overruns", async () => { + const { campaign } = fixture(); + const directory = await mkdtemp(path.join(tmpdir(), "scheduled-model-readiness-")); + const file = path.join(directory, "ledger.jsonl"), now = Date.now(); + const ledger = await openEvalCostLedger(file, { campaignId: "scheduled-fixture", maxUsd: "20", + deadlineAt: new Date(now + 60_000).toISOString(), maxRequests: 10, maxConcurrent: 1, + overhead: { version: 1, accountingReadiness: "model_requests", reference: "synthetic-fixture", runner: "github-hosted", + verifiedAt: new Date(now - 1000).toISOString(), expiresAt: new Date(now + 86_400_000).toISOString(), + components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: "synthetic-fixture" })) }, + maxAttemptsPerExecution: 1 }); + try { + await ledger.admitAttempt(campaign.expected[0]!, "one"); + await ledger.reserve("model", { executionId: campaign.expected[0]!, attemptId: "one", phase: "setup" }, "2"); + await ledger.finish("model", "cancelled"); + campaign.operations = await readEvalOperationsReport(file); + expect(scheduledCampaignFindings(campaign)).toContain("accounting_incomplete"); + await ledger.observe({ observationId: "model-final", requestId: "model", source: "provider", provider: "anthropic", + currency: "USD", route: "direct", classification: "reported", costUsd: "0.5", final: true }); + campaign.operations = await readEvalOperationsReport(file); + expect(scheduledCampaignFindings(campaign)).not.toContain("accounting_incomplete"); + expect(campaign.operations).toMatchObject({ heldUsd: "4.000000000", committedUsd: "4.500000000" }); + await ledger.observe({ observationId: "overrun", requestId: "overhead:ci_compute", source: "invoice", provider: "aws", + currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true }); + campaign.operations = await readEvalOperationsReport(file); + expect(scheduledCampaignFindings(campaign)).toEqual(expect.arrayContaining(["accounting_incomplete", "accounting_over_budget"])); + } finally { await ledger.close(); await rm(directory, { recursive: true, force: true }); } +});