test(evals): verify scheduled model readiness and policy hash binding

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/OPERATIONS.md

@@ -223,7 +223,8 @@ stable key-sorted JSON hashing. The model configuration contains `provider`,
 `mode` (the manifest route), `price`, `maximumInputTokens`,
 `maximumOutputTokens`, and `reservation` (the nine-decimal value from
 `evalRequestReservation`). An operating configuration contains allowance
-`reference`, `runner`, `verifiedAt`, `expiresAt`, and a `component` object with
+`reference`, `runner`, `verifiedAt`, `expiresAt`, optional `accountingReadiness`
+(only when present), and a `component` object with
 its `component`, `maxUsd`, and `reference`. Changing any of those values
 invalidates the approval hash. Evidence metadata itself is excluded from this
 configuration hash.
@@ -897,3 +898,14 @@ an operations manifest. They use the same metered connection, prices, campaign
 journal, setup boundary, and durable attempt allowance as core compatibility.
 Collection and the independent saved-evidence oracle require no additional
 model call. The synthetic items need review as test data, not product incidents.
+
+
+### Qualify model spend while operating reconciliation is pending
+
+Set `accountingReadiness: "model_requests"` on the reviewed allowance to use
+complete model receipts as the scheduled readiness gate. This is an explicit
+operator policy. The report retains the full-accounting status, operating
+reservations and model-readiness status separately. It does not release any
+historical hold or certify a full-lifecycle vendor cost bound. Missing model
+receipts, conflicts, active requests, excess commitments, and the existing
+product, cleanup, feedback and trace checks still fail qualification.

tests/runner-e2e/operating-overhead.test.ts

@@ -3,9 +3,11 @@ import { tmpdir } from "node:os";
 import path from "node:path";
 import { expect, it } from "vitest";
 import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js";
-import { EVAL_OVERHEAD_COMPONENTS, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";
+import { EVAL_OVERHEAD_COMPONENTS, operatingReservationBound, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js";
 import { evalAccountingStatus, evalReadinessStatus, readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js";
 
+import { costBoundConfigurationSha256 } from "./cost-bound-evidence.js";
+
 const now = Date.parse("2026-10-21T00:00:00Z");
 const allowance = (): EvalOperatingOverhead => ({ version: 1, reference: "fixture-only", runner: "aws-fleet",
   verifiedAt: "2026-10-20T00:00:00Z", expiresAt: "2026-11-19T00:00:00Z",
@@ -151,3 +153,17 @@ it("rejects unknown readiness policies instead of silently loosening accounting"
   expect(() => validateOperatingOverhead({ ...allowance(), accountingReadiness: "ignore_costs" } as never, "20"))
     .toThrow("Invalid accounting readiness policy");
 });
+
+it("binds optional readiness policy to reviewed operating evidence without changing legacy hash recipes", () => {
+  for (const accountingReadiness of [undefined, "model_requests"] as const) {
+    const value = { ...allowance(), ...(accountingReadiness ? { accountingReadiness } : {}) };
+    const component = value.components[0]!;
+    const calculation = { reference: value.reference, runner: value.runner, verifiedAt: value.verifiedAt,
+      expiresAt: value.expiresAt, component: { ...component }, ...(accountingReadiness ? { accountingReadiness } : {}) };
+    component.provisionalBound = { version: 1, scope: "full_lifecycle", evidenceSha256: "a".repeat(64),
+      configurationSha256: costBoundConfigurationSha256(calculation) };
+    expect(operatingReservationBound(value, component)?.calculationSha256).toBe(component.provisionalBound.configurationSha256);
+    const changed = { ...value, accountingReadiness: accountingReadiness ? undefined : "model_requests" as const };
+    expect(() => operatingReservationBound(changed, component)).toThrow("Reviewed cost bound configuration changed");
+  }
+});

tests/runner-e2e/scheduled-finalization.test.ts

@@ -1,3 +1,9 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { openEvalCostLedger } from "./cost-ledger.js";
+import { readEvalOperationsReport } from "./operations-report.js";
+import { EVAL_OVERHEAD_COMPONENTS } from "./operating-overhead.js";
 import { expect, it } from "vitest";
 import { buildRunnerCampaign } from "./history.js";
 import { runnerMatrix } from "./catalog.js";
@@ -52,3 +58,31 @@ it("keeps incomplete accounting and trace export visible independently of a pass
   campaign.feedbackReview.decisions[0]!.disposition = "needs_investigation";
   expect(scheduledCampaignFindings(campaign)).toContain("feedback_needs_review");
 });
+
+it("qualifies scheduled model receipts while retaining operating holds, but rejects missing usage and overruns", async () => {
+  const { campaign } = fixture();
+  const directory = await mkdtemp(path.join(tmpdir(), "scheduled-model-readiness-"));
+  const file = path.join(directory, "ledger.jsonl"), now = Date.now();
+  const ledger = await openEvalCostLedger(file, { campaignId: "scheduled-fixture", maxUsd: "20",
+    deadlineAt: new Date(now + 60_000).toISOString(), maxRequests: 10, maxConcurrent: 1,
+    overhead: { version: 1, accountingReadiness: "model_requests", reference: "synthetic-fixture", runner: "github-hosted",
+      verifiedAt: new Date(now - 1000).toISOString(), expiresAt: new Date(now + 86_400_000).toISOString(),
+      components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: "synthetic-fixture" })) },
+    maxAttemptsPerExecution: 1 });
+  try {
+    await ledger.admitAttempt(campaign.expected[0]!, "one");
+    await ledger.reserve("model", { executionId: campaign.expected[0]!, attemptId: "one", phase: "setup" }, "2");
+    await ledger.finish("model", "cancelled");
+    campaign.operations = await readEvalOperationsReport(file);
+    expect(scheduledCampaignFindings(campaign)).toContain("accounting_incomplete");
+    await ledger.observe({ observationId: "model-final", requestId: "model", source: "provider", provider: "anthropic",
+      currency: "USD", route: "direct", classification: "reported", costUsd: "0.5", final: true });
+    campaign.operations = await readEvalOperationsReport(file);
+    expect(scheduledCampaignFindings(campaign)).not.toContain("accounting_incomplete");
+    expect(campaign.operations).toMatchObject({ heldUsd: "4.000000000", committedUsd: "4.500000000" });
+    await ledger.observe({ observationId: "overrun", requestId: "overhead:ci_compute", source: "invoice", provider: "aws",
+      currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true });
+    campaign.operations = await readEvalOperationsReport(file);
+    expect(scheduledCampaignFindings(campaign)).toEqual(expect.arrayContaining(["accounting_incomplete", "accounting_over_budget"]));
+  } finally { await ledger.close(); await rm(directory, { recursive: true, force: true }); }
+});