diff --git a/tests/runner-e2e/OPERATIONS.md b/tests/runner-e2e/OPERATIONS.md index 7d0015d3a6a..1481b4451b1 100644 --- a/tests/runner-e2e/OPERATIONS.md +++ b/tests/runner-e2e/OPERATIONS.md @@ -223,7 +223,8 @@ stable key-sorted JSON hashing. The model configuration contains `provider`, `mode` (the manifest route), `price`, `maximumInputTokens`, `maximumOutputTokens`, and `reservation` (the nine-decimal value from `evalRequestReservation`). An operating configuration contains allowance -`reference`, `runner`, `verifiedAt`, `expiresAt`, and a `component` object with +`reference`, `runner`, `verifiedAt`, `expiresAt`, optional `accountingReadiness` +(only when present), and a `component` object with its `component`, `maxUsd`, and `reference`. Changing any of those values invalidates the approval hash. Evidence metadata itself is excluded from this configuration hash. @@ -897,3 +898,14 @@ an operations manifest. They use the same metered connection, prices, campaign journal, setup boundary, and durable attempt allowance as core compatibility. Collection and the independent saved-evidence oracle require no additional model call. The synthetic items need review as test data, not product incidents. + + +### Qualify model spend while operating reconciliation is pending + +Set `accountingReadiness: "model_requests"` on the reviewed allowance to use +complete model receipts as the scheduled readiness gate. This is an explicit +operator policy. The report retains the full-accounting status, operating +reservations and model-readiness status separately. It does not release any +historical hold or certify a full-lifecycle vendor cost bound. Missing model +receipts, conflicts, active requests, excess commitments, and the existing +product, cleanup, feedback and trace checks still fail qualification. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index a97116cc818..554b10b32d1 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -2983,3 +2983,22 @@ quality or survival after an abrupt worker kill. pnpm test:e2e:runner -- --list --suite feedback-persistence pnpm test:e2e:runner -- --id feedback-persistence.runner-acpx-claude.local.complaint-and-suggestion --max-automatic-retries 0 --max-parallel 1 ``` + + +### Model-spend readiness and operating reconciliation + +A reviewed operating allowance can opt into `accountingReadiness: "model_requests"`. +This policy requires at least one model request and complete final model-cost +receipts before scheduled qualification succeeds. Missing or conflicting model +receipts, active requests, failed product/cleanup/trace/feedback checks, and a +campaign commitment above its limit still fail qualification. The gateway still +reserves every request before forwarding it. Attempts, deadlines and concurrency +remain bounded. + +The report shows the model subtotal and model-readiness status separately from +full accounting status. Unknown operating charges remain unknown. Operating +reservations stay in the same campaign admission allowance until reconciled; +they are not reported as charges or converted into reviewed lifecycle bounds. +This mode therefore does not certify a full-lifecycle operating-cost ceiling. +Omitting the policy preserves the stricter full-accounting gate. Historical +journals and their holds are not rewritten by changing future campaign policy. diff --git a/tests/runner-e2e/chat-scenario-catalog.test.ts b/tests/runner-e2e/chat-scenario-catalog.test.ts index eef37691e65..a2be85fd04f 100644 --- a/tests/runner-e2e/chat-scenario-catalog.test.ts +++ b/tests/runner-e2e/chat-scenario-catalog.test.ts @@ -37,7 +37,7 @@ it("validates the complete planned chat catalog against its schema and feature m } expect(ids.size).toBe(244); expect([...covered].sort()).toEqual([...mapped].sort()); - expect(mapped.size).toBe(191); + expect(mapped.size).toBe(192); const defaults = parseDocument(readFileSync(new URL("defaults.yaml", catalog), "utf8"), { uniqueKeys: true }); expect(defaults.errors).toEqual([]); expect(defaults.toJS()).toMatchObject({ schema_version: 1, status: "planned" }); diff --git a/tests/runner-e2e/chat-scenarios/runs-adapters.yaml b/tests/runner-e2e/chat-scenarios/runs-adapters.yaml index cdabc6614e1..7d7a3fd2610 100644 --- a/tests/runner-e2e/chat-scenarios/runs-adapters.yaml +++ b/tests/runner-e2e/chat-scenarios/runs-adapters.yaml @@ -7,6 +7,7 @@ scenarios: - id: runs-adapters.list-model-options capabilities: - runs-adapters/availability + - runs-adapters/model-accounts context: >- The company has admitted model discovery and configured coding accounts; some adapter inventories remain board-only. diff --git a/tests/runner-e2e/daytona-image-content.ts b/tests/runner-e2e/daytona-image-content.ts index aae91fe8ccd..8ab17a54a2e 100644 --- a/tests/runner-e2e/daytona-image-content.ts +++ b/tests/runner-e2e/daytona-image-content.ts @@ -66,6 +66,7 @@ export const DAYTONA_IMAGE_INPUT_PATHS = [ "packages/paperclip-runner/test/fixtures/copilot-profile-v57-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v58-identity.json", "packages/paperclip-runner/test/fixtures/copilot-profile-v61-identity.json", + "packages/paperclip-runner/test/fixtures/copilot-profile-v65-identity.json", "packages/paperclip-runner/scripts/materialize-cursor-distribution.mjs", "packages/paperclip-runner/scripts/cursor-runtime-patch.mjs", "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs", diff --git a/tests/runner-e2e/operating-overhead.test.ts b/tests/runner-e2e/operating-overhead.test.ts index c837450da00..85bdeedf1a1 100644 --- a/tests/runner-e2e/operating-overhead.test.ts +++ b/tests/runner-e2e/operating-overhead.test.ts @@ -3,8 +3,10 @@ import { tmpdir } from "node:os"; import path from "node:path"; import { expect, it } from "vitest"; import { EvalSpendState, openEvalCostLedger, readEvalCostLedger, type EvalSpendLimits } from "./cost-ledger.js"; -import { EVAL_OVERHEAD_COMPONENTS, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; -import { readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js"; +import { EVAL_OVERHEAD_COMPONENTS, operatingReservationBound, scheduledOperatingOverhead, validateOperatingOverhead, type EvalOperatingOverhead } from "./operating-overhead.js"; +import { evalAccountingStatus, evalReadinessStatus, readEvalOperationsReport, renderEvalOperationsReport } from "./operations-report.js"; + +import { costBoundConfigurationSha256 } from "./cost-bound-evidence.js"; const now = Date.parse("2026-10-21T00:00:00Z"); const allowance = (): EvalOperatingOverhead => ({ version: 1, reference: "fixture-only", runner: "aws-fleet", @@ -117,3 +119,51 @@ it("stops new admission after allowance expiry while preserving late invoice rec expect(() => state.validate({ type: "observe", observation: { observationId: "late-invoice", requestId: "overhead:ci_compute", source: "invoice", provider: "aws", currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "0.5", final: true } }, expiredAt)).not.toThrow(); }); + + +it("separates model readiness from unresolved overhead without releasing holds or masking model gaps", async () => { + const directory = await mkdtemp(path.join(tmpdir(), "eval-model-readiness-")); + const file = path.join(directory, "ledger.jsonl"); + const overhead = { ...currentAllowance(), accountingReadiness: "model_requests" as const }; + const ledger = await openEvalCostLedger(file, { campaignId: "model-readiness", maxUsd: "20", + deadlineAt: "2099-01-01T00:00:00Z", maxRequests: 10, maxConcurrent: 1, maxAttemptsPerExecution: 1, overhead }); + try { + expect(evalReadinessStatus(await readEvalOperationsReport(file))).toBe("blocked"); + await ledger.admitAttempt(modelIdentity.executionId, modelIdentity.attemptId); + await ledger.reserve("model", modelIdentity, "2"); + expect(evalReadinessStatus(await readEvalOperationsReport(file))).toBe("blocked"); + await ledger.finish("model", "cancelled"); + expect(evalReadinessStatus(await readEvalOperationsReport(file))).toBe("blocked"); + await ledger.observe({ observationId: "model-final", requestId: "model", source: "provider", provider: "anthropic", + currency: "USD", route: "direct", classification: "reported", costUsd: "0.5", final: true }); + const report = await readEvalOperationsReport(file); + expect(evalAccountingStatus(report)).toBe("blocked"); + expect(evalReadinessStatus(report)).toBe("receipts_complete"); + expect(evalReadinessStatus({ ...report, accountingReadiness: undefined } as never)).toBe("blocked"); + expect(report).toMatchObject({ accountingReadiness: "model_requests", heldUsd: "4.000000000", + committedUsd: "4.500000000", modelAccounting: { requests: 1, knownUsd: "0.500000000", unresolvedRequests: 0 } }); + expect(renderEvalOperationsReport(report)).toContain("Operating costs remain separately unreconciled"); + await ledger.observe({ observationId: "overrun", requestId: "overhead:ci_compute", source: "invoice", provider: "aws", + currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true }); + expect(evalReadinessStatus(await readEvalOperationsReport(file))).toBe("blocked"); + } finally { await ledger.close(); await rm(directory, { recursive: true, force: true }); } +}); + +it("rejects unknown readiness policies instead of silently loosening accounting", () => { + expect(() => validateOperatingOverhead({ ...allowance(), accountingReadiness: "ignore_costs" } as never, "20")) + .toThrow("Invalid accounting readiness policy"); +}); + +it("binds optional readiness policy to reviewed operating evidence without changing legacy hash recipes", () => { + for (const accountingReadiness of [undefined, "model_requests"] as const) { + const value = { ...allowance(), ...(accountingReadiness ? { accountingReadiness } : {}) }; + const component = value.components[0]!; + const calculation = { reference: value.reference, runner: value.runner, verifiedAt: value.verifiedAt, + expiresAt: value.expiresAt, component: { ...component }, ...(accountingReadiness ? { accountingReadiness } : {}) }; + component.provisionalBound = { version: 1, scope: "full_lifecycle", evidenceSha256: "a".repeat(64), + configurationSha256: costBoundConfigurationSha256(calculation) }; + expect(operatingReservationBound(value, component)?.calculationSha256).toBe(component.provisionalBound.configurationSha256); + const changed = { ...value, accountingReadiness: accountingReadiness ? undefined : "model_requests" as const }; + expect(() => operatingReservationBound(changed, component)).toThrow("Reviewed cost bound configuration changed"); + } +}); diff --git a/tests/runner-e2e/operating-overhead.ts b/tests/runner-e2e/operating-overhead.ts index 43eb5de338e..40c57283fc5 100644 --- a/tests/runner-e2e/operating-overhead.ts +++ b/tests/runner-e2e/operating-overhead.ts @@ -4,6 +4,7 @@ import { usdToUnits } from "../../packages/shared/src/money.js"; export const EVAL_OVERHEAD_COMPONENTS = ["ci_compute", "artifact_storage", "observability", "shared_subscription"] as const; export interface EvalOperatingOverhead { version: 1; + accountingReadiness?: "model_requests"; reference: string; runner: "github-hosted" | "aws-fleet"; verifiedAt: string; @@ -20,13 +21,15 @@ const reference = (value: unknown) => typeof value === "string" && /^[A-Za-z0-9] export function operatingReservationBound(allowance: EvalOperatingOverhead, component: EvalOperatingOverhead["components"][number]) { if (component.provisionalBound === undefined) return undefined; return reservationBound("operating_allowance", component.provisionalBound, { + ...(allowance.accountingReadiness ? { accountingReadiness: allowance.accountingReadiness } : {}), reference: allowance.reference, runner: allowance.runner, verifiedAt: allowance.verifiedAt, expiresAt: allowance.expiresAt, component: { component: component.component, maxUsd: component.maxUsd, reference: component.reference }, }); } export function validateOperatingOverhead(value: EvalOperatingOverhead, totalUsd: string): bigint { - object(value, ["version", "reference", "runner", "verifiedAt", "expiresAt", "components"]); + object(value, ["version", "accountingReadiness", "reference", "runner", "verifiedAt", "expiresAt", "components"]); + if (value.accountingReadiness !== undefined && value.accountingReadiness !== "model_requests") throw new Error("Invalid accounting readiness policy"); if (value.version !== 1 || !reference(value.reference) || !["github-hosted", "aws-fleet"].includes(value.runner) || typeof value.verifiedAt !== "string" || typeof value.expiresAt !== "string" || !Number.isFinite(Date.parse(value.verifiedAt)) || !Number.isFinite(Date.parse(value.expiresAt)) diff --git a/tests/runner-e2e/operations-report.ts b/tests/runner-e2e/operations-report.ts index b751d24bed6..80c3affe067 100644 --- a/tests/runner-e2e/operations-report.ts +++ b/tests/runner-e2e/operations-report.ts @@ -7,6 +7,8 @@ interface EvalOperationsAvailableReport { version: 1; scope: "shared_gateway_ledger"; status?: never; + accountingReadiness?: "model_requests"; + modelAccounting?: { requests: number; unresolvedRequests: number; conflictingRequests: number; inflight: number; knownUsd: string; heldUsd: string }; ledgerCampaignId: string; journalHash: string; records: number; @@ -39,7 +41,7 @@ export function validateEvalOperationsReport(report: EvalOperationsReport): void } const dollars = (value: unknown): bigint => typeof value === "string" && /^(?:0|[1-9][0-9]*)\.[0-9]{9}$/.test(value) ? usdToUnits(value) : fail(); const count = (value: unknown) => { if (!Number.isSafeInteger(value) || Number(value) < 0) fail(); }; - fields(report, ["version", "scope", "ledgerCampaignId", "journalHash", "records", "deadlineAt", "limitUsd", "knownUsd", "heldUsd", "committedUsd", "requests", "unresolvedRequests", "boundedUnresolvedRequests", "conflictingRequests", "inflight", "breakdown", "attempts"]); + fields(report, ["version", "scope", "accountingReadiness", "modelAccounting", "ledgerCampaignId", "journalHash", "records", "deadlineAt", "limitUsd", "knownUsd", "heldUsd", "committedUsd", "requests", "unresolvedRequests", "boundedUnresolvedRequests", "conflictingRequests", "inflight", "breakdown", "attempts"]); if (report.version !== 1 || report.scope !== "shared_gateway_ledger" || !validEvalLedgerId(report.ledgerCampaignId) || !/^[a-f0-9]{64}$/.test(report.journalHash) || !isPublicReportTimestamp(report.deadlineAt) || !Array.isArray(report.attempts)) fail(); [report.records, report.requests, report.unresolvedRequests, report.conflictingRequests, report.inflight].forEach(count); @@ -48,6 +50,16 @@ export function validateEvalOperationsReport(report: EvalOperationsReport): void count(report.boundedUnresolvedRequests); if (report.boundedUnresolvedRequests > report.unresolvedRequests - report.conflictingRequests) fail(); } + if (report.accountingReadiness !== undefined && (report.accountingReadiness !== "model_requests" || !report.modelAccounting)) fail(); + if (report.modelAccounting !== undefined) { + const model = report.modelAccounting; + fields(model, ["requests", "unresolvedRequests", "conflictingRequests", "inflight", "knownUsd", "heldUsd"]); + [model.requests, model.unresolvedRequests, model.conflictingRequests, model.inflight].forEach(count); + if (model.requests > report.requests || model.unresolvedRequests > model.requests || model.unresolvedRequests > report.unresolvedRequests + || model.conflictingRequests > model.unresolvedRequests || model.conflictingRequests > report.conflictingRequests + || model.inflight > model.requests || model.inflight > report.inflight + || dollars(model.knownUsd) > dollars(report.knownUsd) || dollars(model.heldUsd) > dollars(report.heldUsd)) fail(); + } dollars(report.limitUsd); const known = dollars(report.knownUsd), held = dollars(report.heldUsd); if (known + held !== dollars(report.committedUsd)) fail(); @@ -86,7 +98,17 @@ async function readAvailableEvalOperationsReport(file: string): Promise request.phase !== "infrastructure"); + const modelKnown = modelRequests.reduce((sum, request) => sum + usdToUnits(request.costUsd ?? "0"), 0n); + const modelHeld = modelRequests.reduce((sum, request) => { + const cost = usdToUnits(request.costUsd ?? "0"), reserved = usdToUnits(request.reservedUsd); + return sum + ((!request.finished || !request.receiptComplete) && reserved > cost ? reserved - cost : 0n); + }, 0n); const report: EvalOperationsAvailableReport = { version: 1, scope: "shared_gateway_ledger", ledgerCampaignId: snapshot.campaignId, + ...(loaded.state.limits.overhead?.accountingReadiness ? { accountingReadiness: loaded.state.limits.overhead.accountingReadiness } : {}), + modelAccounting: { requests: modelRequests.length, unresolvedRequests: modelRequests.filter(request => !request.receiptComplete).length, + conflictingRequests: modelRequests.filter(request => request.conflict).length, inflight: modelRequests.filter(request => !request.finished).length, + knownUsd: money(modelKnown), heldUsd: money(modelHeld) }, journalHash: loaded.previous, records: loaded.sequence, deadlineAt: new Date(loaded.state.limits.deadlineAt).toISOString(), limitUsd: snapshot.limitUsd, knownUsd: snapshot.knownUsd, heldUsd: snapshot.heldUsd, committedUsd: snapshot.committedUsd, requests: snapshot.requests.length, unresolvedRequests: snapshot.unknownRequests, boundedUnresolvedRequests: snapshot.boundedUnresolvedRequests, conflictingRequests: snapshot.conflictingRequests, @@ -103,12 +125,23 @@ export function evalAccountingStatus(report: EvalOperationsReport): "unavailable return report.unresolvedRequests ? "provisional" : "receipts_complete"; } +export function evalReadinessStatus(report: EvalOperationsReport): ReturnType { + const full = evalAccountingStatus(report); + if (report.status === "unavailable" || report.accountingReadiness !== "model_requests") return full; + const model = report.modelAccounting!; + if (!model.requests || model.unresolvedRequests || model.inflight || report.conflictingRequests || report.inflight + || usdToUnits(report.committedUsd) > usdToUnits(report.limitUsd)) return "blocked"; + return "receipts_complete"; +} + export function operationsReportText(report: EvalOperationsReport): string[] { if (report.status === "unavailable") return [ "Request ledger accounting unavailable: the journal could not be read or validated. Request amounts, counts, and held reservations are unknown.", "Retained test results are preserved. Recover and validate the original journal before resuming paid work; do not reset the allowance or treat missing amounts as zero.", ]; return [ + ...(report.modelAccounting ? [`Model requests: ${report.modelAccounting.requests}; $${report.modelAccounting.knownUsd} known token cost; $${report.modelAccounting.heldUsd} additional held; ${report.modelAccounting.unresolvedRequests} unresolved.`] : []), + ...(report.accountingReadiness === "model_requests" ? [`Readiness scope: model requests; status: ${evalReadinessStatus(report)}. Operating costs remain separately unreconciled unless final receipts establish otherwise. Their reservations remain inside campaign admission; this does not establish a full-lifecycle operating-cost bound.`] : []), `Accounting status: ${evalAccountingStatus(report)}. ${report.boundedUnresolvedRequests ?? 0} pending charges have reviewed bounds; ${report.unresolvedRequests - (report.boundedUnresolvedRequests ?? 0)} do not. Provisional completion preserves holds and requires later reconciliation; it does not establish invoice completeness.`, `Shared request ledger ${report.ledgerCampaignId}: $${report.knownUsd} known subtotal; $${report.heldUsd} additional held reservations; $${report.committedUsd} committed against $${report.limitUsd}.`, `Charges: $${report.breakdown.reconciled} reconciled; $${report.breakdown.reported} reported; $${report.breakdown.estimated} estimated; $${report.breakdown.allocated} allocated.`, diff --git a/tests/runner-e2e/scheduled-finalization.test.ts b/tests/runner-e2e/scheduled-finalization.test.ts index 8b90c9fd377..5f740c4c5b2 100644 --- a/tests/runner-e2e/scheduled-finalization.test.ts +++ b/tests/runner-e2e/scheduled-finalization.test.ts @@ -1,3 +1,9 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { openEvalCostLedger } from "./cost-ledger.js"; +import { readEvalOperationsReport } from "./operations-report.js"; +import { EVAL_OVERHEAD_COMPONENTS } from "./operating-overhead.js"; import { expect, it } from "vitest"; import { buildRunnerCampaign } from "./history.js"; import { runnerMatrix } from "./catalog.js"; @@ -52,3 +58,31 @@ it("keeps incomplete accounting and trace export visible independently of a pass campaign.feedbackReview.decisions[0]!.disposition = "needs_investigation"; expect(scheduledCampaignFindings(campaign)).toContain("feedback_needs_review"); }); + +it("qualifies scheduled model receipts while retaining operating holds, but rejects missing usage and overruns", async () => { + const { campaign } = fixture(); + const directory = await mkdtemp(path.join(tmpdir(), "scheduled-model-readiness-")); + const file = path.join(directory, "ledger.jsonl"), now = Date.now(); + const ledger = await openEvalCostLedger(file, { campaignId: "scheduled-fixture", maxUsd: "20", + deadlineAt: new Date(now + 60_000).toISOString(), maxRequests: 10, maxConcurrent: 1, + overhead: { version: 1, accountingReadiness: "model_requests", reference: "synthetic-fixture", runner: "github-hosted", + verifiedAt: new Date(now - 1000).toISOString(), expiresAt: new Date(now + 86_400_000).toISOString(), + components: EVAL_OVERHEAD_COMPONENTS.map(component => ({ component, maxUsd: "1", reference: "synthetic-fixture" })) }, + maxAttemptsPerExecution: 1 }); + try { + await ledger.admitAttempt(campaign.expected[0]!, "one"); + await ledger.reserve("model", { executionId: campaign.expected[0]!, attemptId: "one", phase: "setup" }, "2"); + await ledger.finish("model", "cancelled"); + campaign.operations = await readEvalOperationsReport(file); + expect(scheduledCampaignFindings(campaign)).toContain("accounting_incomplete"); + await ledger.observe({ observationId: "model-final", requestId: "model", source: "provider", provider: "anthropic", + currency: "USD", route: "direct", classification: "reported", costUsd: "0.5", final: true }); + campaign.operations = await readEvalOperationsReport(file); + expect(scheduledCampaignFindings(campaign)).not.toContain("accounting_incomplete"); + expect(campaign.operations).toMatchObject({ heldUsd: "4.000000000", committedUsd: "4.500000000" }); + await ledger.observe({ observationId: "overrun", requestId: "overhead:ci_compute", source: "invoice", provider: "aws", + currency: "USD", route: "infrastructure", classification: "reconciled", costUsd: "21", final: true }); + campaign.operations = await readEvalOperationsReport(file); + expect(scheduledCampaignFindings(campaign)).toEqual(expect.arrayContaining(["accounting_incomplete", "accounting_over_budget"])); + } finally { await ledger.close(); await rm(directory, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/scheduled-finalization.ts b/tests/runner-e2e/scheduled-finalization.ts index 5d93e4fe8be..ec31a870e53 100644 --- a/tests/runner-e2e/scheduled-finalization.ts +++ b/tests/runner-e2e/scheduled-finalization.ts @@ -1,5 +1,5 @@ import { usdToUnits } from "../../packages/shared/src/money.js"; -import { evalAccountingStatus } from "./operations-report.js"; +import { evalAccountingStatus, evalReadinessStatus } from "./operations-report.js"; import { readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { pathToFileURL } from "node:url"; @@ -49,7 +49,7 @@ export function scheduledCampaignFindings(campaign: RunnerE2ECampaign): string[] const operations = campaign.operations; if (!operations || operations.status === "unavailable") findings.push("accounting_unavailable"); else { - if (evalAccountingStatus(operations) === "blocked") findings.push("accounting_incomplete"); + if (evalReadinessStatus(operations) === "blocked") findings.push("accounting_incomplete"); if (usdToUnits(operations.committedUsd) > usdToUnits(operations.limitUsd)) findings.push("accounting_over_budget"); } const feedback = summarizeCampaignFeedback(campaign.accounting?.attempts ?? campaign.results); @@ -131,6 +131,7 @@ export async function finalizeScheduledCampaign(env: NodeJS.ProcessEnv) { await writeFile(path.join(directory, "findings.json"), `${JSON.stringify({ version: 1, findings, nativeIndexVerification: indexing, gatewayIndexVerification: gatewayIndexing, invoiceReconciliation: "pending", accountingStatus: campaign.operations ? evalAccountingStatus(campaign.operations) : "unavailable", + readinessStatus: campaign.operations ? evalReadinessStatus(campaign.operations) : "unavailable", feedbackReviewMethod: "Only exact synthetic fixture bodies with all corroborating graders are automatically dismissed; all other feedback requires review" }, null, 2)}\n`, { mode: 0o600 }); for (const finding of findings) console.error(`::error title=Eval operations::${finding}`); return findings;