fix: enforce accounting coverage and recovery regression gates

Co-Authored-By: Paperclip <noreply@paperclip.ing>

scripts/__tests__/release-verify-workflow.test.mjs

@@ -456,3 +456,40 @@ test("direct protocol concurrency override only lowers the configured ceiling",
     if (expected !== null) assert.equal(result.stdout, expected);
   }
 });
+
+for (const workflow of ["release.yml", "e2e.yml", "storybook-visual.yml", "runner-live-evals.yml"]) {
+  test(`${workflow} keeps its frozen dependency install and fails closed on stale locks`, () => {
+    const text = readWorkflow(workflow);
+    const job = workflow === "release.yml" ? text.split("  smoke_canary_onboarding:\n")[1].split("\n  # ----- Nightly lane")[0] : text;
+    const install = job.match(/(?:- name: Install (?:test )?dependencies\n\s+run: ([^\n]+)|- run: (pnpm install[^\n]*))/);
+    assert.ok(install, "source workflow must expose a dependency install step");
+    const script = install[1] ?? install[2];
+    for (const exitCode of [0, 17]) {
+      const run = spawnSync("bash", ["-euo", "pipefail", "-c", `
+        pnpm() { printf '%s\n' "$*"; return "$INSTALL_EXIT"; }
+        ${script}
+      `], { encoding: "utf8", env: { ...process.env, INSTALL_EXIT: String(exitCode) } });
+      assert.equal(run.status, exitCode, run.stderr);
+      assert.equal(run.stdout.trim(), "install --frozen-lockfile", "never resolve new packages or retry after a frozen install failure");
+    }
+  });
+}
+
+test("the accounting CI lane installs only the reviewed lockfile and gates coverage on that install", () => {
+  const job = readWorkflow("pr-trusted.yml").split("  general_tests:\n")[1].split("  docker_context_integrity:\n")[0];
+  const install = job.split("      - name: Install dependencies\n")[1].split("      - name: Run grouped general test suites\n")[0];
+  const script = install.split("        run: |\n")[1];
+  assert.ok(script, "accounting lane must have an inspectable install command");
+  for (const exitCode of [0, 17]) {
+    const result = spawnSync("bash", ["-euo", "pipefail", "-c", `
+      pnpm() { printf '%s\\n' "$*"; return "$INSTALL_EXIT"; }
+      ${script}
+    `], { encoding: "utf8", env: { ...process.env, ACCOUNTING_GATE: "true", INSTALL_EXIT: String(exitCode) } });
+    assert.equal(result.status, exitCode, result.stderr);
+    assert.equal(result.stdout.trim(), "install --frozen-lockfile", "a stale lock must not resolve executable coverage code inline");
+  }
+  assert.match(install, /id: install_dependencies/);
+  assert.match(install, /ACCOUNTING_GATE: \$\{\{ matrix\.group == 'general-server-without-chat' && matrix\.shard_index == 0 \}\}/);
+  const coverage = job.split("      - name: Verify accounting coverage and crash recovery\n")[1].split("      - name: Upload accounting coverage\n")[0];
+  assert.match(coverage, /steps\.install_dependencies\.outcome == 'success'/, "failed installs must never load the coverage provider");
+});

scripts/test-accounting-mutations.mjs

@@ -0,0 +1,57 @@
+import { spawn } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile, mkdir, copyFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+const root = fileURLToPath(new URL("..", import.meta.url));
+const directory = await mkdtemp(path.join(tmpdir(), "accounting-mutations-"));
+const artifactDirectory = path.join(root, "coverage/accounting/mutations");
+const targets = { deduplication: "deduplicates a retried receipt", company: "isolates company reporting", projection: "conserves agent projections", threshold: "stops at the exact budget boundary" };
+const evidence = [];
+const persist = () => writeFile(path.join(root, "coverage/accounting/mutations.json"), JSON.stringify({ generatedAt: new Date().toISOString(), evidence }, null, 2));
+await rm(artifactDirectory, { recursive: true, force: true });
+await mkdir(artifactDirectory, { recursive: true });
+try {
+  for (const name of ["baseline", ...Object.keys(targets)]) {
+    const report = path.join(directory, `${name}.json`);
+    const entry = { mutation: name, result: "error", assertions: [] };
+    evidence.push(entry);
+    await persist();
+    const result = await new Promise((resolve) => {
+      const child = spawn(process.execPath, ["node_modules/vitest/vitest.mjs", "run", "--config", "server/vitest.accounting-mutations.config.ts", "--reporter=json", `--outputFile=${report}`], {
+        cwd: root, env: { ...process.env, PAPERCLIP_ACCOUNTING_MUTATION: name === "baseline" ? "" : name }, stdio: ["ignore", "pipe", "pipe"],
+      });
+      let log = ""; const capture = data => { log += data; };
+      child.stdout.on("data", capture); child.stderr.on("data", capture);
+      child.once("error", error => resolve({ code: null, log: `${log}\n${error.message}` }));
+      child.once("close", code => resolve({ code, log }));
+    });
+    await writeFile(path.join(artifactDirectory, `${name}.log`), result.log);
+    await copyFile(report, path.join(artifactDirectory, `${name}.json`)).catch(() => {});
+    const json = JSON.parse(await readFile(report, "utf8").catch(() => { throw new Error(`${name} produced no test evidence: ${result.log}`); }));
+    const assertions = json.testResults.flatMap(file => file.assertionResults);
+    entry.assertions = assertions.map(({ title, status, failureMessages }) => ({ title, status, failureMessages }));
+    const completeRoster = assertions.length === 4 && Object.values(targets).every(title => assertions.filter(test => test.title === title).length === 1);
+    if (name === "baseline") {
+      entry.result = result.code === 0 && completeRoster && assertions.every(test => test.status === "passed") ? "passed" : "baseline_failed";
+      if (entry.result !== "passed") throw new Error(`Mutation baseline failed or skipped: ${result.log}`);
+    } else {
+      const failures = assertions.filter(test => test.status === "failed");
+      const sentinel = failures[0];
+      // A marked AssertionError proves the intended comparison ran. Setup
+      // errors, timeouts, unrelated assertions, and extra failures are invalid.
+      const intendedFailure = failures.length === 1 && sentinel.title === targets[name]
+        && sentinel.failureMessages?.length === 1
+        && sentinel.failureMessages[0].startsWith(`AssertionError: ACCOUNTING_ASSERTION ${name}:`);
+      entry.result = result.code === 1 && result.log.split(/\r?\n/).includes(`ACCOUNTING_MUTATION_APPLIED ${name}`)
+        && completeRoster && assertions.every(test => test.status === "passed" || test === sentinel)
+        && intendedFailure ? "killed" : "survived_or_invalid";
+      if (entry.result !== "killed") throw new Error(`Mutation ${name} survived or failed without its sentinel: ${result.log}`);
+    }
+    await persist();
+    process.stdout.write(`${name}: ${entry.result}\n`);
+  }
+} finally {
+  // Failed baselines, surviving mutants, and missing reports are evidence too.
+  try { await persist(); } finally { await rm(directory, { recursive: true, force: true }); }
+}

scripts/test-accounting-mutations.test.mjs

@@ -0,0 +1,49 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, mkdir, copyFile, readFile, writeFile, rm, access } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+
+for (const scenario of ["success", "baseline", "survives", "missing", "database", "timeout", "wrong_assertion", "wrong_sentinel", "missing_marker", "extra_failure", "skipped"]) {
+  test(`mutation gate retains evidence: ${scenario}`, async () => {
+    const root = await mkdtemp(path.join(tmpdir(), "mutation-evidence-test-"));
+    try {
+      await mkdir(path.join(root, "scripts"));
+      await mkdir(path.join(root, "node_modules/vitest"), { recursive: true });
+      await copyFile(path.resolve("scripts/test-accounting-mutations.mjs"), path.join(root, "scripts/test-accounting-mutations.mjs"));
+      await writeFile(path.join(root, "node_modules/vitest/vitest.mjs"), `
+import { writeFileSync } from 'node:fs';
+const name = process.env.PAPERCLIP_ACCOUNTING_MUTATION;
+const scenario = process.env.MUTATION_EVIDENCE_TEST_SCENARIO;
+const titles = { deduplication: 'deduplicates a retried receipt', company: 'isolates company reporting', projection: 'conserves agent projections', threshold: 'stops at the exact budget boundary' };
+console.log(name && scenario !== 'missing_marker' ? 'ACCOUNTING_MUTATION_APPLIED ' + name : 'baseline diagnostics');
+if (name && scenario === 'missing') process.exit(1);
+const fail = name ? scenario !== 'survives' : scenario === 'baseline';
+const failureMessage = scenario === 'database' ? 'Error: connection refused'
+  : scenario === 'timeout' ? 'Error: Test timed out in 10000ms'
+  : scenario === 'wrong_assertion' ? 'AssertionError: expected 1 to be 2'
+  : 'AssertionError: ACCOUNTING_ASSERTION ' + name + ': expected value to match';
+const assertionResults = Object.values(titles).map(title => {
+  const failed = fail && (!name || title === titles[scenario === 'wrong_sentinel' ? 'company' : name] || scenario === 'extra_failure');
+  return { title, status: failed ? 'failed' : name && scenario === 'skipped' ? 'pending' : 'passed', failureMessages: failed ? [failureMessage] : [] };
+});
+writeFileSync(process.argv.find(arg => arg.startsWith('--outputFile=')).slice(13), JSON.stringify({ testResults: [{ assertionResults }] }));
+process.exit(fail ? 1 : 0);
+`);
+      if (scenario === "missing") {
+        await mkdir(path.join(root, "coverage/accounting/mutations"), { recursive: true });
+        await writeFile(path.join(root, "coverage/accounting/mutations/deduplication.json"), '{"stale":true}');
+      }
+      const result = spawnSync(process.execPath, ["scripts/test-accounting-mutations.mjs"], { cwd: root,
+        env: { ...process.env, MUTATION_EVIDENCE_TEST_SCENARIO: scenario }, encoding: "utf8" });
+      assert.equal(result.status, scenario === "success" ? 0 : 1, result.stderr);
+      const report = JSON.parse(await readFile(path.join(root, "coverage/accounting/mutations.json"), "utf8"));
+      assert.equal(report.evidence.at(-1).result, { success: "killed", baseline: "baseline_failed", missing: "error" }[scenario] ?? "survived_or_invalid");
+      const last = report.evidence.at(-1).mutation;
+      assert.ok((await readFile(path.join(root, `coverage/accounting/mutations/${last}.log`), "utf8")).length);
+      if (scenario === "missing") await assert.rejects(access(path.join(root, `coverage/accounting/mutations/${last}.json`)));
+      if (scenario !== "missing") assert.ok(JSON.parse(await readFile(path.join(root, `coverage/accounting/mutations/${last}.json`), "utf8")).testResults.length);
+    } finally { await rm(root, { recursive: true, force: true }); }
+  });
+}

scripts/verify-accounting-test-results.mjs

@@ -0,0 +1,39 @@
+import { readFile } from "node:fs/promises";
+
+const required = [
+  {
+    file: "server/src/__tests__/cost-accounting-crash.test.ts",
+    suite: "accounting survives real server SIGKILL",
+    tests: ["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"]
+      .map(phase => `recovers exactly once after ${phase}`),
+  },
+  {
+    file: "packages/db/src/cost-accounting-migration.test.ts",
+    suite: "cost accounting historical upgrade",
+    tests: [
+      "preserves historical receipts, counters, references, finance currencies and incident history",
+      "rejects invalid historical amounts atomically and can retry after explicit repair",
+    ],
+  },
+];
+
+try {
+  const reports = process.argv.slice(2);
+  if (reports.length !== required.length) throw new Error("Expected server and migration JSON report paths");
+  for (const [index, spec] of required.entries()) {
+    const report = JSON.parse(await readFile(reports[index], "utf8"));
+    if (report.success !== true || !Array.isArray(report.testResults)) throw new Error(`Unsuccessful or invalid report: ${reports[index]}`);
+    const files = report.testResults.filter(result => result.name?.replaceAll("\\", "/").endsWith(`/${spec.file}`));
+    if (files.length !== 1 || !Array.isArray(files[0].assertionResults)) throw new Error(`Missing or duplicate suite: ${spec.file}`);
+    for (const title of spec.tests) {
+      const matches = files[0].assertionResults.filter(result => result.fullName === `${spec.suite} ${title}`);
+      if (matches.length !== 1 || matches[0].status !== "passed") {
+        throw new Error(`Required accounting test did not pass: ${spec.suite} ${title}`);
+      }
+    }
+    process.stdout.write(`Verified ${spec.tests.length} required tests: ${spec.file}\n`);
+  }
+} catch (error) {
+  console.error(`Accounting gate: ${error.message}`);
+  process.exitCode = 1;
+}

scripts/verify-accounting-test-results.test.mjs

@@ -0,0 +1,50 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { mkdtemp, writeFile, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { spawnSync } from "node:child_process";
+
+function reports() {
+  return [
+    { success: true, testResults: [{ name: "/repo/server/src/__tests__/cost-accounting-crash.test.ts", assertionResults:
+      ["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"].map(phase => ({
+        fullName: `accounting survives real server SIGKILL recovers exactly once after ${phase}`, status: "passed",
+      })),
+    }] },
+    { success: true, testResults: [{ name: "/repo/packages/db/src/cost-accounting-migration.test.ts", assertionResults:
+      ["preserves historical receipts, counters, references, finance currencies and incident history",
+        "rejects invalid historical amounts atomically and can retry after explicit repair"].map(title => ({
+        fullName: `cost accounting historical upgrade ${title}`, status: "passed",
+      })),
+    }] },
+  ];
+}
+
+for (const scenario of ["passed", "skipped_crash", "skipped_migration", "missing_case", "missing_suite", "duplicate_case", "duplicate_suite", "wrong_suite", "failed", "malformed", "missing_report"]) {
+  test(`required accounting evidence: ${scenario}`, async () => {
+    const directory = await mkdtemp(path.join(tmpdir(), "accounting-required-tests-"));
+    try {
+      const data = reports();
+      const crash = data[0].testResults[0];
+      const migration = data[1].testResults[0];
+      if (scenario === "skipped_crash") crash.assertionResults[0].status = "pending";
+      if (scenario === "skipped_migration") migration.assertionResults[1].status = "pending";
+      if (scenario === "missing_case") crash.assertionResults.pop();
+      if (scenario === "missing_suite") data[1].testResults = [];
+      if (scenario === "duplicate_case") migration.assertionResults.push(migration.assertionResults[0]);
+      if (scenario === "duplicate_suite") data[0].testResults.push(crash);
+      if (scenario === "wrong_suite") crash.assertionResults[0].fullName = "another test with the same title";
+      if (scenario === "failed") data[0].success = false;
+      const files = data.map((_, index) => path.join(directory, `${index}.json`));
+      for (const [index, report] of data.entries()) {
+        if (scenario === "missing_report" && index === 1) continue;
+        await writeFile(files[index], scenario === "malformed" ? "{" : JSON.stringify(report));
+      }
+      const result = spawnSync(process.execPath, ["scripts/verify-accounting-test-results.mjs", ...files], { encoding: "utf8" });
+      assert.equal(result.status, scenario === "passed" ? 0 : 1, result.stderr);
+      if (scenario === "passed") assert.match(result.stdout, /Verified 4 required tests:[\s\S]*Verified 2 required tests:/);
+      else assert.match(result.stderr, /Accounting gate:/);
+    } finally { await rm(directory, { recursive: true, force: true }); }
+  });
+}