test: preserve reports while rejecting missing or duplicate proof

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -16,7 +16,10 @@ Set `PAPERCLIP_DEPENDABLE_QUALIFICATION_MANIFEST` to a private JSON file matchin
 writes `dependable-qualification.json` beside the existing report and makes an
 incomplete checkpoint fail that report command. The checkpoint reads the entire
 retained attempt pool before best-attempt selection. Failed attempts and missing
-observer receipts remain visible after later successes. The ordinary report's
+observer receipts remain visible after later successes. Duplicate receipts make
+the checkpoint incomplete while preserving report output. Qualification also
+requires every manifest-listed proof file to exist inside its attempt directory;
+missing files, symlinks, and unsafe paths cannot count as inspected evidence. The ordinary report's
 costs, timings, evidence scanning, and attempt records remain authoritative.
 
 The observer receipt is typed and validated as `RunnerE2EResult.dependableWork`;

tests/runner-e2e/dependable-qualification.test.ts

@@ -1,6 +1,6 @@
 import { expect, it } from "vitest";
 import { execFile } from "node:child_process";
-import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { promisify } from "node:util";
@@ -85,7 +85,10 @@ it.each([
 
 it("rejects duplicate receipts instead of counting one run twice", () => {
   const row = attempt(DEPENDABLE_CELLS[0]!, 1);
-  expect(() => gradeDependableQualification(config, [row, structuredClone(row)])).toThrow("Duplicate");
+  const report = gradeDependableQualification(config, [row, structuredClone(row)]);
+  expect(report.complete).toBe(false);
+  expect(report.observations).toHaveLength(2);
+  expect(report.observations.every(value => !value.qualified && value.problems.includes("duplicate-attempt-receipt"))).toBe(true);
 });
 
 it("does not count an existing task or run as a fresh attempt under a new receipt", () => {
@@ -180,3 +183,41 @@ it("keeps a complete rotation for valid UTC dates before the Unix epoch", () =>
   expect(dates.every(date => dependableDailySelection(date).length === 4)).toBe(true);
   expect([...new Set(dates.flatMap(dependableDailySelection))].sort()).toEqual([...DEPENDABLE_CELLS].sort());
 });
+
+it.each(["missing", "duplicate", "linked", "unsafe"])("writes an incomplete report for %s retained proof", async defect => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "dependable-retained-proof-"));
+  try {
+    const rows = complete();
+    if (defect === "duplicate") rows.push(structuredClone(rows[0]!));
+    for (const [index, row] of rows.entries()) {
+      const directory = path.join(root, `campaign-${index}`, "attempt-1"); await mkdir(directory, { recursive: true });
+      const { manifestVerified: _manifest, secretScanPassed: _scan, ...evidence } = row.evidence;
+      await writeFile(path.join(directory, "result.json"), JSON.stringify({ ...row.result,
+        dependableWork: { campaignId: row.campaignId, cell: row.cell, phase: row.phase, evidence } }));
+      const files = defect === "unsafe" && index === 0 ? ["final-state.png", "../outside.png"] : ["final-state.png"];
+      await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files, leaks: [], missing: [] }));
+      if (defect === "linked" && index === 0) {
+        const outside = path.join(root, "outside.png"); await writeFile(outside, "external proof");
+        await symlink(outside, path.join(directory, "final-state.png"));
+      } else if (defect !== "missing" || index !== 0) await writeFile(path.join(directory, "final-state.png"), Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aK1cAAAAASUVORK5CYII=", "base64"));
+    }
+    const manifest = path.join(root, "qualification.json"); await writeFile(manifest, JSON.stringify(config));
+    const repository = path.resolve(import.meta.dirname, "../.."), output = path.join(root, "report");
+    let exitCode = 0;
+    try {
+      await promisify(execFile)(process.execPath, [path.join(repository, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repository, "tests/runner-e2e/report.ts")], {
+        cwd: repository, timeout: 25_000, env: { PATH: process.env.PATH, HOME: process.env.HOME, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
+          PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([rows[0]!.result.executionId]),
+          PAPERCLIP_DEPENDABLE_QUALIFICATION_MANIFEST: manifest },
+      });
+    } catch (error) { exitCode = (error as { code: number }).code; }
+    expect(exitCode).toBe(1);
+    const report = JSON.parse(await readFile(path.join(output, "dependable-qualification.json"), "utf8"));
+    expect(report.complete).toBe(false);
+    expect(report.observations).toHaveLength(rows.length);
+    const original = report.observations.find((value: Record<string, any>) => value.campaignId === rows[0]!.campaignId);
+    expect(original?.problems).toContain(defect === "duplicate" ? "duplicate-attempt-receipt" : "evidence-unverified");
+    expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("checkpoint: incomplete");
+    expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("html");
+  } finally { await rm(root, { recursive: true, force: true }); }
+});

tests/runner-e2e/dependable-qualification.ts

@@ -102,18 +102,20 @@ export function gradeDependableQualification(config: DependableQualification, at
     !config.models.codie.trim() || !config.models.claude.trim() ||
     !digest.test(config.settingsHashes.codie) || !digest.test(config.settingsHashes.claude) ||
     !/^sha256:[a-f0-9]{64}$/.test(config.runnerArtifact)) throw new Error("Incomplete dependable-work qualification identity");
-  const receipts = new Set<string>();
+  const receipts = new Map<string, number>();
   const issues = new Map<string, number>();
   const runs = new Map<string, number>();
   for (const attempt of attempts) {
     const key = `${attempt.campaignId}:${attempt.result.executionId}:${attempt.result.attempt}`;
-    if (!attempt.campaignId || receipts.has(key)) throw new Error("Duplicate or missing attempt receipt");
-    receipts.add(key);
+    receipts.set(key, (receipts.get(key) ?? 0) + 1);
     if (attempt.result.issueId) issues.set(attempt.result.issueId, (issues.get(attempt.result.issueId) ?? 0) + 1);
     for (const run of attempt.result.runIds ?? []) runs.set(run, (runs.get(run) ?? 0) + 1);
   }
   const observations = attempts.map(attempt => {
     const problems = attemptProblems(attempt, config);
+    if (!attempt.campaignId) problems.push("missing-attempt-receipt");
+    if (receipts.get(`${attempt.campaignId}:${attempt.result.executionId}:${attempt.result.attempt}`)! > 1)
+      problems.push("duplicate-attempt-receipt");
     if ((attempt.result.issueId && issues.get(attempt.result.issueId)! > 1) || attempt.result.runIds?.some(id => runs.get(id)! > 1))
       problems.push("reused-task-or-run");
     return { campaignId: attempt.campaignId, executionId: attempt.result.executionId, attempt: attempt.result.attempt,
@@ -130,7 +132,8 @@ export function gradeDependableQualification(config: DependableQualification, at
       complete: checkpointAt !== null && staging.length > 0 };
   });
   return { schema: "paperclip.e2e.dependable-qualification.v1", revision: config.revision,
-    requiredCells: 24, requiredCandidatePasses: 72, complete: cells.every(cell => cell.complete), cells,
+    requiredCells: 24, requiredCandidatePasses: 72, complete: cells.every(cell => cell.complete) && !observations.some(row =>
+      row.problems.includes("duplicate-attempt-receipt") || row.problems.includes("missing-attempt-receipt")), cells,
     observations, failedAttempts: observations.filter(row => row.originalStatus === "failed").length,
     incompleteAttempts: observations.filter(row => !row.qualified).length };
 }

tests/runner-e2e/report.ts

@@ -4,6 +4,8 @@ import path from "node:path";
 import {
   copyFile,
   mkdir,
+  lstat,
+  realpath,
   readFile,
   readdir,
   writeFile,
@@ -67,7 +69,7 @@ function validate(result: RunnerE2EResult, evidence: EvidenceManifest | null) {
     errors.push(`missing evidence=${evidence.missing.join(",")}`);
   if (
     result.status === "passed" &&
-    !evidence?.files?.includes("final-state.png")
+    (!Array.isArray(evidence?.files) || !evidence.files.includes("final-state.png"))
   ) {
     errors.push("passing final-state screenshot missing");
   }
@@ -82,6 +84,22 @@ function safeEvidenceRelative(relative: string) {
   return segments;
 }
 
+async function retainedProofAvailable(directory: string, evidence: EvidenceManifest | null): Promise<boolean> {
+  if (!Array.isArray(evidence?.files) || evidence.files.length === 0) return false;
+  const root = await realpath(directory);
+  for (const file of evidence.files) {
+    const segments = typeof file === "string" ? safeEvidenceRelative(file) : null;
+    if (!segments) return false;
+    const source = path.join(root, ...segments);
+    const valid = await lstat(source).then(async stat => {
+      const resolved = await realpath(source);
+      return stat.isFile() && !stat.isSymbolicLink() && resolved.startsWith(`${root}${path.sep}`);
+    }).catch(() => false);
+    if (!valid) return false;
+  }
+  return true;
+}
+
 async function stageDashboardEvidence(
   selected: readonly AggregatedResult[],
   output: string,
@@ -196,6 +214,7 @@ async function main() {
       .then((value) => JSON.parse(value) as EvidenceManifest)
       .catch(() => null);
     const errors = validate(result, evidence);
+    if (result.dependableWork && !await retainedProofAvailable(directory, evidence)) errors.push("Retained qualification proof is missing or invalid");
     const entry = {
       result,
       evidence,