From 2e7efc625becdc12b2c58169330b387f6885d262 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 08:29:29 -0500 Subject: [PATCH] test: preserve reports while rejecting missing or duplicate proof Co-Authored-By: Paperclip --- tests/runner-e2e/README.md | 5 ++- .../dependable-qualification.test.ts | 45 ++++++++++++++++++- tests/runner-e2e/dependable-qualification.ts | 11 +++-- tests/runner-e2e/report.ts | 21 ++++++++- 4 files changed, 74 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 3a28542ac68..7d964caede1 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -16,7 +16,10 @@ Set `PAPERCLIP_DEPENDABLE_QUALIFICATION_MANIFEST` to a private JSON file matchin writes `dependable-qualification.json` beside the existing report and makes an incomplete checkpoint fail that report command. The checkpoint reads the entire retained attempt pool before best-attempt selection. Failed attempts and missing -observer receipts remain visible after later successes. The ordinary report's +observer receipts remain visible after later successes. Duplicate receipts make +the checkpoint incomplete while preserving report output. Qualification also +requires every manifest-listed proof file to exist inside its attempt directory; +missing files, symlinks, and unsafe paths cannot count as inspected evidence. The ordinary report's costs, timings, evidence scanning, and attempt records remain authoritative. The observer receipt is typed and validated as `RunnerE2EResult.dependableWork`; diff --git a/tests/runner-e2e/dependable-qualification.test.ts b/tests/runner-e2e/dependable-qualification.test.ts index b756b400b7d..079785b1d07 100644 --- a/tests/runner-e2e/dependable-qualification.test.ts +++ b/tests/runner-e2e/dependable-qualification.test.ts @@ -1,6 +1,6 @@ import { expect, it } from "vitest"; import { execFile } from "node:child_process"; -import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { mkdtemp, mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; @@ -85,7 +85,10 @@ it.each([ it("rejects duplicate receipts instead of counting one run twice", () => { const row = attempt(DEPENDABLE_CELLS[0]!, 1); - expect(() => gradeDependableQualification(config, [row, structuredClone(row)])).toThrow("Duplicate"); + const report = gradeDependableQualification(config, [row, structuredClone(row)]); + expect(report.complete).toBe(false); + expect(report.observations).toHaveLength(2); + expect(report.observations.every(value => !value.qualified && value.problems.includes("duplicate-attempt-receipt"))).toBe(true); }); it("does not count an existing task or run as a fresh attempt under a new receipt", () => { @@ -180,3 +183,41 @@ it("keeps a complete rotation for valid UTC dates before the Unix epoch", () => expect(dates.every(date => dependableDailySelection(date).length === 4)).toBe(true); expect([...new Set(dates.flatMap(dependableDailySelection))].sort()).toEqual([...DEPENDABLE_CELLS].sort()); }); + +it.each(["missing", "duplicate", "linked", "unsafe"])("writes an incomplete report for %s retained proof", async defect => { + const root = await mkdtemp(path.join(os.tmpdir(), "dependable-retained-proof-")); + try { + const rows = complete(); + if (defect === "duplicate") rows.push(structuredClone(rows[0]!)); + for (const [index, row] of rows.entries()) { + const directory = path.join(root, `campaign-${index}`, "attempt-1"); await mkdir(directory, { recursive: true }); + const { manifestVerified: _manifest, secretScanPassed: _scan, ...evidence } = row.evidence; + await writeFile(path.join(directory, "result.json"), JSON.stringify({ ...row.result, + dependableWork: { campaignId: row.campaignId, cell: row.cell, phase: row.phase, evidence } })); + const files = defect === "unsafe" && index === 0 ? ["final-state.png", "../outside.png"] : ["final-state.png"]; + await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files, leaks: [], missing: [] })); + if (defect === "linked" && index === 0) { + const outside = path.join(root, "outside.png"); await writeFile(outside, "external proof"); + await symlink(outside, path.join(directory, "final-state.png")); + } else if (defect !== "missing" || index !== 0) await writeFile(path.join(directory, "final-state.png"), Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+aK1cAAAAASUVORK5CYII=", "base64")); + } + const manifest = path.join(root, "qualification.json"); await writeFile(manifest, JSON.stringify(config)); + const repository = path.resolve(import.meta.dirname, "../.."), output = path.join(root, "report"); + let exitCode = 0; + try { + await promisify(execFile)(process.execPath, [path.join(repository, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repository, "tests/runner-e2e/report.ts")], { + cwd: repository, timeout: 25_000, env: { PATH: process.env.PATH, HOME: process.env.HOME, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, + PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([rows[0]!.result.executionId]), + PAPERCLIP_DEPENDABLE_QUALIFICATION_MANIFEST: manifest }, + }); + } catch (error) { exitCode = (error as { code: number }).code; } + expect(exitCode).toBe(1); + const report = JSON.parse(await readFile(path.join(output, "dependable-qualification.json"), "utf8")); + expect(report.complete).toBe(false); + expect(report.observations).toHaveLength(rows.length); + const original = report.observations.find((value: Record) => value.campaignId === rows[0]!.campaignId); + expect(original?.problems).toContain(defect === "duplicate" ? "duplicate-attempt-receipt" : "evidence-unverified"); + expect(await readFile(path.join(output, "summary.md"), "utf8")).toContain("checkpoint: incomplete"); + expect(await readFile(path.join(output, "index.html"), "utf8")).toContain("html"); + } finally { await rm(root, { recursive: true, force: true }); } +}); diff --git a/tests/runner-e2e/dependable-qualification.ts b/tests/runner-e2e/dependable-qualification.ts index 00eb2fcc597..ed915212de8 100644 --- a/tests/runner-e2e/dependable-qualification.ts +++ b/tests/runner-e2e/dependable-qualification.ts @@ -102,18 +102,20 @@ export function gradeDependableQualification(config: DependableQualification, at !config.models.codie.trim() || !config.models.claude.trim() || !digest.test(config.settingsHashes.codie) || !digest.test(config.settingsHashes.claude) || !/^sha256:[a-f0-9]{64}$/.test(config.runnerArtifact)) throw new Error("Incomplete dependable-work qualification identity"); - const receipts = new Set(); + const receipts = new Map(); const issues = new Map(); const runs = new Map(); for (const attempt of attempts) { const key = `${attempt.campaignId}:${attempt.result.executionId}:${attempt.result.attempt}`; - if (!attempt.campaignId || receipts.has(key)) throw new Error("Duplicate or missing attempt receipt"); - receipts.add(key); + receipts.set(key, (receipts.get(key) ?? 0) + 1); if (attempt.result.issueId) issues.set(attempt.result.issueId, (issues.get(attempt.result.issueId) ?? 0) + 1); for (const run of attempt.result.runIds ?? []) runs.set(run, (runs.get(run) ?? 0) + 1); } const observations = attempts.map(attempt => { const problems = attemptProblems(attempt, config); + if (!attempt.campaignId) problems.push("missing-attempt-receipt"); + if (receipts.get(`${attempt.campaignId}:${attempt.result.executionId}:${attempt.result.attempt}`)! > 1) + problems.push("duplicate-attempt-receipt"); if ((attempt.result.issueId && issues.get(attempt.result.issueId)! > 1) || attempt.result.runIds?.some(id => runs.get(id)! > 1)) problems.push("reused-task-or-run"); return { campaignId: attempt.campaignId, executionId: attempt.result.executionId, attempt: attempt.result.attempt, @@ -130,7 +132,8 @@ export function gradeDependableQualification(config: DependableQualification, at complete: checkpointAt !== null && staging.length > 0 }; }); return { schema: "paperclip.e2e.dependable-qualification.v1", revision: config.revision, - requiredCells: 24, requiredCandidatePasses: 72, complete: cells.every(cell => cell.complete), cells, + requiredCells: 24, requiredCandidatePasses: 72, complete: cells.every(cell => cell.complete) && !observations.some(row => + row.problems.includes("duplicate-attempt-receipt") || row.problems.includes("missing-attempt-receipt")), cells, observations, failedAttempts: observations.filter(row => row.originalStatus === "failed").length, incompleteAttempts: observations.filter(row => !row.qualified).length }; } diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index bea5b0d0ded..81103993b2a 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -4,6 +4,8 @@ import path from "node:path"; import { copyFile, mkdir, + lstat, + realpath, readFile, readdir, writeFile, @@ -67,7 +69,7 @@ function validate(result: RunnerE2EResult, evidence: EvidenceManifest | null) { errors.push(`missing evidence=${evidence.missing.join(",")}`); if ( result.status === "passed" && - !evidence?.files?.includes("final-state.png") + (!Array.isArray(evidence?.files) || !evidence.files.includes("final-state.png")) ) { errors.push("passing final-state screenshot missing"); } @@ -82,6 +84,22 @@ function safeEvidenceRelative(relative: string) { return segments; } +async function retainedProofAvailable(directory: string, evidence: EvidenceManifest | null): Promise { + if (!Array.isArray(evidence?.files) || evidence.files.length === 0) return false; + const root = await realpath(directory); + for (const file of evidence.files) { + const segments = typeof file === "string" ? safeEvidenceRelative(file) : null; + if (!segments) return false; + const source = path.join(root, ...segments); + const valid = await lstat(source).then(async stat => { + const resolved = await realpath(source); + return stat.isFile() && !stat.isSymbolicLink() && resolved.startsWith(`${root}${path.sep}`); + }).catch(() => false); + if (!valid) return false; + } + return true; +} + async function stageDashboardEvidence( selected: readonly AggregatedResult[], output: string, @@ -196,6 +214,7 @@ async function main() { .then((value) => JSON.parse(value) as EvidenceManifest) .catch(() => null); const errors = validate(result, evidence); + if (result.dependableWork && !await retainedProofAvailable(directory, evidence)) errors.push("Retained qualification proof is missing or invalid"); const entry = { result, evidence,