fix(runner): retain hashed declared screenshots

Playwright stores screenshot attachments with a content hash, while normalized runner results declare stable screenshot basenames. Materialize declared names from manifest-listed hashed attachments so history publication can verify and retain warm-turn screenshots.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-18 22:05:48 -05:00
1 parent 5dfce25497
commit 86d1992ac4
2 files changed
+113

No files matched your search

+79
View File
@@ -403,6 +403,85 @@ describe("runner E2E report aggregation", () => {
});
});
it("materializes declared screenshots from hashed Playwright attachments", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-report-screenshot-alias-")
);
cleanupDirectories.push(root);
const executionId = "daytona-warm-continuity.legacy-codex.daytona.warm-three-turn";
const directory = path.join(root, "attempt-1");
const attachment =
"playwright-output/warm-turn/attachments/warm-turn-1-deadbeef.png";
await mkdir(path.join(directory, path.dirname(attachment)), {
recursive: true,
});
await writeFile(path.join(directory, "final-state.png"), "final-png");
await writeFile(path.join(directory, attachment), "warm-turn-png");
await writeFile(
path.join(directory, "result.json"),
JSON.stringify({
schema: "paperclip.runner-e2e.result/v1",
executionId,
attempt: 1,
status: "passed",
profileId: "legacy-codex",
environmentId: "daytona",
caseId: "warm-three-turn",
provider: "codex",
model: "fixture-model",
runtimeMode: "legacy",
startedAt: "2026-08-26T00:00:00.000Z",
finishedAt: "2026-08-26T00:00:01.000Z",
durationMs: 1_000,
cleanup: "passed",
screenshots: [
{
id: "warm-turn-1",
label: "Warm Daytona turn 1 awaiting review",
file: "warm-turn-1.png",
},
{
id: "final-state",
label: "Final visible task state",
file: "final-state.png",
},
],
} satisfies RunnerE2EResult),
);
await writeFile(
path.join(directory, "evidence-manifest.json"),
JSON.stringify({
files: ["final-state.png", attachment],
leaks: [],
missing: [],
}),
);
const output = path.join(root, "merged");
await execFileAsync(
process.execPath,
[
path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"),
path.join(repositoryRoot, "tests/runner-e2e/report.ts"),
],
{
cwd: repositoryRoot,
env: {
...process.env,
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]),
},
},
);
expect(
await readFile(
path.join(output, "evidence", executionId, "attempt-1", "warm-turn-1.png"),
"utf8",
),
).toBe("warm-turn-png");
});
it("constructs the public root JUnit from fixed markup and escaped fields", async () => {
const root = await mkdtemp(
path.join(os.tmpdir(), "runner-e2e-report-junit-test-"),
+34
View File
@@ -105,6 +105,40 @@ async function stageDashboardEvidence(
.catch(() => false);
if (didCopy) copied.push(segments.join("/"));
}
// Playwright renames attachment files with a content hash, while runner
// results retain the stable screenshot basename used by the dashboard and
// history publisher. Materialize each declared screenshot under that
// basename when its hashed attachment is present in the evidence manifest.
// The source is still restricted to manifest-listed files, so this cannot
// expand the evidence set beyond what the test recorded.
for (const screenshot of entry.result.screenshots ?? []) {
if (copied.includes(screenshot.file)) continue;
const stem = screenshot.file.replace(/\.png$/i, "");
const escapedStem = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
const hashedAttachment = new RegExp(
`^${escapedStem}-[0-9a-f]{8,128}\\.png$`,
"i",
);
const candidates = (entry.evidence?.files ?? []).filter((relative) => {
const basename = path.posix.basename(relative);
return (
basename === screenshot.file ||
hashedAttachment.test(basename)
);
});
if (candidates.length !== 1) continue;
const segments = safeEvidenceRelative(candidates[0]!);
if (!segments) continue;
const source = path.join(entry.directory, ...segments);
const destination = path.join(output, ...baseSegments, screenshot.file);
const didCopy = await mkdir(path.dirname(destination), {
recursive: true,
})
.then(() => copyFile(source, destination))
.then(() => true)
.catch(() => false);
if (didCopy) copied.push(screenshot.file);
}
staged.set(entry.result.executionId, {
baseHref: baseSegments.join("/"),
files: copied,