mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-21 00:05:49 +02:00
fix(runner): retain hashed declared screenshots
Playwright stores screenshot attachments with a content hash, while normalized runner results declare stable screenshot basenames. Materialize declared names from manifest-listed hashed attachments so history publication can verify and retain warm-turn screenshots. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
5dfce25497
commit
86d1992ac4
2 files changed
+113
No files matched your search
@@ -403,6 +403,85 @@ describe("runner E2E report aggregation", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("materializes declared screenshots from hashed Playwright attachments", async () => {
|
||||
const root = await mkdtemp(
|
||||
path.join(os.tmpdir(), "runner-e2e-report-screenshot-alias-")
|
||||
);
|
||||
cleanupDirectories.push(root);
|
||||
const executionId = "daytona-warm-continuity.legacy-codex.daytona.warm-three-turn";
|
||||
const directory = path.join(root, "attempt-1");
|
||||
const attachment =
|
||||
"playwright-output/warm-turn/attachments/warm-turn-1-deadbeef.png";
|
||||
await mkdir(path.join(directory, path.dirname(attachment)), {
|
||||
recursive: true,
|
||||
});
|
||||
await writeFile(path.join(directory, "final-state.png"), "final-png");
|
||||
await writeFile(path.join(directory, attachment), "warm-turn-png");
|
||||
await writeFile(
|
||||
path.join(directory, "result.json"),
|
||||
JSON.stringify({
|
||||
schema: "paperclip.runner-e2e.result/v1",
|
||||
executionId,
|
||||
attempt: 1,
|
||||
status: "passed",
|
||||
profileId: "legacy-codex",
|
||||
environmentId: "daytona",
|
||||
caseId: "warm-three-turn",
|
||||
provider: "codex",
|
||||
model: "fixture-model",
|
||||
runtimeMode: "legacy",
|
||||
startedAt: "2026-08-26T00:00:00.000Z",
|
||||
finishedAt: "2026-08-26T00:00:01.000Z",
|
||||
durationMs: 1_000,
|
||||
cleanup: "passed",
|
||||
screenshots: [
|
||||
{
|
||||
id: "warm-turn-1",
|
||||
label: "Warm Daytona turn 1 awaiting review",
|
||||
file: "warm-turn-1.png",
|
||||
},
|
||||
{
|
||||
id: "final-state",
|
||||
label: "Final visible task state",
|
||||
file: "final-state.png",
|
||||
},
|
||||
],
|
||||
} satisfies RunnerE2EResult),
|
||||
);
|
||||
await writeFile(
|
||||
path.join(directory, "evidence-manifest.json"),
|
||||
JSON.stringify({
|
||||
files: ["final-state.png", attachment],
|
||||
leaks: [],
|
||||
missing: [],
|
||||
}),
|
||||
);
|
||||
|
||||
const output = path.join(root, "merged");
|
||||
await execFileAsync(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"),
|
||||
path.join(repositoryRoot, "tests/runner-e2e/report.ts"),
|
||||
],
|
||||
{
|
||||
cwd: repositoryRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root,
|
||||
PAPERCLIP_RUNNER_E2E_REPORT_OUT: output,
|
||||
PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]),
|
||||
},
|
||||
},
|
||||
);
|
||||
expect(
|
||||
await readFile(
|
||||
path.join(output, "evidence", executionId, "attempt-1", "warm-turn-1.png"),
|
||||
"utf8",
|
||||
),
|
||||
).toBe("warm-turn-png");
|
||||
});
|
||||
|
||||
it("constructs the public root JUnit from fixed markup and escaped fields", async () => {
|
||||
const root = await mkdtemp(
|
||||
path.join(os.tmpdir(), "runner-e2e-report-junit-test-"),
|
||||
|
||||
@@ -105,6 +105,40 @@ async function stageDashboardEvidence(
|
||||
.catch(() => false);
|
||||
if (didCopy) copied.push(segments.join("/"));
|
||||
}
|
||||
// Playwright renames attachment files with a content hash, while runner
|
||||
// results retain the stable screenshot basename used by the dashboard and
|
||||
// history publisher. Materialize each declared screenshot under that
|
||||
// basename when its hashed attachment is present in the evidence manifest.
|
||||
// The source is still restricted to manifest-listed files, so this cannot
|
||||
// expand the evidence set beyond what the test recorded.
|
||||
for (const screenshot of entry.result.screenshots ?? []) {
|
||||
if (copied.includes(screenshot.file)) continue;
|
||||
const stem = screenshot.file.replace(/\.png$/i, "");
|
||||
const escapedStem = stem.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
const hashedAttachment = new RegExp(
|
||||
`^${escapedStem}-[0-9a-f]{8,128}\\.png$`,
|
||||
"i",
|
||||
);
|
||||
const candidates = (entry.evidence?.files ?? []).filter((relative) => {
|
||||
const basename = path.posix.basename(relative);
|
||||
return (
|
||||
basename === screenshot.file ||
|
||||
hashedAttachment.test(basename)
|
||||
);
|
||||
});
|
||||
if (candidates.length !== 1) continue;
|
||||
const segments = safeEvidenceRelative(candidates[0]!);
|
||||
if (!segments) continue;
|
||||
const source = path.join(entry.directory, ...segments);
|
||||
const destination = path.join(output, ...baseSegments, screenshot.file);
|
||||
const didCopy = await mkdir(path.dirname(destination), {
|
||||
recursive: true,
|
||||
})
|
||||
.then(() => copyFile(source, destination))
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (didCopy) copied.push(screenshot.file);
|
||||
}
|
||||
staged.set(entry.result.executionId, {
|
||||
baseHref: baseSegments.join("/"),
|
||||
files: copied,
|
||||
|
||||
Reference in new issue
Block a user