diff --git a/tests/runner-e2e/generated-report-carriers.test.ts b/tests/runner-e2e/generated-report-carriers.test.ts new file mode 100644 index 00000000000..080bb08cee2 --- /dev/null +++ b/tests/runner-e2e/generated-report-carriers.test.ts @@ -0,0 +1,50 @@ +import { afterEach, expect, it } from "vitest"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { assertGeneratedReportCarriers } from "./manual-private-transfer.js"; +import { sealManualResultEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { canonicalGeneratedReportAsset } from "./manual-private-report-assets.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL } from "./manual-private-retention.js"; +import { privateRefusalCode } from "./private-refusal-code.js"; + +const roots: string[] = []; +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +async function report(files: Record) { + const root = await mkdtemp(path.join(os.tmpdir(), "generated-report-carriers-")); roots.push(root); + const source = path.join(root, "runner-e2e-merged-report"); + for (const [name, data] of Object.entries({ "html/index.html": "", "normalized/summary.md": "summary", ...files })) { + await mkdir(path.dirname(path.join(source, name)), { recursive: true }); + await writeFile(path.join(source, name), data); + } + return { root, source }; +} +const cell = "normalized/evidence/chat-features.runner-opencode.local.case/attempt-1/html-report/trace"; + +it("accepts standard carriers and authenticated trace viewer assets", async () => { + const logo = await readFile(canonicalGeneratedReportAsset("html/trace/playwright-logo.svg")!); + const { source } = await report({ "html/trace/playwright-logo.svg": logo, "html/data/0123456789abcdef0123456789abcdef01234567.zip": "zip" }); + await expect(assertGeneratedReportCarriers(source)).resolves.toBeUndefined(); +}); + +it.each([ + [`${cell}/codicon.DCmgc-ay.ttf`, "font"], + [`${cell}/playwright-logo.svg`, "svg"], + [`${cell}/manifest.webmanifest`, "webmanifest"], + ["html/data/0123456789abcdef0123456789abcdef01234567.jpeg", "image"], + ["normalized/report.js.map", "source-map"], + ["normalized/LICENSE", "no-extension"], + ["normalized/archive.tar", "other"], + ["html/trace/index.js.map", "trace-viewer-asset"], + ["html/trace/playwright-logo.svg", "trace-viewer-bytes"], +])("names a refused %s carrier by its closed category, never its path", async (name, category) => { + const { root, source } = await report({ [name]: "synthetic carrier" }); + const refusal = await assertGeneratedReportCarriers(source).then(() => undefined, (error: unknown) => error); + const code = privateRefusalCode(refusal); + expect(code).toBe(`unsupported-generated-report-carrier-${category}`); + expect(code).not.toContain("chat-features"); + // The authoritative seal refuses the same tree, so the preflight never hides an admissible report. + const context = { root, private: true, protocol: MANUAL_PRIVATE_PROTOCOL, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", + attempt: "1", executionIds: [MANUAL_PRIVATE_EXECUTIONS[0]!] } as ManualPrivateHandoff; + await expect(sealManualResultEvidence(context, [], source, "generated-report")).rejects.toThrow("Unsupported packaged evidence carrier"); +}); diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 471982ed396..6f5f7d2ffef 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -40,6 +40,7 @@ import { privateCellEvidenceOwner } from "./manual-private-report-assets.js"; import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; import { requiresPrivateManualEvidence } from "./manual-private-retention.js"; import { copySharedEvidence } from "./shared-evidence.js"; +import { dropDuplicatePlaywrightOutput } from "./private-report-copies.js"; import { classifyFailure } from "./failure-classifier.js"; import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js"; import { effectiveAutomaticRetryLimit, executeWithAutomaticRetry } from "./automatic-retry.js"; @@ -826,6 +827,10 @@ async function runAttempt(input: { `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`, "utf8", ); + // html-report/data already holds identical bytes of each failure trace + // and video; drop the playwright-output copies so failures fit the + // private evidence allowance with their HTML report still viewable. + await dropDuplicatePlaywrightOutput(casePrivateDir); let evidence = await packageEvidence({ privateDir: casePrivateDir, uploadDir, diff --git a/tests/runner-e2e/manual-private-cell-report-assets.test.ts b/tests/runner-e2e/manual-private-cell-report-assets.test.ts index 52bb2afb8a4..04e178d41d8 100644 --- a/tests/runner-e2e/manual-private-cell-report-assets.test.ts +++ b/tests/runner-e2e/manual-private-cell-report-assets.test.ts @@ -19,6 +19,7 @@ vi.mock("node:fs/promises", async original => { }; }); import { copySharedEvidence } from "./shared-evidence.js"; +import { dropDuplicatePlaywrightOutput } from "./private-report-copies.js"; import { packageEvidence, publishRunnerE2EAttemptEvidence, isStandardEvidencePath } from "./evidence.js"; import { readManualPrivateHandoff, sealStandardOriginalEvidence, sealManualResultEvidence, preserveOriginalFrameworkResult, type ManualPrivateHandoff } from "./manual-private-handoff.js"; import { canonicalGeneratedReportAsset, cellTraceAsset, inspectCellReportProducer, privateCellEvidenceOwner, readPrivateCellTraceAsset, type CellReportProducerIdentity } from "./manual-private-report-assets.js"; @@ -273,6 +274,14 @@ it.each([0, 1])("actual launcher packaging call %s retains the verified private for (const name of ["playwright-logo.svg", "manifest.webmanifest", "index.html"]) expect(packaged.files).toContain(`html-report/trace/${name}`); }); +async function walkFiles(root: string, relative = ""): Promise { + const found: string[] = []; + for (const item of await readdir(path.join(root, relative), { withFileTypes: true })) { + const name = relative ? `${relative}/${item.name}` : item.name; + if (item.isDirectory()) found.push(...await walkFiles(root, name)); else found.push(name); + } + return found; +} async function actualLauncherEvidenceBranch(packageCall = packageEvidence) { const source = await readFile(new URL("./launch.ts", import.meta.url), "utf8"); const start = source.indexOf(" if (manualPrivateHandoff) {\n await preserveOriginalFrameworkResult(casePrivateDir);"); @@ -280,8 +289,8 @@ async function actualLauncherEvidenceBranch(packageCall = packageEvidence) { expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start); const code = stripTypeScriptTypes(`async function actualBranch(manualPrivateHandoff, candidate, casePrivateDir, resultsRoot, uploadDir, credentials, result, resultPath) { ${source.slice(start, end)} return result; }`, { mode: "strip" }); const publisher = vi.fn(publishRunnerE2EAttemptEvidence); - const branch = Function("preserveOriginalFrameworkResult", "writeFile", "sanitizeJson", "packageEvidence", "privateCellEvidenceOwner", "sealStandardOriginalEvidence", "publishRunnerE2EAttemptEvidence", `${code}; return actualBranch;`)( - preserveOriginalFrameworkResult, writeFile, sanitizeJson, packageCall, privateCellEvidenceOwner, sealStandardOriginalEvidence, publisher, + const branch = Function("preserveOriginalFrameworkResult", "writeFile", "sanitizeJson", "packageEvidence", "privateCellEvidenceOwner", "sealStandardOriginalEvidence", "publishRunnerE2EAttemptEvidence", "dropDuplicatePlaywrightOutput", `${code}; return actualBranch;`)( + preserveOriginalFrameworkResult, writeFile, sanitizeJson, packageCall, privateCellEvidenceOwner, sealStandardOriginalEvidence, publisher, dropDuplicatePlaywrightOutput, ) as (context: ManualPrivateHandoff | undefined, candidate: { id: string }, source: string, root: string, target: string, secrets: string[], result: RunnerE2EResult, resultPath: string) => Promise; return { branch, publisher }; } @@ -300,6 +309,16 @@ it("actual launcher public branch uses its normal publisher without retaining pr await expect(readFile(path.join(uploadDir, "html-report/trace/playwright-logo.svg"))).rejects.toMatchObject({ code: "ENOENT" }); await expect(readFile(path.join(f.cell, "snapshots/manual-original-framework-result.json"))).rejects.toMatchObject({ code: "ENOENT" }); }); +it("actual launcher private branch seals one HTML report copy of the failure trace instead of a second playwright-output copy", async () => { + const f = await fixture(), { branch, publisher } = await actualLauncherEvidenceBranch(); + const traces = async (root: string) => (await walkFiles(root)).filter(name => name.endsWith("trace.zip") || /^html-report\/data\/[0-9a-f]{40}\.zip$/.test(name)); + expect((await traces(f.cell)).length).toBe(2); + const result = await branch(f.context, { id: f.id }, f.cell, path.join(f.root, "results"), f.upload, [], failedLauncherResult(f.id), path.join(f.cell, "result.json")); + expect(result.status).toBe("failed"); expect(publisher).not.toHaveBeenCalled(); + const sealed = await validateSealedOriginals(f.root, f.id, f.context); + const retained = sealed.files.map((row: { path: string }) => row.path).filter((name: string) => name.endsWith("trace.zip") || /^html-report\/data\/[0-9a-f]{40}\.zip$/.test(name)); + expect(retained).toHaveLength(1); expect(retained[0]).toMatch(/^html-report\/data\//); +}); it("actual launcher private package failure preserves original bytes and never falls back to ordinary publication", async () => { const f = await fixture(), original = await readFile(path.join(f.cell, "result.json")); const fail = vi.fn().mockRejectedValue(new Error("synthetic private packaging failure")); diff --git a/tests/runner-e2e/manual-private-transfer.ts b/tests/runner-e2e/manual-private-transfer.ts index 81f77bb9ca4..e6b5e3f6967 100644 --- a/tests/runner-e2e/manual-private-transfer.ts +++ b/tests/runner-e2e/manual-private-transfer.ts @@ -206,6 +206,39 @@ async function verifyDownloads(env: NodeJS.ProcessEnv) { } await writeFile(path.join(root, "manual-report-input-completeness.json"), JSON.stringify({ meaning: "available-files-and-transfer-only", doesNotAttestFeatureOutcome: true, cells }), { mode: 0o600, flag: "wx" }); } +/** + * Names the first carrier the generated-report seal would refuse with a closed + * category, so a refused report is diagnosable without its paths. The seal stays + * authoritative; this only runs before it and never admits anything. + */ +export async function assertGeneratedReportCarriers(source: string) { + let visited = 0; + async function visit(relative: string): Promise { + for (const entry of await readdir(path.join(source, relative), { withFileTypes: true })) { + if (++visited > 4096) return; + const name = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isDirectory() && !entry.isSymbolicLink()) { if (name.split("/").length < 24) await visit(name); continue; } + if (!entry.isFile() || entry.isSymbolicLink()) continue; + const canonical = canonicalGeneratedReportAsset(name); + if (name.startsWith("html/trace/") || canonical !== undefined) { + if (!canonical) throw new Error("Unsupported generated report carrier (trace viewer asset)"); + if (!await isTrustedGeneratedReportAsset(name, await readFile(path.join(source, name)))) + throw new Error("Unsupported generated report carrier (trace viewer bytes)"); + continue; + } + if (isStandardEvidenceFile(`html-report/${name}`)) continue; + const extension = path.extname(name).toLowerCase(); + if ([".ttf", ".otf", ".woff", ".woff2"].includes(extension)) throw new Error("Unsupported generated report carrier (font)"); + if (extension === ".svg") throw new Error("Unsupported generated report carrier (svg)"); + if (extension === ".webmanifest") throw new Error("Unsupported generated report carrier (webmanifest)"); + if ([".jpg", ".jpeg", ".gif", ".webp", ".avif", ".bmp", ".ico"].includes(extension)) throw new Error("Unsupported generated report carrier (image)"); + if (extension === ".map") throw new Error("Unsupported generated report carrier (source map)"); + if (!extension) throw new Error("Unsupported generated report carrier (no extension)"); + throw new Error("Unsupported generated report carrier (other)"); + } + } + await visit(""); +} export async function uploadReport(env: NodeJS.ProcessEnv) { requireSession(env); const { approval, bucket } = request(env), source = await realpath(env.PRIVATE_REPORT_SOURCE ?? ""), root = await realpath(env.PRIVATE_REPORT_ROOT ?? ""); if (!source.startsWith(`${root}${path.sep}`)) throw new Error("Private report escaped owned staging"); @@ -213,6 +246,7 @@ export async function uploadReport(env: NodeJS.ProcessEnv) { if (evidenceCompleteness.meaning !== "available-files-and-transfer-only" || evidenceCompleteness.doesNotAttestFeatureOutcome !== true || evidenceCompleteness.cells.map(cell => cell.executionId).sort().join() !== [...approval.executionIds].sort().join()) throw new Error("Private report lacks exact cell completeness receipts"); const secrets = normalizedSecrets([...CREDENTIAL_NAMES.map(name => env[name]), env.AWS_ACCESS_KEY_ID, env.AWS_SECRET_ACCESS_KEY, env.AWS_SESSION_TOKEN]); + await assertGeneratedReportCarriers(source); await sealManualResultEvidence({ root, protocol: MANUAL_PRIVATE_PROTOCOL, private: true, workflowSha: approval.workflowSha, targetSha: approval.targetSha, runId: approval.runId, attempt: approval.attempt, executionIds: [...approval.executionIds] }, secrets, source, "generated-report"); const sealed = path.join(root, "sealed-results"); diff --git a/tests/runner-e2e/private-report-copies.test.ts b/tests/runner-e2e/private-report-copies.test.ts new file mode 100644 index 00000000000..6ac84a59d59 --- /dev/null +++ b/tests/runner-e2e/private-report-copies.test.ts @@ -0,0 +1,147 @@ +import { afterAll, afterEach, beforeAll, expect, it } from "vitest"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createRequire } from "node:module"; +import { cp, lstat, mkdir, mkdtemp, readdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; +import { copySharedEvidence } from "./shared-evidence.js"; +import { packageEvidence } from "./evidence.js"; +import { sealManualResultEvidence, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { inspectCellReportProducer, privateCellEvidenceOwner, type CellReportProducerIdentity } from "./manual-private-report-assets.js"; +import { stageSealedPrivateCell, validateSealedInventory, validateSealedOriginals } from "./manual-private-transfer.js"; +import { MANUAL_PRIVATE_EXECUTIONS, MANUAL_PRIVATE_PROTOCOL, manualCellKind } from "./manual-private-retention.js"; +import { archivePrivateEvidence } from "./private-history.js"; +import { dropDuplicatePlaywrightOutput, PRIVATE_REPORT_COPIES_RECEIPT } from "./private-report-copies.js"; + +const sha1 = (value: Buffer) => createHash("sha1").update(value).digest("hex"); +const roots: string[] = []; +const temporaryRoot = await realpath(os.tmpdir()); +let reporter: string, producerRoot: string, producer: CellReportProducerIdentity; + +async function files(root: string, relative = ""): Promise { + const found: string[] = []; + for (const item of await readdir(path.join(root, relative), { withFileTypes: true })) { + const name = relative ? `${relative}/${item.name}` : item.name; + if (item.isDirectory()) found.push(...await files(root, name)); else found.push(name); + } + return found.sort(); +} +async function bytesOf(root: string) { + let total = 0; + for (const name of await files(root)) total += (await lstat(path.join(root, name))).size; + return total; +} + +beforeAll(async () => { + producerRoot = await mkdtemp(path.join(temporaryRoot, "synthetic-report-copies-")); + reporter = path.join(producerRoot, "reporter"); const tests = path.join(producerRoot, "node-tests"); + await Promise.all([mkdir(reporter), mkdir(tests)]); + const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); + await writeFile(path.join(tests, "failure.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('synthetic retained failure',async({},info)=>{await info.attach('synthetic-video',{body:Buffer.from('synthetic video bytes'),contentType:'video/webm'});expect('actual wrong outcome').toBe('expected outcome');});`); + const config = path.join(tests, "isolated.config.cjs"); + // The same reporter set as the private chat lane: HTML, JUnit and blob. + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failure.spec.cjs", workers: 1, outputDir: path.join(reporter, "playwright-output"), use: { trace: "retain-on-failure" }, reporter: [["html", { outputFolder: path.join(reporter, "html-report"), open: "never" }], ["junit", { outputFile: path.join(reporter, "junit.xml") }], ["blob", { outputDir: path.join(reporter, "blob-report") }]] })};`); + const failed = await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, timeout: 30_000, maxBuffer: 1024 * 1024, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "", PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1" } }).then(() => false, error => { expect(error.code).toBe(1); return true; }); + expect(failed).toBe(true); + producer = await inspectCellReportProducer(); +}, 30_000); +afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); }); +afterAll(async () => { await rm(producerRoot, { recursive: true, force: true }); }); + +async function fixture() { + const root = await mkdtemp(path.join(temporaryRoot, "synthetic-private-copies-")); roots.push(root); + const id = MANUAL_PRIVATE_EXECUTIONS[0]!, cell = path.join(root, "capture/cases/activity.agent-feedback"); await mkdir(cell, { recursive: true }); + await copySharedEvidence(reporter, cell, path.join(root, "controller"), []); + await writeFile(path.join(cell, "result.json"), '{"status":"failed","failureClass":"assertion"}'); + const context = { root, private: true, protocol: MANUAL_PRIVATE_PROTOCOL, workflowSha: "a".repeat(40), targetSha: "b".repeat(40), runId: "123", attempt: "1", executionIds: [id], cellReportProducer: producer } as ManualPrivateHandoff; + const prefix = `gha-123-1-${id}/chat-features/runner-opencode/local/activity.agent-feedback/attempt-1`; + return { root, id, cell, context, prefix, upload: path.join(root, "results", prefix) }; +} +async function traceOf(cell: string) { + const outputs = (await files(cell)).filter(name => /^playwright-output\/[^/]+\/trace\.zip$/.test(name)); + expect(outputs).toHaveLength(1); + return { relative: outputs[0]!, bytes: await readFile(path.join(cell, outputs[0]!)) }; +} + +it("drops only playwright-output copies the HTML report already holds, leaving the HTML report and blob report unchanged", async () => { + const f = await fixture(), trace = await traceOf(f.cell); + const keptAt = `html-report/data/${sha1(trace.bytes)}.zip`; + expect(await readFile(path.join(f.cell, keptAt))).toEqual(trace.bytes); + const before = await files(f.cell), reports = before.filter(name => /^(?:html-report|blob-report)\//.test(name)); + const reportBytes = await Promise.all(reports.map(name => readFile(path.join(f.cell, name)))); + const dropped = await dropDuplicatePlaywrightOutput(f.cell); + expect(dropped.map(row => row.path)).toContain(trace.relative); + expect(dropped.find(row => row.path === trace.relative)).toMatchObject({ keptAt, bytes: trace.bytes.length }); + // Every dropped file has its exact bytes at the recorded HTML report path. + for (const row of dropped) expect(sha1(await readFile(path.join(f.cell, row.keptAt)))).toBe(path.basename(row.keptAt, path.extname(row.keptAt))); + const after = await files(f.cell); + expect(after.filter(name => name.endsWith("trace.zip"))).toEqual([]); + expect(after).toContain(keptAt); + expect(await Promise.all(reports.map(name => readFile(path.join(f.cell, name))))).toEqual(reportBytes); + // Files the HTML report does not copy, such as the run bookkeeping, stay. + expect(after).toContain("playwright-output/.last-run.json"); + expect(JSON.parse(await readFile(path.join(f.cell, PRIVATE_REPORT_COPIES_RECEIPT), "utf8")).dropped).toEqual(dropped); +}); + +it("keeps every validator accepting the deduplicated cell through package, both seals, trusted transfer, archive and report staging", async () => { + const f = await fixture(), trace = await traceOf(f.cell); + const keptAt = `html-report/data/${sha1(trace.bytes)}.zip`; + const originalBytes = await bytesOf(f.cell); + const dropped = await dropDuplicatePlaywrightOutput(f.cell); + const receiptBytes = (await lstat(path.join(f.cell, PRIVATE_REPORT_COPIES_RECEIPT))).size; + expect(originalBytes - await bytesOf(f.cell) + receiptBytes).toBe(dropped.reduce((sum, row) => sum + row.bytes, 0)); + const owner = await privateCellEvidenceOwner(f.context, f.id); + const evidence = await packageEvidence({ privateDir: f.cell, uploadDir: f.upload, secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } }); + expect(evidence.leaks).toEqual([]); + const original = await sealStandardOriginalEvidence({ context: f.context, executionId: f.id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false }); + await sealManualResultEvidence(f.context, []); + const originalDir = path.join(f.root, "originals", manualCellKind(f.id)), finalDir = path.join(f.root, "sealed-results/html-report"); + expect((await validateSealedOriginals(f.root, f.id, f.context)).inventorySha256).toBe(original.inventorySha256); + const final = await validateSealedInventory(finalDir, "../result-evidence-inventory.json", undefined, "standard", f.context); + for (const tree of [original.files.map((row: { path: string }) => row.path), final.files.map((row: { path: string }) => row.path.slice(f.prefix.length + 1))]) { + expect(tree).toContain(keptAt); + expect(tree).toContain(PRIVATE_REPORT_COPIES_RECEIPT); + expect(tree.filter((name: string) => name.endsWith("trace.zip"))).toEqual([]); + expect(tree.some((name: string) => /^blob-report\/.+\.zip$/.test(name))).toBe(true); + } + expect(await readFile(path.join(originalDir, keptAt))).toEqual(trace.bytes); + const bundle = path.join(f.root, "bundle"); await mkdir(bundle); + await cp(path.join(f.root, "originals"), path.join(bundle, "originals"), { recursive: true }); + await cp(path.join(f.root, "sealed-results"), path.join(bundle, "sealed-results"), { recursive: true }); + await writeFile(path.join(bundle, "identity.json"), JSON.stringify(f.context)); + const zip = path.join(f.root, "retained.zip"), restored = path.join(f.root, "restored"); + await archivePrivateEvidence("pack", bundle, zip); await archivePrivateEvidence("unpack", zip, restored); + const downloaded = { ...f.context, root: restored }; + expect((await validateSealedOriginals(restored, f.id, downloaded)).complete).toBe(true); + const selected = path.join(f.root, "selected"); await mkdir(selected); + await stageSealedPrivateCell(restored, selected, f.id, downloaded); + const staged = path.join(selected, manualCellKind(f.id), f.prefix); + expect(await readFile(path.join(staged, keptAt))).toEqual(trace.bytes); +}); + +it("keeps the playwright-output file when the HTML report copy is missing, differs, or is not a regular file", async () => { + const f = await fixture(), trace = await traceOf(f.cell), keptAt = path.join(f.cell, `html-report/data/${sha1(trace.bytes)}.zip`); + const tampered = Buffer.from(trace.bytes); tampered[tampered.length - 1] ^= 1; + await writeFile(keptAt, tampered); + expect((await dropDuplicatePlaywrightOutput(f.cell)).map(row => row.path)).not.toContain(trace.relative); + expect(await readFile(path.join(f.cell, trace.relative))).toEqual(trace.bytes); + await rm(path.join(f.cell, PRIVATE_REPORT_COPIES_RECEIPT), { force: true }); + await rm(keptAt); await symlink(path.join(f.cell, trace.relative), keptAt); + expect((await dropDuplicatePlaywrightOutput(f.cell)).map(row => row.path)).not.toContain(trace.relative); + expect(await readFile(path.join(f.cell, trace.relative))).toEqual(trace.bytes); +}); + +it("leaves trees alone when the HTML data directory is a symlink or the framework JSON reporter owns playwright-output", async () => { + const linked = await fixture(), data = path.join(linked.cell, "html-report/data"), moved = path.join(linked.root, "moved-data"); + await cp(data, moved, { recursive: true }); await rm(data, { recursive: true }); await symlink(moved, data); + const linkedBefore = await files(linked.cell); + expect(await dropDuplicatePlaywrightOutput(linked.cell)).toEqual([]); + expect(await files(linked.cell)).toEqual(linkedBefore); + const framework = await fixture(); + await writeFile(path.join(framework.cell, "framework-report.json"), "{}"); + const frameworkBefore = await files(framework.cell); + expect(await dropDuplicatePlaywrightOutput(framework.cell)).toEqual([]); + expect(await files(framework.cell)).toEqual(frameworkBefore); +}); diff --git a/tests/runner-e2e/private-report-copies.ts b/tests/runner-e2e/private-report-copies.ts new file mode 100644 index 00000000000..96624135827 --- /dev/null +++ b/tests/runner-e2e/private-report-copies.ts @@ -0,0 +1,100 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { lstat, mkdir, readdir, realpath, rmdir, unlink, writeFile } from "node:fs/promises"; +import path from "node:path"; + +/** Records which Playwright output files were dropped because the per-cell HTML + * report already holds the identical bytes. It names paths and digests only. */ +export const PRIVATE_REPORT_COPIES_RECEIPT = "snapshots/playwright-output-copies.json"; + +export interface DroppedReportCopy { path: string; keptAt: string; bytes: number; sha256: string } + +const outputDirectory = "playwright-output"; +const dataDirectory = "html-report/data"; +const entryAllowance = 4096; +const depthAllowance = 16; +// The JSON framework reporter registers playwright-output attachments by path; +// those trees have no HTML report copy and must stay untouched. +const frameworkReports = ["framework-report.json", "snapshots/playwright-framework.json"]; + +async function entry(file: string) { + return lstat(file).catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return undefined; + throw error; + }); +} + +async function digests(file: string) { + const sha1 = createHash("sha1"), sha256 = createHash("sha256"); + let bytes = 0; + for await (const chunk of createReadStream(file, { flags: "r" }) as AsyncIterable) { + sha1.update(chunk); sha256.update(chunk); bytes += chunk.length; + } + return { sha1: sha1.digest("hex"), sha256: sha256.digest("hex"), bytes }; +} + +/** + * Playwright's HTML reporter copies every test attachment, including the failure + * trace and video, into html-report/data/. The same + * bytes also stay in playwright-output, so a retained failure stores its largest + * files twice before the blob report adds a third (compressed) copy. + * + * This removes a playwright-output file only when the HTML report holds an + * independently verified, byte-identical regular file under the content-derived + * name. The HTML report and its trace viewer keep working, the blob report is + * untouched, and every remaining file still passes the normal packaging, + * sealing and transfer checks. Anything unexpected leaves the file in place. + */ +export async function dropDuplicatePlaywrightOutput(cellDirectory: string): Promise { + const root = await realpath(cellDirectory); + const rootInfo = await lstat(cellDirectory); + if (!rootInfo.isDirectory() || rootInfo.isSymbolicLink() || root !== path.resolve(cellDirectory)) + throw new Error("Invalid private report cell directory"); + for (const relative of frameworkReports) if (await entry(path.join(root, relative))) return []; + const output = path.join(root, outputDirectory), data = path.join(root, dataDirectory); + for (const directory of [output, path.join(root, "html-report"), data]) { + const info = await entry(directory); + if (!info || !info.isDirectory() || info.isSymbolicLink()) return []; + } + const dropped: DroppedReportCopy[] = []; + let visited = 0; + async function visit(relative: string, depth: number): Promise { + if (depth > depthAllowance) return false; + const directory = path.join(root, relative); + let remaining = 0; + for (const item of await readdir(directory, { withFileTypes: true })) { + if (++visited > entryAllowance) throw new Error("Playwright output exceeds entry allowance"); + const name = `${relative}/${item.name}`, file = path.join(root, name); + if (item.isDirectory() && !item.isSymbolicLink()) { + if (await visit(name, depth + 1)) await rmdir(file); else remaining++; + continue; + } + remaining++; + if (!item.isFile() || item.isSymbolicLink()) continue; + const before = await lstat(file); + if (!before.isFile() || before.isSymbolicLink() || before.size === 0) continue; + const extension = path.extname(item.name); + if (!/^\.[A-Za-z0-9]{1,16}$/.test(extension)) continue; + const source = await digests(file); + const keptAt = `${dataDirectory}/${source.sha1}${extension}`, kept = path.join(root, keptAt); + const keptInfo = await entry(kept); + if (!keptInfo || !keptInfo.isFile() || keptInfo.isSymbolicLink() || keptInfo.size !== before.size || source.bytes !== before.size) continue; + const copy = await digests(kept); + const after = await lstat(file); + if (copy.sha256 !== source.sha256 || copy.bytes !== source.bytes || after.ino !== before.ino + || after.size !== before.size || after.mtimeMs !== before.mtimeMs) continue; + await unlink(file); + remaining--; + dropped.push({ path: name, keptAt, bytes: source.bytes, sha256: source.sha256 }); + } + return remaining === 0; + } + await visit(outputDirectory, 0); + if (!dropped.length) return dropped; + dropped.sort((a, b) => a.path.localeCompare(b.path)); + const receipt = path.join(root, PRIVATE_REPORT_COPIES_RECEIPT); + await mkdir(path.dirname(receipt), { recursive: true, mode: 0o700 }); + await writeFile(receipt, `${JSON.stringify({ schema: "paperclip.private-report-copies/v1", + meaning: "playwright-output files removed because html-report/data holds identical bytes", dropped }, null, 2)}\n`, { mode: 0o600, flag: "wx" }); + return dropped; +} diff --git a/tests/runner-e2e/report.test.ts b/tests/runner-e2e/report.test.ts index c65610db9d3..8a492b5156a 100644 --- a/tests/runner-e2e/report.test.ts +++ b/tests/runner-e2e/report.test.ts @@ -697,6 +697,33 @@ describe("runner E2E report aggregation", () => { ).toBe("warm-turn-png"); }); + it("does not stage a private cell's trace viewer font, SVG or manifest into the merged report", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "runner-e2e-report-carriers-")); + cleanupDirectories.push(root); + const executionId = "chat-features.runner-opencode.local.runs-adapters.inspect-failed-run"; + const directory = path.join(root, "attempt-1"); + const staged = ["final-state.png", "html-report/index.html", "html-report/trace/index.html", "html-report/trace/index.KZ4wOW1K.js", "html-report/data/0123456789abcdef0123456789abcdef01234567.zip"]; + const refused = ["html-report/trace/codicon.DCmgc-ay.ttf", "html-report/trace/manifest.webmanifest", "html-report/trace/playwright-logo.svg"]; + for (const file of [...staged, ...refused]) { + await mkdir(path.join(directory, path.dirname(file)), { recursive: true }); + await writeFile(path.join(directory, file), `synthetic ${file}`); + } + await writeFile(path.join(directory, "result.json"), JSON.stringify({ + schema: "paperclip.runner-e2e.result/v1", executionId, attempt: 1, status: "passed", + profileId: "runner-opencode", environmentId: "local", caseId: "runs-adapters.inspect-failed-run", provider: "opencode", model: "fixture-model", + runtimeMode: "native", startedAt: "2026-11-07T00:00:00.000Z", finishedAt: "2026-11-07T00:00:01.000Z", durationMs: 1_000, cleanup: "passed", + } satisfies RunnerE2EResult)); + await writeFile(path.join(directory, "evidence-manifest.json"), JSON.stringify({ files: [...staged, ...refused], leaks: [], missing: [] })); + const output = path.join(root, "merged"); + await execFileAsync(process.execPath, [path.join(repositoryRoot, "cli/node_modules/tsx/dist/cli.mjs"), path.join(repositoryRoot, "tests/runner-e2e/report.ts")], { + cwd: repositoryRoot, + env: { ...process.env, PAPERCLIP_RUNNER_E2E_REPORT_ROOT: root, PAPERCLIP_RUNNER_E2E_REPORT_OUT: output, PAPERCLIP_RUNNER_E2E_EXPECTED_IDS: JSON.stringify([executionId]) }, + }).catch(error => { throw new Error(String(error.stderr).slice(-1500)); }); + const evidence = path.join(output, "evidence", executionId, "attempt-1"); + for (const file of staged) expect(await readFile(path.join(evidence, file), "utf8")).toBe(`synthetic ${file}`); + for (const file of refused) await expect(readFile(path.join(evidence, file))).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("constructs the public root JUnit from fixed markup and escaped fields", async () => { const root = await mkdtemp( path.join(os.tmpdir(), "runner-e2e-report-junit-test-"), diff --git a/tests/runner-e2e/report.ts b/tests/runner-e2e/report.ts index 7167e7316c1..7a71042e642 100644 --- a/tests/runner-e2e/report.ts +++ b/tests/runner-e2e/report.ts @@ -24,6 +24,7 @@ import { upgradeRunnerResult, } from "./history.js"; import { resolveRunnerE2ESource } from "./source.js"; +import { isStandardEvidenceFile } from "./evidence.js"; import { runnerE2ESummaryLinks } from "./summary-links.js"; import type { RunnerE2EResult } from "./types.js"; import { gradeRetainedDependableResults, type DependableQualification } from "./dependable-qualification.js"; @@ -120,6 +121,12 @@ async function stageDashboardEvidence( for (const relative of entry.evidence?.files ?? []) { const segments = safeEvidenceRelative(relative); if (!segments) continue; + // A private cell also lists its HTML report's trace viewer program files + // (font, SVG, web manifest). They are not evidence: the merged report has + // its own authenticated viewer, and the private report seal refuses these + // carrier types anywhere else, losing the whole report. Apply that same + // carrier rule here; the cell archive still keeps the files. + if (!isStandardEvidenceFile(path.join("html-report", ...segments))) continue; const source = path.join(entry.directory, ...segments); const destination = path.join(output, ...baseSegments, ...segments); const didCopy = await mkdir(path.dirname(destination), {