From 91d905c38f06d1a1145592df383ae8a6a11d9c13 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:09:16 -0600 Subject: [PATCH] fix(evals): retain canonical private framework attachments once --- tests/runner-e2e/README.md | 21 +++ tests/runner-e2e/evidence.ts | 8 +- tests/runner-e2e/launch.ts | 5 +- tests/runner-e2e/manual-private-transfer.ts | 13 ++ tests/runner-e2e/playwright.config.ts | 6 +- .../private-framework-report.test.ts | 140 ++++++++++++++++ tests/runner-e2e/private-framework-report.ts | 154 ++++++++++++++++++ .../runner-e2e/private-reporter-artifacts.ts | 4 +- tests/runner-e2e/shared-evidence.ts | 2 + 9 files changed, 348 insertions(+), 5 deletions(-) create mode 100644 tests/runner-e2e/private-framework-report.test.ts create mode 100644 tests/runner-e2e/private-framework-report.ts diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index b6847207e18..06194eb8062 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -634,6 +634,27 @@ matching API-key or subscription credential. Never put credentials in `catalog.ts`, screenshots, fixture metadata, workflow inputs, or a tracked env file. +## Private framework evidence + +An approved private-cell preflight uses Playwright's official JSON and JUnit +reporters. Other runs keep the existing HTML and blob reporters. The immutable +JSON source is retained in `snapshots/playwright-framework-original.txt` and is +parsed and deeply inspected as registered Playwright JSON, including decoded +inline text/JSON attachment bodies. The text suffix preserves source bytes; it +does not exempt the file from JSON privacy checks. Unsupported inline carriers +fail closed. + +`snapshots/playwright-framework.json` preserves the report and maps attachment +paths to their owned canonical files with byte counts and SHA-256 hashes. +Original status, errors, and nested user steps remain unchanged. Sealed originals +retain all owned attachments. The final-results tree references those originals +instead of storing another copy of registered Playwright output attachments. +The downloader verifies each reference against the sealed original inventory. +Unknown, missing, changed, or foreign attachments fail validation. Native +snapshots and other unregistered evidence are not removed. The existing physical +byte, entry, archive, ZIP expansion, time, and privacy bounds remain unchanged. +A complete evidence transfer does not attest that the feature passed. + ## Local commands When the host cannot start embedded Postgres, set diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index 2cc4307e2dc..6581a880c06 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -1,3 +1,4 @@ +import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport, inspectPrivateFrameworkMetadata } from "./private-framework-report.js"; import { spawn } from "node:child_process"; import { mkdir, @@ -246,6 +247,7 @@ export async function packageEvidence(input: { continue; } else if (TEXT_EXTENSIONS.has(extension)) { const raw = await readFile(source, "utf8"); + if (input.privateCell) inspectPrivateFrameworkMetadata(relative.split(path.sep).join("/"), raw, input.secrets); if (extension === ".html") { const issue = await inspectEmbeddedReport(raw, input.secrets); if (issue) { if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(issue); leaks.push({ file: relative, reason: issue }); continue; } @@ -301,10 +303,12 @@ export async function packageEvidence(input: { } if (budget) { budget.inventoryComplete = true; budget.scanFinished = true; } + const compact = input.privateCell?.layout === "original" && files.includes(PRIVATE_FRAMEWORK_REPORT) + ? await validatePrivateFrameworkReport(input.privateDir) : undefined; const missing = input.expectPassScreenshot ? [ - ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)), - ...(files.some( + ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required) && !(compact && required === path.join("html-report", "index.html"))), + ...(compact || files.some( (file) => file.startsWith(`blob-report${path.sep}`) && file.endsWith(".zip"), ) diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index dba1467e211..c88df15f509 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -18,6 +18,7 @@ import { readUpgradeArtifacts, verifyUpgradeArtifact, upgradeArtifactKey } from import { recoverWorkerTiming } from "./worker-timing.js"; import { openEvalOperations, type EvalOperations } from "./operations.js"; import { readRecoveredFeedback } from "./feedback-recovery.js"; +import { PRIVATE_FRAMEWORK_REPORT, referenceSealedFrameworkAttachments } from "./private-framework-report.js"; import { readInterruptedAttemptTrace } from "./trace-recovery.js"; import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js"; import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js"; @@ -53,7 +54,7 @@ import { packageEvidence, publishRunnerE2EAttemptEvidence } from "./evidence.js" import { prepareRunnerE2ECampaignDirectory, resolveRunnerE2EResultsRoot } from "./results-root.js"; import { privateCellEvidenceOwner } from "./manual-private-report-assets.js"; import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js"; -import { requiresPrivateManualEvidence } from "./manual-private-retention.js"; +import { requiresPrivateManualEvidence, manualCellKind } from "./manual-private-retention.js"; import { copySharedEvidence } from "./shared-evidence.js"; import { classifyFailure } from "./failure-classifier.js"; import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js"; @@ -941,6 +942,8 @@ async function runAttempt(input: { manualDiagnostic!.stage = "seal-original"; await sealStandardOriginalEvidence({ context: manualPrivateHandoff, executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed }); + if (evidence.files.includes(PRIVATE_FRAMEWORK_REPORT)) await referenceSealedFrameworkAttachments(uploadDir, + path.join(manualPrivateHandoff.root, "originals", manualCellKind(candidate.id)), candidate.id); } else { result = await publishRunnerE2EAttemptEvidence({ result, diff --git a/tests/runner-e2e/manual-private-transfer.ts b/tests/runner-e2e/manual-private-transfer.ts index 3c70f966486..700bd055233 100644 --- a/tests/runner-e2e/manual-private-transfer.ts +++ b/tests/runner-e2e/manual-private-transfer.ts @@ -1,3 +1,4 @@ +import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport } from "./private-framework-report.js"; import { createHash } from "node:crypto"; import { appendFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -95,6 +96,18 @@ export async function validateSealedInventory(directory: string, inventory: stri } await visit(""); if (actual.size !== expected.size) throw new Error("Sealed evidence is incomplete"); + if (treeKind === "standard") for (const row of receipt.files) { + if (row.path === PRIVATE_FRAMEWORK_REPORT || row.path.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`)) { + const cell = path.join(directory, row.path.slice(0, -PRIVATE_FRAMEWORK_REPORT.length)); + if (id !== undefined) await validatePrivateFrameworkReport(cell); + else { + const metadata = JSON.parse(await readFile(path.join(directory, row.path), "utf8")); + const owner = metadata.canonicalExecutionId; + if (!context || typeof owner !== "string" || !context.executionIds.includes(owner)) throw new Error("Unbound canonical framework evidence"); + await validatePrivateFrameworkReport(cell, path.join(context.root, "originals", manualCellKind(owner)), owner); + } + } + } return receipt; } export async function validateSealedOriginals(root: string, id: string, context?: ManualPrivateHandoff) { diff --git a/tests/runner-e2e/playwright.config.ts b/tests/runner-e2e/playwright.config.ts index 4cdf143167c..9cf4f4c5d9d 100644 --- a/tests/runner-e2e/playwright.config.ts +++ b/tests/runner-e2e/playwright.config.ts @@ -81,7 +81,11 @@ export default defineConfig({ stderr: "pipe", }, outputDir: path.join(privateDir, "playwright-output"), - reporter: [ + reporter: process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT ? [ + ["list"], + ["json", { outputFile: path.join(privateDir, "framework-report.json") }], + ["junit", { outputFile: path.join(privateDir, "junit.xml") }], + ] : [ ["list"], ["blob", { outputDir: path.join(privateDir, "blob-report") }], ["junit", { outputFile: path.join(privateDir, "junit.xml") }], diff --git a/tests/runner-e2e/private-framework-report.test.ts b/tests/runner-e2e/private-framework-report.test.ts new file mode 100644 index 00000000000..cc4fb594f66 --- /dev/null +++ b/tests/runner-e2e/private-framework-report.test.ts @@ -0,0 +1,140 @@ +import type { ManualPrivateHandoff } from "./manual-private-handoff.js"; +import { afterAll, expect, it } from "vitest"; +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import { cp, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { promisify } from "node:util"; +import { copyPrivateFrameworkReport, inspectPrivateFrameworkMetadata, PRIVATE_FRAMEWORK_ORIGINAL_REPORT, PRIVATE_FRAMEWORK_REPORT, referenceSealedFrameworkAttachments, validatePrivateFrameworkReport } from "./private-framework-report.js"; +const roots: string[] = []; +afterAll(async () => { for (const root of roots) await rm(root, { recursive: true, force: true }); }); +async function fixture() { + const root = await mkdtemp(path.join(await realpath(os.tmpdir()), "framework-json-control-")); roots.push(root); + const source = path.join(root, "capture"), cell = path.join(source, "cases/activity.agent-feedback"), output = path.join(source, "playwright-output/owned"); + await mkdir(cell, { recursive: true }); await mkdir(output, { recursive: true }); + await writeFile(path.join(output, "trace.zip"), "synthetic canonical trace"); + const report = { config: { version: "1.62.1" }, errors: [], suites: [{ specs: [{ tests: [{ results: [{ status: "failed", error: { message: "actual failure" }, errors: [{ message: "actual failure" }], stdout: [], stderr: [], steps: [{ title: "owned failed step", error: { message: "actual failure" } }], attachments: [{ name: "trace", contentType: "application/zip", path: path.join(output, "trace.zip") }] }] }] }], suites: [] }] }; + await writeFile(path.join(source, "framework-report.json"), JSON.stringify(report)); + await cp(path.join(source, "playwright-output"), path.join(cell, "playwright-output"), { recursive: true }); + return { root, source, cell, report }; +} +it("keeps official failed status/errors/steps and binds one canonical attachment across original and result trees", async () => { + const f = await fixture(); await writeFile(path.join(f.cell, "playwright-output/unregistered.log"), "unexplained evidence retained"); await copyPrivateFrameworkReport(f.source, f.cell, []); + const value = await validatePrivateFrameworkReport(f.cell); + expect((value.report.suites as any)[0].specs[0].tests[0].results[0]).toMatchObject({ status: "failed", error: { message: "actual failure" }, steps: [{ title: "owned failed step" }] }); + const original = path.join(f.root, "original"); await cp(f.cell, original, { recursive: true }); + await referenceSealedFrameworkAttachments(f.cell, original, "owned"); + await expect(readFile(path.join(f.cell, "playwright-output/owned/trace.zip"))).rejects.toMatchObject({ code: "ENOENT" }); + expect(await readFile(path.join(f.cell, "playwright-output/unregistered.log"), "utf8")).toBe("unexplained evidence retained"); + expect(await readFile(path.join(original, "playwright-output/owned/trace.zip"), "utf8")).toBe("synthetic canonical trace"); + await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).resolves.toMatchObject({ canonicalExecutionId: "owned" }); + await expect(validatePrivateFrameworkReport(f.cell, original, "foreign")).rejects.toThrow(); + await writeFile(path.join(original, "playwright-output/owned/trace.zip"), "tampered"); + await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow(); +}); +it.each(["empty", "malformed", "foreign", "missing", "secret", "body-secret", "foreign-version", "unknown-status"])("refuses %s framework evidence", async kind => { + const f = await fixture(); + if (kind === "foreign-version") f.report.config.version = "0.0.0"; + if (kind === "unknown-status") f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.status = "invented"; + if (kind === "empty") f.report.suites = []; + if (kind === "foreign") f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]!.path = path.join(f.root, "outside.zip"); + if (kind === "missing") await rm(path.join(f.cell, "playwright-output/owned/trace.zip")); + if (kind === "secret") f.report.errors = [{ message: "synthetic-private-credential" }] as any; + if (kind === "body-secret") (f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0] as any).body = Buffer.from("synthetic-private-credential").toString("base64"); + await writeFile(path.join(f.source, "framework-report.json"), kind === "malformed" ? "{" : JSON.stringify(f.report)); + await expect(copyPrivateFrameworkReport(f.source, f.cell, ["synthetic-private-credential"])).rejects.toThrow(); +}); +it("uses the real official Playwright JSON producer with failed errors, user steps, and automatic trace", async () => { + const f = await fixture(), tests = path.join(f.root, "tests"); await mkdir(tests); + const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); + await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('original failed result',async()=>{await test.step('owned failed step',async()=>expect('actual').toBe('expected'));});`); + const config = path.join(tests, "config.cjs"); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, outputDir: path.join(f.source, "playwright-output"), use: { trace: "retain-on-failure" }, reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }]] })};`); + await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" }, timeout: 30000 }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); }); + await rm(path.join(f.cell, "playwright-output"), { recursive: true }); + await cp(path.join(f.source, "playwright-output"), path.join(f.cell, "playwright-output"), { recursive: true }); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const retained = await validatePrivateFrameworkReport(f.cell), result = (retained.report.suites as any)[0].specs[0].tests[0].results[0]; + expect(result.status).toBe("failed"); expect(result.errors.length).toBeGreaterThan(0); expect(result.steps[0].title).toBe("owned failed step"); + expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true); +}); +it("keeps the unchanged 256MiB packaging/archive caps with one canonical large attachment", async () => { + const { packageEvidence } = await import("./evidence.js"); + const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js"); + const { archivePrivateEvidence } = await import("./private-history.js"); + const { sealStandardOriginalEvidence, sealManualResultEvidence } = await import("./manual-private-handoff.js"); + const { validateSealedOriginals, validateSealedInventory } = await import("./manual-private-transfer.js"); + const { MANUAL_PRIVATE_PROTOCOL, manualCellKind } = await import("./manual-private-retention.js"); + const f = await fixture(), id = "chat-features.runner-opencode.local.activity.agent-feedback"; + const context: ManualPrivateHandoff = { root: await (await import("node:fs/promises")).realpath(f.root), private: true as const, protocol: MANUAL_PRIVATE_PROTOCOL, + runId: "123", attempt: "1", workflowSha: "a".repeat(40), targetSha: "b".repeat(40), executionIds: [id], cellReportProducer: await inspectCellReportProducer() }; + const zip = path.join(f.root, "input.zip"); + await promisify(execFile)("python3", ["-c", "import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_STORED);f=z.open('synthetic.trace','w');[f.write(b'0'*1048576) for _ in range(140)];f.close();z.close()", zip], { timeout: 30000 }); + const tests = path.join(f.root, "budget-tests"); await mkdir(tests); + const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); + await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>expect(1).toBe(2));});`); + const config = path.join(tests, "config.cjs"); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, use: { trace: "retain-on-failure" }, outputDir: path.join(f.source, "playwright-output"), reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`); + await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, timeout: 30000, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" } }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); }); + await rm(path.join(f.cell, "playwright-output"), { recursive: true }); + for (const folder of ["playwright-output", "html-report", "blob-report"]) await cp(path.join(f.source, folder), path.join(f.cell, folder), { recursive: true }); + const owner = await privateCellEvidenceOwner(context, id), prefix = `gha-123-1-${id}/chat-features/runner-opencode/local/activity.agent-feedback/attempt-1`; + const upload = path.join(context.root, "results", prefix); + const input = { privateDir: f.cell, uploadDir: upload, secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" as const } }; + await expect(packageEvidence(input)).rejects.toThrow("Private cell evidence exceeds byte allowance"); + for (const folder of ["html-report", "blob-report"]) await rm(path.join(f.cell, folder), { recursive: true }); + await copyPrivateFrameworkReport(f.source, f.cell, []); + await writeFile(path.join(f.cell, "result.json"), '{"status":"failed","error":"original failed assertion"}'); + const evidence = await packageEvidence(input); expect(evidence.leaks).toEqual([]); expect(evidence.budget?.bytesExamined).toBeLessThan(256 * 1024 * 1024); + await sealStandardOriginalEvidence({ context, executionId: id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false }); + const original = path.join(context.root, "originals", manualCellKind(id)); + await referenceSealedFrameworkAttachments(upload, original, id); + await sealManualResultEvidence(context, []); + expect((await validateSealedOriginals(context.root, id, context)).complete).toBe(true); + expect((await validateSealedInventory(path.join(context.root, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", context)).complete).toBe(true); + const bundle = path.join(context.root, "bundle"); await mkdir(bundle); + await cp(path.join(context.root, "originals"), path.join(bundle, "originals"), { recursive: true }); + await cp(path.join(context.root, "sealed-results"), path.join(bundle, "sealed-results"), { recursive: true }); + const archive = path.join(context.root, "retained.zip"), restored = path.join(context.root, "restored"); + await archivePrivateEvidence("pack", bundle, archive); await archivePrivateEvidence("unpack", archive, restored); + const downloaded = { ...context, root: restored }; + await expect(validateSealedOriginals(restored, id, downloaded)).resolves.toMatchObject({ complete: true }); + await expect(validateSealedInventory(path.join(restored, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", downloaded)).resolves.toMatchObject({ complete: true }); + expect(JSON.parse(await readFile(path.join(restored, "originals", manualCellKind(id), "result.json"), "utf8")).status).toBe("failed"); +}, 30000); + +it.each(["literal", "escaped", "base64-body", "escaped-json-body"])("deep-scans registered immutable raw JSON text: %s", async kind => { + const f = await fixture(), canary = "synthetic-private-credential"; + const attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any; + if (kind === "literal" || kind === "escaped") f.report.errors = [{ message: canary }] as any; + else { + attachment.contentType = kind === "base64-body" ? "text/plain" : "application/json"; + attachment.body = Buffer.from(kind === "base64-body" ? canary : JSON.stringify({ value: canary }).replace("s", "\\u0073")).toString("base64"); + } + let raw = JSON.stringify(f.report); + if (kind === "escaped") raw = raw.replace(canary, "\\u0073" + canary.slice(1)); + expect(() => inspectPrivateFrameworkMetadata(PRIVATE_FRAMEWORK_ORIGINAL_REPORT, raw, [canary])).toThrow(); +}); +it("retains and checks a clean official inline text attachment without losing its body", async () => { + const f = await fixture(); const attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any; + attachment.contentType = "text/plain"; attachment.body = Buffer.from("safe diagnostic").toString("base64"); + await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report)); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const retained = await validatePrivateFrameworkReport(f.cell); + expect((retained.report.suites as any)[0].specs[0].tests[0].results[0].attachments[0].body).toBe(attachment.body); +}); + +it("rejects a symbolic attachment and changed final framework outcome", async () => { + const f = await fixture(), file = path.join(f.cell, "playwright-output/owned/trace.zip"); + await rm(file); await symlink(path.join(f.source, "playwright-output/owned/trace.zip"), file); + await expect(copyPrivateFrameworkReport(f.source, f.cell, [])).rejects.toThrow(); + await rm(file); await cp(path.join(f.source, "playwright-output/owned/trace.zip"), file); + await copyPrivateFrameworkReport(f.source, f.cell, []); + const original = path.join(f.root, "original"); await cp(f.cell, original, { recursive: true }); + await referenceSealedFrameworkAttachments(f.cell, original, "owned"); + const sidecar = JSON.parse(await readFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), "utf8")); + sidecar.report.suites[0].specs[0].tests[0].results[0].status = "passed"; + await writeFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), JSON.stringify(sidecar)); + await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow(); +}); diff --git a/tests/runner-e2e/private-framework-report.ts b/tests/runner-e2e/private-framework-report.ts new file mode 100644 index 00000000000..d00c5f9056c --- /dev/null +++ b/tests/runner-e2e/private-framework-report.ts @@ -0,0 +1,154 @@ +import { createRequire } from "node:module"; +import { createHash } from "node:crypto"; +import { lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets } from "./redaction.js"; + +export const PRIVATE_FRAMEWORK_ORIGINAL_REPORT = "snapshots/playwright-framework-original.txt"; +export const PRIVATE_FRAMEWORK_REPORT = "snapshots/playwright-framework.json"; +const producerVersion: string = createRequire(import.meta.url)("@playwright/test/package.json").version; +const LIMIT = 256 * 1024 * 1024; +const hash = (data: Buffer) => createHash("sha256").update(data).digest("hex"); +const fail = (): never => { throw new Error("Invalid private framework evidence"); }; +type Row = { path: string; bytes: number; sha256: string }; +type FrameworkReport = { schema: string; sourceReportSha256: string; report: Record; attachments: Row[]; canonicalExecutionId?: string }; +async function regular(root: string, relative: string, limit = LIMIT) { + if (!relative || relative.includes("\\") || relative.split("/").some(p => !p || p === "." || p === "..") || /[\x00-\x1f:]/.test(relative)) fail(); + let current = root; + for (const part of relative.split("/")) { current = path.join(current, part); const entry = await lstat(current); if (entry.isSymbolicLink()) fail(); } + const info = await lstat(current); + if (!info.isFile() || info.size > limit || !(await realpath(current)).startsWith(`${await realpath(root)}${path.sep}`)) fail(); + const data = await readFile(current), after = await lstat(current); + if (data.length !== info.size || after.ino !== info.ino || after.mtimeMs !== info.mtimeMs || after.ctimeMs !== info.ctimeMs) fail(); + return data; +} +function results(report: Record) { + if (!report.config || typeof report.config !== "object" || !Array.isArray(report.suites) || !report.suites.length || !Array.isArray(report.errors)) fail(); + if ((report.config as Record).version !== producerVersion) fail(); + const found: Array> = []; let visited = 0; + const suite = (value: any) => { + if (++visited > 4096 || !value || (value.specs !== undefined && !Array.isArray(value.specs)) || (value.suites !== undefined && !Array.isArray(value.suites))) fail(); + for (const spec of value.specs ?? []) { + if (!spec || !Array.isArray(spec.tests) || !spec.tests.length) fail(); + for (const test of spec.tests) { + if (!test || !Array.isArray(test.results) || !test.results.length) fail(); + for (const result of test.results) { + if (++visited > 4096 || !result || !["passed", "failed", "timedOut", "skipped", "interrupted"].includes(result.status) + || !Array.isArray(result.attachments) || !Array.isArray(result.errors) || !Array.isArray(result.stdout) || !Array.isArray(result.stderr)) fail(); + found.push(result); + } + } + } + for (const child of value.suites ?? []) suite(child); + }; + for (const value of report.suites as unknown[]) suite(value); + if (!found.length) fail(); + return found; +} +export function inspectPrivateFrameworkMetadata(relative: string, raw: string, secrets: readonly string[]) { + const original = relative === PRIVATE_FRAMEWORK_ORIGINAL_REPORT || relative.endsWith(`/${PRIVATE_FRAMEWORK_ORIGINAL_REPORT}`) || relative === "framework-report.json"; + const sidecar = relative === PRIVATE_FRAMEWORK_REPORT || relative.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`); + if (!original && !sidecar) return false; + if (Buffer.byteLength(raw) > 128 * 1024 * 1024) fail(); + const parsed = JSON.parse(raw), report = original ? parsed : parsed.report; + const expanded = normalizedSecrets(secrets).flatMap(v => [v, encodeURIComponent(v), Buffer.from(v).toString("base64"), Buffer.from(v).toString("base64url")]); + if (findSecretLeakInJsonValues(report, expanded)) fail(); + for (const result of results(report)) for (const attachment of result.attachments as any[]) { + if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail(); + if (attachment.body !== undefined) { + if (typeof attachment.body !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.body)) fail(); + const body = Buffer.from(attachment.body, "base64"), text = body.toString("utf8"); + if (!["text/plain", "application/json"].includes(attachment.contentType) || !Buffer.from(text).equals(body) || findSecretLeak(body, expanded)) fail(); + if (attachment.contentType === "application/json" && findSecretLeak(JSON.stringify(JSON.parse(text)), expanded)) fail(); + } + } + return true; +} +export async function copyPrivateFrameworkReport(source: string, destination: string, secrets: readonly string[]) { + const raw = await regular(source, "framework-report.json", 128 * 1024 * 1024); + assertSecretFree(raw.toString("utf8"), secrets, "private framework metadata"); + const report = JSON.parse(raw.toString("utf8")); + const attachments = new Map(); + inspectPrivateFrameworkMetadata("framework-report.json", raw.toString("utf8"), secrets); + for (const result of results(report)) for (const attachment of result.attachments as any[]) { + if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail(); + if (attachment.path === undefined) { if (attachment.body === undefined) fail(); continue; } + if (typeof attachment.path !== "string" || !path.isAbsolute(attachment.path)) fail(); + const absolute = path.resolve(attachment.path), output = path.join(source, "playwright-output"); + let relative: string; + if (absolute.startsWith(`${path.resolve(destination)}${path.sep}`)) relative = path.relative(destination, absolute); + else if (absolute.startsWith(`${path.resolve(output)}${path.sep}`)) relative = path.relative(source, absolute); + else fail(); + relative = relative!.split(path.sep).join("/"); + const data = await regular(absolute.startsWith(`${path.resolve(destination)}${path.sep}`) ? destination : source, relative); + if (relative === "result.json") { + relative = "snapshots/playwright-attached-result.json"; + await mkdir(path.dirname(path.join(destination, relative)), { recursive: true, mode: 0o700 }); + await writeFile(path.join(destination, relative), data, { flag: "wx", mode: 0o600 }); + } else { + const copied = await regular(destination, relative); + if (!copied.equals(data)) fail(); + } + attachment.path = relative; + const row = { path: relative, bytes: data.length, sha256: hash(data) }; + const previous = attachments.get(relative); if (previous && JSON.stringify(previous) !== JSON.stringify(row)) fail(); + attachments.set(relative, row); + } + const sidecar: FrameworkReport = { schema: "paperclip.private-framework-json/v1", sourceReportSha256: hash(raw), report, + attachments: [...attachments.values()].sort((a, b) => a.path.localeCompare(b.path)) }; + await mkdir(path.join(destination, "snapshots"), { recursive: true, mode: 0o700 }); + await writeFile(path.join(destination, PRIVATE_FRAMEWORK_ORIGINAL_REPORT), raw, { flag: "wx", mode: 0o600 }); + await writeFile(path.join(destination, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(sidecar)}\n`, { flag: "wx", mode: 0o600 }); +} +export async function validatePrivateFrameworkReport(directory: string, canonical = directory, executionId?: string) { + const raw = await regular(directory, PRIVATE_FRAMEWORK_REPORT, 128 * 1024 * 1024); + const value = JSON.parse(raw.toString("utf8")) as FrameworkReport; + const keys = Object.keys(value).sort().join(); + if (keys !== (executionId ? "attachments,canonicalExecutionId,report,schema,sourceReportSha256" : "attachments,report,schema,sourceReportSha256") + || value.schema !== "paperclip.private-framework-json/v1" || !/^[0-9a-f]{64}$/.test(value.sourceReportSha256) + || executionId && value.canonicalExecutionId !== executionId || !Array.isArray(value.attachments) || value.attachments.length > 4096) fail(); + if (executionId) { + const original = await validatePrivateFrameworkReport(canonical); + const { canonicalExecutionId: _, ...retained } = value; + if (JSON.stringify(retained) !== JSON.stringify(original)) fail(); + } + const originalRaw = await regular(directory, PRIVATE_FRAMEWORK_ORIGINAL_REPORT, 128 * 1024 * 1024); + if (hash(originalRaw) !== value.sourceReportSha256) fail(); + inspectPrivateFrameworkMetadata(PRIVATE_FRAMEWORK_ORIGINAL_REPORT, originalRaw.toString("utf8"), []); + const authoritative = JSON.parse(originalRaw.toString("utf8")); + const originalResults = results(authoritative), retainedResults = results(value.report); + if (originalResults.length !== retainedResults.length) fail(); + for (let i = 0; i < originalResults.length; i++) { + const originalAttachments = originalResults[i]!.attachments as any[], retainedAttachments = retainedResults[i]!.attachments as any[]; + if (originalAttachments.length !== retainedAttachments.length) fail(); + for (let j = 0; j < originalAttachments.length; j++) { + if (originalAttachments[j].path !== undefined) originalAttachments[j].path = retainedAttachments[j].path; + } + } + if (JSON.stringify(authoritative) !== JSON.stringify(value.report)) fail(); + const referenced = new Set(); + for (const result of results(value.report)) for (const attachment of result.attachments as any[]) { + if (attachment.path !== undefined) { if (typeof attachment.path !== "string") fail(); referenced.add(attachment.path); } + } + const seen = new Set(); + for (const row of value.attachments) { + if (!row || Object.keys(row).sort().join() !== "bytes,path,sha256" || !referenced.has(row.path) || seen.has(row.path) + || !Number.isSafeInteger(row.bytes) || row.bytes < 0 || !/^[0-9a-f]{64}$/.test(row.sha256)) fail(); + const data = await regular(canonical, row.path); if (data.length !== row.bytes || hash(data) !== row.sha256) fail(); + seen.add(row.path); + } + if (seen.size !== referenced.size) fail(); + return value; +} +export async function referenceSealedFrameworkAttachments(directory: string, original: string, executionId: string) { + const value = await validatePrivateFrameworkReport(directory); + const sealed = await validatePrivateFrameworkReport(original); + if (JSON.stringify(value) !== JSON.stringify(sealed)) fail(); + for (const row of value.attachments.filter(r => r.path.startsWith("playwright-output/"))) { + const current = await regular(directory, row.path), canonical = await regular(original, row.path); + if (!current.equals(canonical)) fail(); + await rm(path.join(directory, row.path)); + } + value.canonicalExecutionId = executionId; + await writeFile(path.join(directory, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(value)}\n`, { mode: 0o600 }); +} diff --git a/tests/runner-e2e/private-reporter-artifacts.ts b/tests/runner-e2e/private-reporter-artifacts.ts index 99e1513816c..6d788cb3e5e 100644 --- a/tests/runner-e2e/private-reporter-artifacts.ts +++ b/tests/runner-e2e/private-reporter-artifacts.ts @@ -1,3 +1,4 @@ +import { inspectPrivateFrameworkMetadata } from "./private-framework-report.js"; import { lstat, readFile, readdir, rm, writeFile } from "node:fs/promises"; import path from "node:path"; import { inspectEmbeddedReport, inspectZip } from "./evidence.js"; @@ -5,7 +6,7 @@ import { diagnosticSecrets, type DiagnosticSecretExpansion } from "./private-dia import { findSecretLeak, redactText, sanitizeJson } from "./redaction.js"; import { cellTraceAsset, readPrivateCellTraceAsset, type PrivateCellEvidenceOwner } from "./manual-private-report-assets.js"; export interface ReporterPrivacyIssue { file: string; disposition: "redacted" | "omitted"; reason: string } -const sources = ["server.log", "playwright.log", "junit.xml", "html-report", "blob-report", "playwright-output"]; +const sources = ["server.log", "playwright.log", "junit.xml", "html-report", "blob-report", "playwright-output", "framework-report.json"]; const text = new Set([".log", ".txt", ".md", ".xml", ".html", ".json", ".js", ".css"]); /** Only reporter/diagnostic sinks are touched, after independent product oracles * finish. Product API snapshots and work-product bytes are never pre-sanitized. @@ -36,6 +37,7 @@ export async function sanitizePrivateReporterArtifacts(root: string, values: rea } else if (text.has(extension)) { if (info.size > 128 * 1024 * 1024) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter text exceeds inspection bound" }); return; } const raw = await readFile(filename, "utf8"); + if (relative === "framework-report.json") inspectPrivateFrameworkMetadata(relative, raw, secrets); const opaque = extension === ".html" ? await inspectEmbeddedReport(raw, secrets) : null; if (opaque) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: opaque }); return; } const leak = findSecretLeak(raw, secrets); diff --git a/tests/runner-e2e/shared-evidence.ts b/tests/runner-e2e/shared-evidence.ts index 85738682722..d89d7adc7dc 100644 --- a/tests/runner-e2e/shared-evidence.ts +++ b/tests/runner-e2e/shared-evidence.ts @@ -1,3 +1,4 @@ +import { copyPrivateFrameworkReport } from "./private-framework-report.js"; import type { SourceSdkAuditCapture } from "./source-sdk-evidence.js"; import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; import path from "node:path"; @@ -23,6 +24,7 @@ export async function copySharedEvidence(privateRoot: string, casePrivateDir: st if (error.code !== "ENOENT") throw error; }); } + if (sdkAudit) await copyPrivateFrameworkReport(privateRoot, casePrivateDir, secrets); if (sdkAudit) await (await import("./source-sdk-evidence.js")).copyPreparedSourceSdkAudit(temporaryRoot, casePrivateDir, secrets, sdkAudit); // The supervisor records each phase before it can fail. Copy its receipts // independently of the successful-restart acknowledgement before cleanup.