fix(evals): retain canonical private framework attachments once

tests/runner-e2e/README.md

@@ -634,6 +634,27 @@ matching API-key or subscription credential.
 Never put credentials in `catalog.ts`, screenshots, fixture metadata, workflow
 inputs, or a tracked env file.
 
+## Private framework evidence
+
+An approved private-cell preflight uses Playwright's official JSON and JUnit
+reporters. Other runs keep the existing HTML and blob reporters. The immutable
+JSON source is retained in `snapshots/playwright-framework-original.txt` and is
+parsed and deeply inspected as registered Playwright JSON, including decoded
+inline text/JSON attachment bodies. The text suffix preserves source bytes; it
+does not exempt the file from JSON privacy checks. Unsupported inline carriers
+fail closed.
+
+`snapshots/playwright-framework.json` preserves the report and maps attachment
+paths to their owned canonical files with byte counts and SHA-256 hashes.
+Original status, errors, and nested user steps remain unchanged. Sealed originals
+retain all owned attachments. The final-results tree references those originals
+instead of storing another copy of registered Playwright output attachments.
+The downloader verifies each reference against the sealed original inventory.
+Unknown, missing, changed, or foreign attachments fail validation. Native
+snapshots and other unregistered evidence are not removed. The existing physical
+byte, entry, archive, ZIP expansion, time, and privacy bounds remain unchanged.
+A complete evidence transfer does not attest that the feature passed.
+
 ## Local commands
 
 When the host cannot start embedded Postgres, set

tests/runner-e2e/evidence.ts

@@ -1,3 +1,4 @@
+import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport, inspectPrivateFrameworkMetadata } from "./private-framework-report.js";
 import { spawn } from "node:child_process";
 import {
   mkdir,
@@ -246,6 +247,7 @@ export async function packageEvidence(input: {
       continue;
     } else if (TEXT_EXTENSIONS.has(extension)) {
       const raw = await readFile(source, "utf8");
+      if (input.privateCell) inspectPrivateFrameworkMetadata(relative.split(path.sep).join("/"), raw, input.secrets);
       if (extension === ".html") {
         const issue = await inspectEmbeddedReport(raw, input.secrets);
         if (issue) { if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(issue); leaks.push({ file: relative, reason: issue }); continue; }
@@ -301,10 +303,12 @@ export async function packageEvidence(input: {
   }
 
   if (budget) { budget.inventoryComplete = true; budget.scanFinished = true; }
+  const compact = input.privateCell?.layout === "original" && files.includes(PRIVATE_FRAMEWORK_REPORT)
+    ? await validatePrivateFrameworkReport(input.privateDir) : undefined;
   const missing = input.expectPassScreenshot
     ? [
-        ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required)),
-        ...(files.some(
+        ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required) && !(compact && required === path.join("html-report", "index.html"))),
+        ...(compact || files.some(
           (file) =>
             file.startsWith(`blob-report${path.sep}`) && file.endsWith(".zip"),
         )

tests/runner-e2e/launch.ts

@@ -18,6 +18,7 @@ import { readUpgradeArtifacts, verifyUpgradeArtifact, upgradeArtifactKey } from
 import { recoverWorkerTiming } from "./worker-timing.js";
 import { openEvalOperations, type EvalOperations } from "./operations.js";
 import { readRecoveredFeedback } from "./feedback-recovery.js";
+import { PRIVATE_FRAMEWORK_REPORT, referenceSealedFrameworkAttachments } from "./private-framework-report.js";
 import { readInterruptedAttemptTrace } from "./trace-recovery.js";
 import { EVAL_OBSERVABILITY_SECRET_NAMES, evalBraintrustConfig, finalizeEvalAttemptTrace } from "./braintrust.js";
 import { finalizeEvalAttemptThroughRelay } from "./eval-finalization-relay.js";
@@ -53,7 +54,7 @@ import { packageEvidence, publishRunnerE2EAttemptEvidence } from "./evidence.js"
 import { prepareRunnerE2ECampaignDirectory, resolveRunnerE2EResultsRoot } from "./results-root.js";
 import { privateCellEvidenceOwner } from "./manual-private-report-assets.js";
 import { readManualPrivateHandoff, sealManualResultEvidence, preserveOriginalFrameworkResult, sealStandardOriginalEvidence, type ManualPrivateHandoff } from "./manual-private-handoff.js";
-import { requiresPrivateManualEvidence } from "./manual-private-retention.js";
+import { requiresPrivateManualEvidence, manualCellKind } from "./manual-private-retention.js";
 import { copySharedEvidence } from "./shared-evidence.js";
 import { classifyFailure } from "./failure-classifier.js";
 import { mustPreserveRecoveryState, shouldKeepFailedDiagnostics } from "./cleanup-verification.js";
@@ -941,6 +942,8 @@ async function runAttempt(input: {
         manualDiagnostic!.stage = "seal-original";
         await sealStandardOriginalEvidence({ context: manualPrivateHandoff,
           executionId: candidate.id, source: casePrivateDir, evidence: originalEvidence, secrets: credentials, expectPassScreenshot: originalFeaturePassed });
+        if (evidence.files.includes(PRIVATE_FRAMEWORK_REPORT)) await referenceSealedFrameworkAttachments(uploadDir,
+          path.join(manualPrivateHandoff.root, "originals", manualCellKind(candidate.id)), candidate.id);
       } else {
         result = await publishRunnerE2EAttemptEvidence({
           result,

tests/runner-e2e/manual-private-transfer.ts

@@ -1,3 +1,4 @@
+import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport } from "./private-framework-report.js";
 import { createHash } from "node:crypto";
 import { appendFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, writeFile } from "node:fs/promises";
 import path from "node:path";
@@ -95,6 +96,18 @@ export async function validateSealedInventory(directory: string, inventory: stri
   }
   await visit("");
   if (actual.size !== expected.size) throw new Error("Sealed evidence is incomplete");
+  if (treeKind === "standard") for (const row of receipt.files) {
+    if (row.path === PRIVATE_FRAMEWORK_REPORT || row.path.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`)) {
+      const cell = path.join(directory, row.path.slice(0, -PRIVATE_FRAMEWORK_REPORT.length));
+      if (id !== undefined) await validatePrivateFrameworkReport(cell);
+      else {
+        const metadata = JSON.parse(await readFile(path.join(directory, row.path), "utf8"));
+        const owner = metadata.canonicalExecutionId;
+        if (!context || typeof owner !== "string" || !context.executionIds.includes(owner)) throw new Error("Unbound canonical framework evidence");
+        await validatePrivateFrameworkReport(cell, path.join(context.root, "originals", manualCellKind(owner)), owner);
+      }
+    }
+  }
   return receipt;
 }
 export async function validateSealedOriginals(root: string, id: string, context?: ManualPrivateHandoff) {

tests/runner-e2e/playwright.config.ts

@@ -81,7 +81,11 @@ export default defineConfig({
     stderr: "pipe",
   },
   outputDir: path.join(privateDir, "playwright-output"),
-  reporter: [
+  reporter: process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_PREFLIGHT ? [
+    ["list"],
+    ["json", { outputFile: path.join(privateDir, "framework-report.json") }],
+    ["junit", { outputFile: path.join(privateDir, "junit.xml") }],
+  ] : [
     ["list"],
     ["blob", { outputDir: path.join(privateDir, "blob-report") }],
     ["junit", { outputFile: path.join(privateDir, "junit.xml") }],

tests/runner-e2e/private-framework-report.test.ts

@@ -0,0 +1,140 @@
+import type { ManualPrivateHandoff } from "./manual-private-handoff.js";
+import { afterAll, expect, it } from "vitest";
+import { execFile } from "node:child_process";
+import { createRequire } from "node:module";
+import { cp, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { promisify } from "node:util";
+import { copyPrivateFrameworkReport, inspectPrivateFrameworkMetadata, PRIVATE_FRAMEWORK_ORIGINAL_REPORT, PRIVATE_FRAMEWORK_REPORT, referenceSealedFrameworkAttachments, validatePrivateFrameworkReport } from "./private-framework-report.js";
+const roots: string[] = [];
+afterAll(async () => { for (const root of roots) await rm(root, { recursive: true, force: true }); });
+async function fixture() {
+  const root = await mkdtemp(path.join(await realpath(os.tmpdir()), "framework-json-control-")); roots.push(root);
+  const source = path.join(root, "capture"), cell = path.join(source, "cases/activity.agent-feedback"), output = path.join(source, "playwright-output/owned");
+  await mkdir(cell, { recursive: true }); await mkdir(output, { recursive: true });
+  await writeFile(path.join(output, "trace.zip"), "synthetic canonical trace");
+  const report = { config: { version: "1.62.1" }, errors: [], suites: [{ specs: [{ tests: [{ results: [{ status: "failed", error: { message: "actual failure" }, errors: [{ message: "actual failure" }], stdout: [], stderr: [], steps: [{ title: "owned failed step", error: { message: "actual failure" } }], attachments: [{ name: "trace", contentType: "application/zip", path: path.join(output, "trace.zip") }] }] }] }], suites: [] }] };
+  await writeFile(path.join(source, "framework-report.json"), JSON.stringify(report));
+  await cp(path.join(source, "playwright-output"), path.join(cell, "playwright-output"), { recursive: true });
+  return { root, source, cell, report };
+}
+it("keeps official failed status/errors/steps and binds one canonical attachment across original and result trees", async () => {
+  const f = await fixture(); await writeFile(path.join(f.cell, "playwright-output/unregistered.log"), "unexplained evidence retained"); await copyPrivateFrameworkReport(f.source, f.cell, []);
+  const value = await validatePrivateFrameworkReport(f.cell);
+  expect((value.report.suites as any)[0].specs[0].tests[0].results[0]).toMatchObject({ status: "failed", error: { message: "actual failure" }, steps: [{ title: "owned failed step" }] });
+  const original = path.join(f.root, "original"); await cp(f.cell, original, { recursive: true });
+  await referenceSealedFrameworkAttachments(f.cell, original, "owned");
+  await expect(readFile(path.join(f.cell, "playwright-output/owned/trace.zip"))).rejects.toMatchObject({ code: "ENOENT" });
+  expect(await readFile(path.join(f.cell, "playwright-output/unregistered.log"), "utf8")).toBe("unexplained evidence retained");
+  expect(await readFile(path.join(original, "playwright-output/owned/trace.zip"), "utf8")).toBe("synthetic canonical trace");
+  await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).resolves.toMatchObject({ canonicalExecutionId: "owned" });
+  await expect(validatePrivateFrameworkReport(f.cell, original, "foreign")).rejects.toThrow();
+  await writeFile(path.join(original, "playwright-output/owned/trace.zip"), "tampered");
+  await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow();
+});
+it.each(["empty", "malformed", "foreign", "missing", "secret", "body-secret", "foreign-version", "unknown-status"])("refuses %s framework evidence", async kind => {
+  const f = await fixture();
+  if (kind === "foreign-version") f.report.config.version = "0.0.0";
+  if (kind === "unknown-status") f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.status = "invented";
+  if (kind === "empty") f.report.suites = [];
+  if (kind === "foreign") f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]!.path = path.join(f.root, "outside.zip");
+  if (kind === "missing") await rm(path.join(f.cell, "playwright-output/owned/trace.zip"));
+  if (kind === "secret") f.report.errors = [{ message: "synthetic-private-credential" }] as any;
+  if (kind === "body-secret") (f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0] as any).body = Buffer.from("synthetic-private-credential").toString("base64");
+  await writeFile(path.join(f.source, "framework-report.json"), kind === "malformed" ? "{" : JSON.stringify(f.report));
+  await expect(copyPrivateFrameworkReport(f.source, f.cell, ["synthetic-private-credential"])).rejects.toThrow();
+});
+it("uses the real official Playwright JSON producer with failed errors, user steps, and automatic trace", async () => {
+  const f = await fixture(), tests = path.join(f.root, "tests"); await mkdir(tests);
+  const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test");
+  await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('original failed result',async()=>{await test.step('owned failed step',async()=>expect('actual').toBe('expected'));});`);
+  const config = path.join(tests, "config.cjs");
+  await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, outputDir: path.join(f.source, "playwright-output"), use: { trace: "retain-on-failure" }, reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }]] })};`);
+  await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" }, timeout: 30000 }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); });
+  await rm(path.join(f.cell, "playwright-output"), { recursive: true });
+  await cp(path.join(f.source, "playwright-output"), path.join(f.cell, "playwright-output"), { recursive: true });
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  const retained = await validatePrivateFrameworkReport(f.cell), result = (retained.report.suites as any)[0].specs[0].tests[0].results[0];
+  expect(result.status).toBe("failed"); expect(result.errors.length).toBeGreaterThan(0); expect(result.steps[0].title).toBe("owned failed step");
+  expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true);
+});
+it("keeps the unchanged 256MiB packaging/archive caps with one canonical large attachment", async () => {
+  const { packageEvidence } = await import("./evidence.js");
+  const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js");
+  const { archivePrivateEvidence } = await import("./private-history.js");
+  const { sealStandardOriginalEvidence, sealManualResultEvidence } = await import("./manual-private-handoff.js");
+  const { validateSealedOriginals, validateSealedInventory } = await import("./manual-private-transfer.js");
+  const { MANUAL_PRIVATE_PROTOCOL, manualCellKind } = await import("./manual-private-retention.js");
+  const f = await fixture(), id = "chat-features.runner-opencode.local.activity.agent-feedback";
+  const context: ManualPrivateHandoff = { root: await (await import("node:fs/promises")).realpath(f.root), private: true as const, protocol: MANUAL_PRIVATE_PROTOCOL,
+    runId: "123", attempt: "1", workflowSha: "a".repeat(40), targetSha: "b".repeat(40), executionIds: [id], cellReportProducer: await inspectCellReportProducer() };
+  const zip = path.join(f.root, "input.zip");
+  await promisify(execFile)("python3", ["-c", "import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_STORED);f=z.open('synthetic.trace','w');[f.write(b'0'*1048576) for _ in range(140)];f.close();z.close()", zip], { timeout: 30000 });
+  const tests = path.join(f.root, "budget-tests"); await mkdir(tests);
+  const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test");
+  await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>expect(1).toBe(2));});`);
+  const config = path.join(tests, "config.cjs");
+  await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, use: { trace: "retain-on-failure" }, outputDir: path.join(f.source, "playwright-output"), reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`);
+  await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, timeout: 30000, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" } }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); });
+  await rm(path.join(f.cell, "playwright-output"), { recursive: true });
+  for (const folder of ["playwright-output", "html-report", "blob-report"]) await cp(path.join(f.source, folder), path.join(f.cell, folder), { recursive: true });
+  const owner = await privateCellEvidenceOwner(context, id), prefix = `gha-123-1-${id}/chat-features/runner-opencode/local/activity.agent-feedback/attempt-1`;
+  const upload = path.join(context.root, "results", prefix);
+  const input = { privateDir: f.cell, uploadDir: upload, secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" as const } };
+  await expect(packageEvidence(input)).rejects.toThrow("Private cell evidence exceeds byte allowance");
+  for (const folder of ["html-report", "blob-report"]) await rm(path.join(f.cell, folder), { recursive: true });
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  await writeFile(path.join(f.cell, "result.json"), '{"status":"failed","error":"original failed assertion"}');
+  const evidence = await packageEvidence(input); expect(evidence.leaks).toEqual([]); expect(evidence.budget?.bytesExamined).toBeLessThan(256 * 1024 * 1024);
+  await sealStandardOriginalEvidence({ context, executionId: id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false });
+  const original = path.join(context.root, "originals", manualCellKind(id));
+  await referenceSealedFrameworkAttachments(upload, original, id);
+  await sealManualResultEvidence(context, []);
+  expect((await validateSealedOriginals(context.root, id, context)).complete).toBe(true);
+  expect((await validateSealedInventory(path.join(context.root, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", context)).complete).toBe(true);
+  const bundle = path.join(context.root, "bundle"); await mkdir(bundle);
+  await cp(path.join(context.root, "originals"), path.join(bundle, "originals"), { recursive: true });
+  await cp(path.join(context.root, "sealed-results"), path.join(bundle, "sealed-results"), { recursive: true });
+  const archive = path.join(context.root, "retained.zip"), restored = path.join(context.root, "restored");
+  await archivePrivateEvidence("pack", bundle, archive); await archivePrivateEvidence("unpack", archive, restored);
+  const downloaded = { ...context, root: restored };
+  await expect(validateSealedOriginals(restored, id, downloaded)).resolves.toMatchObject({ complete: true });
+  await expect(validateSealedInventory(path.join(restored, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", downloaded)).resolves.toMatchObject({ complete: true });
+  expect(JSON.parse(await readFile(path.join(restored, "originals", manualCellKind(id), "result.json"), "utf8")).status).toBe("failed");
+}, 30000);
+
+it.each(["literal", "escaped", "base64-body", "escaped-json-body"])("deep-scans registered immutable raw JSON text: %s", async kind => {
+  const f = await fixture(), canary = "synthetic-private-credential";
+  const attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any;
+  if (kind === "literal" || kind === "escaped") f.report.errors = [{ message: canary }] as any;
+  else {
+    attachment.contentType = kind === "base64-body" ? "text/plain" : "application/json";
+    attachment.body = Buffer.from(kind === "base64-body" ? canary : JSON.stringify({ value: canary }).replace("s", "\\u0073")).toString("base64");
+  }
+  let raw = JSON.stringify(f.report);
+  if (kind === "escaped") raw = raw.replace(canary, "\\u0073" + canary.slice(1));
+  expect(() => inspectPrivateFrameworkMetadata(PRIVATE_FRAMEWORK_ORIGINAL_REPORT, raw, [canary])).toThrow();
+});
+it("retains and checks a clean official inline text attachment without losing its body", async () => {
+  const f = await fixture(); const attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any;
+  attachment.contentType = "text/plain"; attachment.body = Buffer.from("safe diagnostic").toString("base64");
+  await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report));
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  const retained = await validatePrivateFrameworkReport(f.cell);
+  expect((retained.report.suites as any)[0].specs[0].tests[0].results[0].attachments[0].body).toBe(attachment.body);
+});
+
+it("rejects a symbolic attachment and changed final framework outcome", async () => {
+  const f = await fixture(), file = path.join(f.cell, "playwright-output/owned/trace.zip");
+  await rm(file); await symlink(path.join(f.source, "playwright-output/owned/trace.zip"), file);
+  await expect(copyPrivateFrameworkReport(f.source, f.cell, [])).rejects.toThrow();
+  await rm(file); await cp(path.join(f.source, "playwright-output/owned/trace.zip"), file);
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  const original = path.join(f.root, "original"); await cp(f.cell, original, { recursive: true });
+  await referenceSealedFrameworkAttachments(f.cell, original, "owned");
+  const sidecar = JSON.parse(await readFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), "utf8"));
+  sidecar.report.suites[0].specs[0].tests[0].results[0].status = "passed";
+  await writeFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), JSON.stringify(sidecar));
+  await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow();
+});

tests/runner-e2e/private-framework-report.ts

@@ -0,0 +1,154 @@
+import { createRequire } from "node:module";
+import { createHash } from "node:crypto";
+import { lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets } from "./redaction.js";
+
+export const PRIVATE_FRAMEWORK_ORIGINAL_REPORT = "snapshots/playwright-framework-original.txt";
+export const PRIVATE_FRAMEWORK_REPORT = "snapshots/playwright-framework.json";
+const producerVersion: string = createRequire(import.meta.url)("@playwright/test/package.json").version;
+const LIMIT = 256 * 1024 * 1024;
+const hash = (data: Buffer) => createHash("sha256").update(data).digest("hex");
+const fail = (): never => { throw new Error("Invalid private framework evidence"); };
+type Row = { path: string; bytes: number; sha256: string };
+type FrameworkReport = { schema: string; sourceReportSha256: string; report: Record<string, unknown>; attachments: Row[]; canonicalExecutionId?: string };
+async function regular(root: string, relative: string, limit = LIMIT) {
+  if (!relative || relative.includes("\\") || relative.split("/").some(p => !p || p === "." || p === "..") || /[\x00-\x1f:]/.test(relative)) fail();
+  let current = root;
+  for (const part of relative.split("/")) { current = path.join(current, part); const entry = await lstat(current); if (entry.isSymbolicLink()) fail(); }
+  const info = await lstat(current);
+  if (!info.isFile() || info.size > limit || !(await realpath(current)).startsWith(`${await realpath(root)}${path.sep}`)) fail();
+  const data = await readFile(current), after = await lstat(current);
+  if (data.length !== info.size || after.ino !== info.ino || after.mtimeMs !== info.mtimeMs || after.ctimeMs !== info.ctimeMs) fail();
+  return data;
+}
+function results(report: Record<string, unknown>) {
+  if (!report.config || typeof report.config !== "object" || !Array.isArray(report.suites) || !report.suites.length || !Array.isArray(report.errors)) fail();
+  if ((report.config as Record<string, unknown>).version !== producerVersion) fail();
+  const found: Array<Record<string, unknown>> = []; let visited = 0;
+  const suite = (value: any) => {
+    if (++visited > 4096 || !value || (value.specs !== undefined && !Array.isArray(value.specs)) || (value.suites !== undefined && !Array.isArray(value.suites))) fail();
+    for (const spec of value.specs ?? []) {
+      if (!spec || !Array.isArray(spec.tests) || !spec.tests.length) fail();
+      for (const test of spec.tests) {
+        if (!test || !Array.isArray(test.results) || !test.results.length) fail();
+        for (const result of test.results) {
+          if (++visited > 4096 || !result || !["passed", "failed", "timedOut", "skipped", "interrupted"].includes(result.status)
+            || !Array.isArray(result.attachments) || !Array.isArray(result.errors) || !Array.isArray(result.stdout) || !Array.isArray(result.stderr)) fail();
+          found.push(result);
+        }
+      }
+    }
+    for (const child of value.suites ?? []) suite(child);
+  };
+  for (const value of report.suites as unknown[]) suite(value);
+  if (!found.length) fail();
+  return found;
+}
+export function inspectPrivateFrameworkMetadata(relative: string, raw: string, secrets: readonly string[]) {
+  const original = relative === PRIVATE_FRAMEWORK_ORIGINAL_REPORT || relative.endsWith(`/${PRIVATE_FRAMEWORK_ORIGINAL_REPORT}`) || relative === "framework-report.json";
+  const sidecar = relative === PRIVATE_FRAMEWORK_REPORT || relative.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`);
+  if (!original && !sidecar) return false;
+  if (Buffer.byteLength(raw) > 128 * 1024 * 1024) fail();
+  const parsed = JSON.parse(raw), report = original ? parsed : parsed.report;
+  const expanded = normalizedSecrets(secrets).flatMap(v => [v, encodeURIComponent(v), Buffer.from(v).toString("base64"), Buffer.from(v).toString("base64url")]);
+  if (findSecretLeakInJsonValues(report, expanded)) fail();
+  for (const result of results(report)) for (const attachment of result.attachments as any[]) {
+    if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail();
+    if (attachment.body !== undefined) {
+      if (typeof attachment.body !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.body)) fail();
+      const body = Buffer.from(attachment.body, "base64"), text = body.toString("utf8");
+      if (!["text/plain", "application/json"].includes(attachment.contentType) || !Buffer.from(text).equals(body) || findSecretLeak(body, expanded)) fail();
+      if (attachment.contentType === "application/json" && findSecretLeak(JSON.stringify(JSON.parse(text)), expanded)) fail();
+    }
+  }
+  return true;
+}
+export async function copyPrivateFrameworkReport(source: string, destination: string, secrets: readonly string[]) {
+  const raw = await regular(source, "framework-report.json", 128 * 1024 * 1024);
+  assertSecretFree(raw.toString("utf8"), secrets, "private framework metadata");
+  const report = JSON.parse(raw.toString("utf8"));
+  const attachments = new Map<string, Row>();
+  inspectPrivateFrameworkMetadata("framework-report.json", raw.toString("utf8"), secrets);
+  for (const result of results(report)) for (const attachment of result.attachments as any[]) {
+    if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail();
+    if (attachment.path === undefined) { if (attachment.body === undefined) fail(); continue; }
+    if (typeof attachment.path !== "string" || !path.isAbsolute(attachment.path)) fail();
+    const absolute = path.resolve(attachment.path), output = path.join(source, "playwright-output");
+    let relative: string;
+    if (absolute.startsWith(`${path.resolve(destination)}${path.sep}`)) relative = path.relative(destination, absolute);
+    else if (absolute.startsWith(`${path.resolve(output)}${path.sep}`)) relative = path.relative(source, absolute);
+    else fail();
+    relative = relative!.split(path.sep).join("/");
+    const data = await regular(absolute.startsWith(`${path.resolve(destination)}${path.sep}`) ? destination : source, relative);
+    if (relative === "result.json") {
+      relative = "snapshots/playwright-attached-result.json";
+      await mkdir(path.dirname(path.join(destination, relative)), { recursive: true, mode: 0o700 });
+      await writeFile(path.join(destination, relative), data, { flag: "wx", mode: 0o600 });
+    } else {
+      const copied = await regular(destination, relative);
+      if (!copied.equals(data)) fail();
+    }
+    attachment.path = relative;
+    const row = { path: relative, bytes: data.length, sha256: hash(data) };
+    const previous = attachments.get(relative); if (previous && JSON.stringify(previous) !== JSON.stringify(row)) fail();
+    attachments.set(relative, row);
+  }
+  const sidecar: FrameworkReport = { schema: "paperclip.private-framework-json/v1", sourceReportSha256: hash(raw), report,
+    attachments: [...attachments.values()].sort((a, b) => a.path.localeCompare(b.path)) };
+  await mkdir(path.join(destination, "snapshots"), { recursive: true, mode: 0o700 });
+  await writeFile(path.join(destination, PRIVATE_FRAMEWORK_ORIGINAL_REPORT), raw, { flag: "wx", mode: 0o600 });
+  await writeFile(path.join(destination, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(sidecar)}\n`, { flag: "wx", mode: 0o600 });
+}
+export async function validatePrivateFrameworkReport(directory: string, canonical = directory, executionId?: string) {
+  const raw = await regular(directory, PRIVATE_FRAMEWORK_REPORT, 128 * 1024 * 1024);
+  const value = JSON.parse(raw.toString("utf8")) as FrameworkReport;
+  const keys = Object.keys(value).sort().join();
+  if (keys !== (executionId ? "attachments,canonicalExecutionId,report,schema,sourceReportSha256" : "attachments,report,schema,sourceReportSha256")
+    || value.schema !== "paperclip.private-framework-json/v1" || !/^[0-9a-f]{64}$/.test(value.sourceReportSha256)
+    || executionId && value.canonicalExecutionId !== executionId || !Array.isArray(value.attachments) || value.attachments.length > 4096) fail();
+  if (executionId) {
+    const original = await validatePrivateFrameworkReport(canonical);
+    const { canonicalExecutionId: _, ...retained } = value;
+    if (JSON.stringify(retained) !== JSON.stringify(original)) fail();
+  }
+  const originalRaw = await regular(directory, PRIVATE_FRAMEWORK_ORIGINAL_REPORT, 128 * 1024 * 1024);
+  if (hash(originalRaw) !== value.sourceReportSha256) fail();
+  inspectPrivateFrameworkMetadata(PRIVATE_FRAMEWORK_ORIGINAL_REPORT, originalRaw.toString("utf8"), []);
+  const authoritative = JSON.parse(originalRaw.toString("utf8"));
+  const originalResults = results(authoritative), retainedResults = results(value.report);
+  if (originalResults.length !== retainedResults.length) fail();
+  for (let i = 0; i < originalResults.length; i++) {
+    const originalAttachments = originalResults[i]!.attachments as any[], retainedAttachments = retainedResults[i]!.attachments as any[];
+    if (originalAttachments.length !== retainedAttachments.length) fail();
+    for (let j = 0; j < originalAttachments.length; j++) {
+      if (originalAttachments[j].path !== undefined) originalAttachments[j].path = retainedAttachments[j].path;
+    }
+  }
+  if (JSON.stringify(authoritative) !== JSON.stringify(value.report)) fail();
+  const referenced = new Set<string>();
+  for (const result of results(value.report)) for (const attachment of result.attachments as any[]) {
+    if (attachment.path !== undefined) { if (typeof attachment.path !== "string") fail(); referenced.add(attachment.path); }
+  }
+  const seen = new Set<string>();
+  for (const row of value.attachments) {
+    if (!row || Object.keys(row).sort().join() !== "bytes,path,sha256" || !referenced.has(row.path) || seen.has(row.path)
+      || !Number.isSafeInteger(row.bytes) || row.bytes < 0 || !/^[0-9a-f]{64}$/.test(row.sha256)) fail();
+    const data = await regular(canonical, row.path); if (data.length !== row.bytes || hash(data) !== row.sha256) fail();
+    seen.add(row.path);
+  }
+  if (seen.size !== referenced.size) fail();
+  return value;
+}
+export async function referenceSealedFrameworkAttachments(directory: string, original: string, executionId: string) {
+  const value = await validatePrivateFrameworkReport(directory);
+  const sealed = await validatePrivateFrameworkReport(original);
+  if (JSON.stringify(value) !== JSON.stringify(sealed)) fail();
+  for (const row of value.attachments.filter(r => r.path.startsWith("playwright-output/"))) {
+    const current = await regular(directory, row.path), canonical = await regular(original, row.path);
+    if (!current.equals(canonical)) fail();
+    await rm(path.join(directory, row.path));
+  }
+  value.canonicalExecutionId = executionId;
+  await writeFile(path.join(directory, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(value)}\n`, { mode: 0o600 });
+}

tests/runner-e2e/private-reporter-artifacts.ts

@@ -1,3 +1,4 @@
+import { inspectPrivateFrameworkMetadata } from "./private-framework-report.js";
 import { lstat, readFile, readdir, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { inspectEmbeddedReport, inspectZip } from "./evidence.js";
@@ -5,7 +6,7 @@ import { diagnosticSecrets, type DiagnosticSecretExpansion } from "./private-dia
 import { findSecretLeak, redactText, sanitizeJson } from "./redaction.js";
 import { cellTraceAsset, readPrivateCellTraceAsset, type PrivateCellEvidenceOwner } from "./manual-private-report-assets.js";
 export interface ReporterPrivacyIssue { file: string; disposition: "redacted" | "omitted"; reason: string }
-const sources = ["server.log", "playwright.log", "junit.xml", "html-report", "blob-report", "playwright-output"];
+const sources = ["server.log", "playwright.log", "junit.xml", "html-report", "blob-report", "playwright-output", "framework-report.json"];
 const text = new Set([".log", ".txt", ".md", ".xml", ".html", ".json", ".js", ".css"]);
 /** Only reporter/diagnostic sinks are touched, after independent product oracles
  * finish. Product API snapshots and work-product bytes are never pre-sanitized.
@@ -36,6 +37,7 @@ export async function sanitizePrivateReporterArtifacts(root: string, values: rea
     } else if (text.has(extension)) {
       if (info.size > 128 * 1024 * 1024) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter text exceeds inspection bound" }); return; }
       const raw = await readFile(filename, "utf8");
+      if (relative === "framework-report.json") inspectPrivateFrameworkMetadata(relative, raw, secrets);
       const opaque = extension === ".html" ? await inspectEmbeddedReport(raw, secrets) : null;
       if (opaque) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: opaque }); return; }
       const leak = findSecretLeak(raw, secrets);

tests/runner-e2e/shared-evidence.ts

@@ -1,3 +1,4 @@
+import { copyPrivateFrameworkReport } from "./private-framework-report.js";
 import type { SourceSdkAuditCapture } from "./source-sdk-evidence.js";
 import { cp, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
 import path from "node:path";
@@ -23,6 +24,7 @@ export async function copySharedEvidence(privateRoot: string, casePrivateDir: st
       if (error.code !== "ENOENT") throw error;
     });
   }
+  if (sdkAudit) await copyPrivateFrameworkReport(privateRoot, casePrivateDir, secrets);
   if (sdkAudit) await (await import("./source-sdk-evidence.js")).copyPreparedSourceSdkAudit(temporaryRoot, casePrivateDir, secrets, sdkAudit);
   // The supervisor records each phase before it can fail. Copy its receipts
   // independently of the successful-restart acknowledgement before cleanup.