fix(evals): preserve private report bytes and restore derived attachments

tests/runner-e2e/README.md

@@ -649,7 +649,10 @@ paths to their owned canonical files with byte counts and SHA-256 hashes.
 Original status, errors, and nested user steps remain unchanged. Sealed originals
 retain all owned attachments. The final-results tree references those originals
 instead of storing another copy of registered Playwright output attachments.
-The downloader verifies each reference against the sealed original inventory.
+The downloader verifies each reference against the sealed original inventory,
+then restores exact referenced bytes into the derived selected report tree.
+Neither sealed inventory changes. Registered metadata and attachments are not
+silently rewritten: JSON values that need privacy redaction fail closed.
 Unknown, missing, changed, or foreign attachments fail validation. Native
 snapshots and other unregistered evidence are not removed. The existing physical
 byte, entry, archive, ZIP expansion, time, and privacy bounds remain unchanged.

tests/runner-e2e/evidence.ts

@@ -215,6 +215,9 @@ export async function packageEvidence(input: {
     byKind: Object.fromEntries(evidenceBudgetKinds.map(kind => [kind, { files: 0, bytes: 0 }])) } : undefined;
   if (input.privateCell && standard.length > 4096) throw budgetFailure("Private cell evidence exceeds entry allowance", budget!, "entries");
   const privateSecrets = input.privateCell ? normalizedSecrets(input.secrets).flatMap(value => [value, encodeURIComponent(value), Buffer.from(value).toString("base64"), Buffer.from(value).toString("base64url")]) : [];
+  const compact = input.privateCell?.layout === "original" && standard.includes(PRIVATE_FRAMEWORK_REPORT)
+    ? await validatePrivateFrameworkReport(input.privateDir) : undefined;
+  const registeredAttachments = new Set(compact?.attachments.map(row => row.path) ?? []);
   let privateBytes = 0;
   for (const relative of standard) {
     const source = path.join(input.privateDir, relative);
@@ -247,7 +250,10 @@ export async function packageEvidence(input: {
       continue;
     } else if (TEXT_EXTENSIONS.has(extension)) {
       const raw = await readFile(source, "utf8");
-      if (input.privateCell) inspectPrivateFrameworkMetadata(relative.split(path.sep).join("/"), raw, input.secrets);
+      if (input.privateCell && inspectPrivateFrameworkMetadata(relative.split(path.sep).join("/"), raw, input.secrets)) {
+        assertSecretFree(raw, input.secrets, "private framework metadata");
+        await copyFile(source, destination); files.push(relative); continue;
+      }
       if (extension === ".html") {
         const issue = await inspectEmbeddedReport(raw, input.secrets);
         if (issue) { if (budget && budget.bound === "none") budget.bound = scannerBudgetBound(issue); leaks.push({ file: relative, reason: issue }); continue; }
@@ -258,6 +264,11 @@ export async function packageEvidence(input: {
           ? findSecretLeakInJsonValues(parsed, input.secrets)
           : findSecretLeak(raw, input.secrets);
       if (leak) leaks.push({ file: relative, reason: leak });
+      if (registeredAttachments.has(relative.split(path.sep).join("/"))) {
+        if (JSON.stringify(sanitizeJson(parsed ?? raw, input.secrets)) !== JSON.stringify(parsed ?? raw)) throw new Error("Private framework attachment requires redaction");
+        if (!leak) { await copyFile(source, destination); files.push(relative); }
+        continue;
+      }
       // Redacting an already serialized JSON string can change escape
       // boundaries around shell commands. Sanitize parsed values instead so
       // uploaded snapshots stay valid JSON.
@@ -303,8 +314,6 @@ export async function packageEvidence(input: {
   }
 
   if (budget) { budget.inventoryComplete = true; budget.scanFinished = true; }
-  const compact = input.privateCell?.layout === "original" && files.includes(PRIVATE_FRAMEWORK_REPORT)
-    ? await validatePrivateFrameworkReport(input.privateDir) : undefined;
   const missing = input.expectPassScreenshot
     ? [
         ...REQUIRED_PASS_FILES.filter((required) => !files.includes(required) && !(compact && required === path.join("html-report", "index.html"))),

tests/runner-e2e/manual-private-transfer.ts

@@ -1,4 +1,4 @@
-import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport } from "./private-framework-report.js";
+import { PRIVATE_FRAMEWORK_REPORT, validatePrivateFrameworkReport, materializePrivateFrameworkAttachments } from "./private-framework-report.js";
 import { createHash } from "node:crypto";
 import { appendFile, cp, lstat, mkdir, mkdtemp, readFile, readdir, realpath, writeFile } from "node:fs/promises";
 import path from "node:path";
@@ -171,11 +171,22 @@ async function downloadCell(env: NodeJS.ProcessEnv) {
     const originals = await validateSealedOriginals(directory, id, { ...identity, root: directory }), summary = cellCompleteness(id, originals);
     if (hash(JSON.stringify(identity.evidenceCompleteness)) !== hash(JSON.stringify(summary))
       || hash(JSON.stringify(receipt.evidenceCompleteness)) !== hash(JSON.stringify(summary))) throw new Error("Private evidence completeness receipt differs from archive");
-    await validateSealedInventory(path.join(directory, "sealed-results", "html-report"), "../result-evidence-inventory.json", undefined, "standard", { ...identity, root: directory });
-    await cp(path.join(directory, "sealed-results", "html-report"), path.join(selected, manualCellKind(id)), { recursive: true, errorOnExist: true, force: false });
+    await stageSealedPrivateCell(directory, selected, id, { ...identity, root: directory });
     await writeFile(path.join(root, `manual-cell-completeness-${index}.json`), JSON.stringify(summary), { mode: 0o600, flag: "wx" });
   }
 }
+export async function stageSealedPrivateCell(directory: string, selected: string, id: string, context: ManualPrivateHandoff) {
+  await validateSealedOriginals(directory, id, context);
+  const inventory = await validateSealedInventory(path.join(directory, "sealed-results", "html-report"), "../result-evidence-inventory.json", undefined, "standard", context);
+  const destination = path.join(selected, manualCellKind(id));
+  await cp(path.join(directory, "sealed-results", "html-report"), destination, { recursive: true, errorOnExist: true, force: false });
+  for (const row of inventory.files) {
+    if (row.path === PRIVATE_FRAMEWORK_REPORT || row.path.endsWith(`/${PRIVATE_FRAMEWORK_REPORT}`)) {
+      const cell = path.join(destination, row.path.slice(0, -PRIVATE_FRAMEWORK_REPORT.length));
+      await materializePrivateFrameworkAttachments(cell, path.join(directory, "originals", manualCellKind(id)), id);
+    }
+  }
+}
 async function verifyDownloads(env: NodeJS.ProcessEnv) {
   const { approval } = request(env), root = await realpath(env.PRIVATE_REPORT_ROOT ?? "");
   const cells: PrivateEvidenceCompleteness["cells"] = [];

tests/runner-e2e/private-framework-report.test.ts

@@ -1,5 +1,6 @@
 import type { ManualPrivateHandoff } from "./manual-private-handoff.js";
 import { afterAll, expect, it } from "vitest";
+import { createHash } from "node:crypto";
 import { execFile } from "node:child_process";
 import { createRequire } from "node:module";
 import { cp, mkdtemp, mkdir, readFile, realpath, rm, symlink, writeFile } from "node:fs/promises";
@@ -54,7 +55,12 @@ it("uses the real official Playwright JSON producer with failed errors, user ste
   await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" }, timeout: 30000 }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); });
   await rm(path.join(f.cell, "playwright-output"), { recursive: true });
   await cp(path.join(f.source, "playwright-output"), path.join(f.cell, "playwright-output"), { recursive: true });
+  const { sanitizePrivateReporterArtifacts } = await import("./private-reporter-artifacts.js");
+  const produced = await readFile(path.join(f.source, "framework-report.json"));
+  expect(await sanitizePrivateReporterArtifacts(f.source, [])).toEqual([]);
+  expect(await readFile(path.join(f.source, "framework-report.json"))).toEqual(produced);
   await copyPrivateFrameworkReport(f.source, f.cell, []);
+  expect(await readFile(path.join(f.cell, "snapshots/playwright-framework-original.txt"))).toEqual(produced);
   const retained = await validatePrivateFrameworkReport(f.cell), result = (retained.report.suites as any)[0].specs[0].tests[0].results[0];
   expect(result.status).toBe("failed"); expect(result.errors.length).toBeGreaterThan(0); expect(result.steps[0].title).toBe("owned failed step");
   expect(retained.attachments.some(a => a.path.endsWith("/trace.zip"))).toBe(true);
@@ -64,7 +70,7 @@ it("keeps the unchanged 256MiB packaging/archive caps with one canonical large a
   const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js");
   const { archivePrivateEvidence } = await import("./private-history.js");
   const { sealStandardOriginalEvidence, sealManualResultEvidence } = await import("./manual-private-handoff.js");
-  const { validateSealedOriginals, validateSealedInventory } = await import("./manual-private-transfer.js");
+  const { validateSealedOriginals, validateSealedInventory, stageSealedPrivateCell } = await import("./manual-private-transfer.js");
   const { MANUAL_PRIVATE_PROTOCOL, manualCellKind } = await import("./manual-private-retention.js");
   const f = await fixture(), id = "chat-features.runner-opencode.local.activity.agent-feedback";
   const context: ManualPrivateHandoff = { root: await (await import("node:fs/promises")).realpath(f.root), private: true as const, protocol: MANUAL_PRIVATE_PROTOCOL,
@@ -73,7 +79,7 @@ it("keeps the unchanged 256MiB packaging/archive caps with one canonical large a
   await promisify(execFile)("python3", ["-c", "import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_STORED);f=z.open('synthetic.trace','w');[f.write(b'0'*1048576) for _ in range(140)];f.close();z.close()", zip], { timeout: 30000 });
   const tests = path.join(f.root, "budget-tests"); await mkdir(tests);
   const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test");
-  await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>expect(1).toBe(2));});`);
+  await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>{throw new Error('The --token option requires a value');});});`);
   const config = path.join(tests, "config.cjs");
   await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, use: { trace: "retain-on-failure" }, outputDir: path.join(f.source, "playwright-output"), reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`);
   await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, timeout: 30000, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" } }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); });
@@ -86,7 +92,10 @@ it("keeps the unchanged 256MiB packaging/archive caps with one canonical large a
   for (const folder of ["html-report", "blob-report"]) await rm(path.join(f.cell, folder), { recursive: true });
   await copyPrivateFrameworkReport(f.source, f.cell, []);
   await writeFile(path.join(f.cell, "result.json"), '{"status":"failed","error":"original failed assertion"}');
-  const evidence = await packageEvidence(input); expect(evidence.leaks).toEqual([]); expect(evidence.budget?.bytesExamined).toBeLessThan(256 * 1024 * 1024);
+  const rawFramework = await readFile(path.join(f.cell, "snapshots/playwright-framework-original.txt"));
+  const { redactText } = await import("./redaction.js"); expect(redactText(rawFramework.toString("utf8"), [])).not.toBe(rawFramework.toString("utf8"));
+  const evidence = await packageEvidence(input); expect(evidence.leaks).toEqual([]);
+  expect(await readFile(path.join(upload, "snapshots/playwright-framework-original.txt"))).toEqual(rawFramework); expect(evidence.budget?.bytesExamined).toBeLessThan(256 * 1024 * 1024);
   await sealStandardOriginalEvidence({ context, executionId: id, source: f.cell, evidence, secrets: [], expectPassScreenshot: false });
   const original = path.join(context.root, "originals", manualCellKind(id));
   await referenceSealedFrameworkAttachments(upload, original, id);
@@ -102,6 +111,14 @@ it("keeps the unchanged 256MiB packaging/archive caps with one canonical large a
   await expect(validateSealedOriginals(restored, id, downloaded)).resolves.toMatchObject({ complete: true });
   await expect(validateSealedInventory(path.join(restored, "sealed-results/html-report"), "../result-evidence-inventory.json", undefined, "standard", downloaded)).resolves.toMatchObject({ complete: true });
   expect(JSON.parse(await readFile(path.join(restored, "originals", manualCellKind(id), "result.json"), "utf8")).status).toBe("failed");
+  const selected = path.join(f.root, "selected-runner-e2e"); await mkdir(selected);
+  const sealedInventory = await readFile(path.join(restored, "sealed-results/result-evidence-inventory.json"));
+  await stageSealedPrivateCell(restored, selected, id, downloaded);
+  const derived = path.join(selected, manualCellKind(id), prefix);
+  for (const attachment of (await validatePrivateFrameworkReport(derived, path.join(restored, "originals", manualCellKind(id)), id)).attachments) {
+    expect(createHash("sha256").update(await readFile(path.join(derived, attachment.path))).digest("hex")).toBe(attachment.sha256);
+  }
+  expect(await readFile(path.join(restored, "sealed-results/result-evidence-inventory.json"))).toEqual(sealedInventory);
 }, 30000);
 
 it.each(["literal", "escaped", "base64-body", "escaped-json-body"])("deep-scans registered immutable raw JSON text: %s", async kind => {
@@ -138,3 +155,32 @@ it("rejects a symbolic attachment and changed final framework outcome", async ()
   await writeFile(path.join(f.cell, PRIVATE_FRAMEWORK_REPORT), JSON.stringify(sidecar));
   await expect(validatePrivateFrameworkReport(f.cell, original, "owned")).rejects.toThrow();
 });
+
+it("preserves exact registered official JSON whitespace through sanitizer and copier", async () => {
+  const f = await fixture(); const raw = Buffer.from("\n" + JSON.stringify(f.report) + "\n\n");
+  const trace = path.join(f.source, "playwright-output/owned/trace.zip");
+  const traceBytes = await readFile(trace); await rm(trace);
+  await writeFile(path.join(f.source, "framework-report.json"), raw);
+  const { sanitizePrivateReporterArtifacts } = await import("./private-reporter-artifacts.js");
+  expect(await sanitizePrivateReporterArtifacts(f.source, [])).toEqual([]);
+  expect(await readFile(path.join(f.source, "framework-report.json"))).toEqual(raw);
+  await writeFile(trace, traceBytes);
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  expect(await readFile(path.join(f.cell, "snapshots/playwright-framework-original.txt"))).toEqual(raw);
+});
+
+it("rejects sensitive JSON keys in registered metadata and JSON attachments without rewriting them", async () => {
+  const f = await fixture(); (f.report.config as any).metadata = { password: "unrecognized-private-value" };
+  await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report));
+  await expect(copyPrivateFrameworkReport(f.source, f.cell, [])).rejects.toThrow();
+  delete (f.report.config as any).metadata;
+  const file = path.join(f.cell, "snapshots/owned.json"); await mkdir(path.dirname(file), { recursive: true }); await writeFile(file, '{"password":"unrecognized-private-value"}');
+  f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]!.path = file;
+  await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report));
+  await copyPrivateFrameworkReport(f.source, f.cell, []);
+  const { packageEvidence } = await import("./evidence.js");
+  const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js");
+  const owner = await privateCellEvidenceOwner({ root: f.root, private: true, runId: "123", attempt: "1", executionIds: ["chat-features.runner-opencode.local.activity.agent-feedback"], cellReportProducer: await inspectCellReportProducer() }, "chat-features.runner-opencode.local.activity.agent-feedback");
+  await expect(packageEvidence({ privateDir: f.cell, uploadDir: path.join(f.root, "upload"), secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } })).rejects.toThrow("requires redaction");
+  expect(await readFile(file, "utf8")).toBe('{"password":"unrecognized-private-value"}');
+});

tests/runner-e2e/private-framework-report.ts

@@ -1,8 +1,9 @@
+import { constants } from "node:fs";
 import { createRequire } from "node:module";
 import { createHash } from "node:crypto";
-import { lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import { copyFile, lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
 import path from "node:path";
-import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets } from "./redaction.js";
+import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets, sanitizeJson } from "./redaction.js";
 
 export const PRIVATE_FRAMEWORK_ORIGINAL_REPORT = "snapshots/playwright-framework-original.txt";
 export const PRIVATE_FRAMEWORK_REPORT = "snapshots/playwright-framework.json";
@@ -52,20 +53,21 @@ export function inspectPrivateFrameworkMetadata(relative: string, raw: string, s
   if (Buffer.byteLength(raw) > 128 * 1024 * 1024) fail();
   const parsed = JSON.parse(raw), report = original ? parsed : parsed.report;
   const expanded = normalizedSecrets(secrets).flatMap(v => [v, encodeURIComponent(v), Buffer.from(v).toString("base64"), Buffer.from(v).toString("base64url")]);
-  if (findSecretLeakInJsonValues(report, expanded)) fail();
+  if (findSecretLeakInJsonValues(parsed, expanded) || JSON.stringify(sanitizeJson(parsed, secrets)) !== JSON.stringify(parsed)) fail();
   for (const result of results(report)) for (const attachment of result.attachments as any[]) {
     if (!attachment || typeof attachment.name !== "string" || typeof attachment.contentType !== "string") fail();
     if (attachment.body !== undefined) {
       if (typeof attachment.body !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.body)) fail();
       const body = Buffer.from(attachment.body, "base64"), text = body.toString("utf8");
       if (!["text/plain", "application/json"].includes(attachment.contentType) || !Buffer.from(text).equals(body) || findSecretLeak(body, expanded)) fail();
-      if (attachment.contentType === "application/json" && findSecretLeak(JSON.stringify(JSON.parse(text)), expanded)) fail();
+      if (attachment.contentType === "application/json" && (findSecretLeak(JSON.stringify(JSON.parse(text)), expanded) || JSON.stringify(sanitizeJson(JSON.parse(text), secrets)) !== JSON.stringify(JSON.parse(text)))) fail();
     }
   }
   return true;
 }
 export async function copyPrivateFrameworkReport(source: string, destination: string, secrets: readonly string[]) {
   const raw = await regular(source, "framework-report.json", 128 * 1024 * 1024);
+  if (!Buffer.from(raw.toString("utf8")).equals(raw)) fail();
   assertSecretFree(raw.toString("utf8"), secrets, "private framework metadata");
   const report = JSON.parse(raw.toString("utf8"));
   const attachments = new Map<string, Row>();
@@ -130,10 +132,11 @@ export async function validatePrivateFrameworkReport(directory: string, canonica
   for (const result of results(value.report)) for (const attachment of result.attachments as any[]) {
     if (attachment.path !== undefined) { if (typeof attachment.path !== "string") fail(); referenced.add(attachment.path); }
   }
-  const seen = new Set<string>();
+  const seen = new Set<string>(); let attachmentBytes = 0;
   for (const row of value.attachments) {
     if (!row || Object.keys(row).sort().join() !== "bytes,path,sha256" || !referenced.has(row.path) || seen.has(row.path)
       || !Number.isSafeInteger(row.bytes) || row.bytes < 0 || !/^[0-9a-f]{64}$/.test(row.sha256)) fail();
+    attachmentBytes += row.bytes; if (attachmentBytes > LIMIT) fail();
     const data = await regular(canonical, row.path); if (data.length !== row.bytes || hash(data) !== row.sha256) fail();
     seen.add(row.path);
   }
@@ -152,3 +155,18 @@ export async function referenceSealedFrameworkAttachments(directory: string, ori
   value.canonicalExecutionId = executionId;
   await writeFile(path.join(directory, PRIVATE_FRAMEWORK_REPORT), `${JSON.stringify(value)}\n`, { mode: 0o600 });
 }
+
+export async function materializePrivateFrameworkAttachments(directory: string, original: string, executionId: string) {
+  const value = await validatePrivateFrameworkReport(directory, original, executionId);
+  let bytes = 0;
+  for (const row of value.attachments) {
+    bytes += row.bytes; if (bytes > LIMIT) fail();
+    if (!row.path.startsWith("playwright-output/")) continue;
+    const source = await regular(original, row.path);
+    if (source.length !== row.bytes || hash(source) !== row.sha256) fail();
+    await mkdir(path.dirname(path.join(directory, row.path)), { recursive: true, mode: 0o700 });
+    await copyFile(path.join(original, row.path), path.join(directory, row.path), constants.COPYFILE_EXCL);
+    const restored = await regular(directory, row.path);
+    if (hash(restored) !== row.sha256) fail();
+  }
+}

tests/runner-e2e/private-reporter-artifacts.ts

@@ -37,7 +37,7 @@ export async function sanitizePrivateReporterArtifacts(root: string, values: rea
     } else if (text.has(extension)) {
       if (info.size > 128 * 1024 * 1024) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter text exceeds inspection bound" }); return; }
       const raw = await readFile(filename, "utf8");
-      if (relative === "framework-report.json") inspectPrivateFrameworkMetadata(relative, raw, secrets);
+      if (relative === "framework-report.json") { inspectPrivateFrameworkMetadata(relative, raw, secrets); return; }
       const opaque = extension === ".html" ? await inspectEmbeddedReport(raw, secrets) : null;
       if (opaque) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: opaque }); return; }
       const leak = findSecretLeak(raw, secrets);