From b72f0f7cbb51d9669badd46fc48025d83e7f3f84 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 21:10:10 -0500 Subject: [PATCH] test(evals): redact private diagnostic streams and opaque reporter sinks --- tests/runner-e2e/evidence.ts | 30 +++++++- tests/runner-e2e/harness-env.ts | 2 + tests/runner-e2e/launch.ts | 55 ++++++++------ tests/runner-e2e/private-diagnostics.test.ts | 37 ++++++++++ tests/runner-e2e/private-diagnostics.ts | 71 +++++++++++++++++++ .../private-reporter-artifacts.test.ts | 42 +++++++++++ .../runner-e2e/private-reporter-artifacts.ts | 39 ++++++++++ tests/runner-e2e/runner.spec.ts | 3 +- 8 files changed, 255 insertions(+), 24 deletions(-) create mode 100644 tests/runner-e2e/private-diagnostics.test.ts create mode 100644 tests/runner-e2e/private-diagnostics.ts create mode 100644 tests/runner-e2e/private-reporter-artifacts.test.ts create mode 100644 tests/runner-e2e/private-reporter-artifacts.ts diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index 259ff499454..93f8683673c 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -6,8 +6,10 @@ import { rm, writeFile, copyFile, + mkdtemp, } from "node:fs/promises"; import path from "node:path"; +import os from "node:os"; import { assertSecretFree, findSecretLeak, @@ -87,7 +89,7 @@ function isAllowed(relative: string) { return ALLOWED_DIRECTORIES.has(segments[0]); } -async function inspectZip(source: string, secrets: readonly string[]) { +export async function inspectZip(source: string, secrets: readonly string[]) { // Failure traces can exceed Node's child-process output buffer. Stream the // expanded archive through the exact-value scanner with enough overlap to // detect a credential split across stdout chunks, without retaining the @@ -103,9 +105,13 @@ async function inspectZip(source: string, secrets: readonly string[]) { let carry = Buffer.alloc(0); let leak: string | null = null; let spawnError: Error | null = null; + let expanded = 0; + const timer = setTimeout(() => { leak = "zip inspection exceeded its time bound"; unzip.kill(); }, 30_000); unzip.stdout.on("data", (chunk: Buffer) => { if (leak) return; + expanded += chunk.length; + if (expanded > 256 * 1024 * 1024) { leak = "zip inspection exceeded its expanded-byte bound"; unzip.kill(); return; } const data = Buffer.concat([carry, chunk]); leak = findSecretLeak(data, secrets, { includeShapes: false }); carry = data.subarray(Math.max(0, data.length - overlap)); @@ -118,6 +124,7 @@ async function inspectZip(source: string, secrets: readonly string[]) { spawnError = error; }); unzip.on("close", (code) => { + clearTimeout(timer); if (leak) return resolve(leak); if (spawnError) { return resolve(`zip could not be inspected: ${spawnError.message}`); @@ -131,6 +138,23 @@ async function inspectZip(source: string, secrets: readonly string[]) { }); } +/** Playwright HTML contains a compressed report, which a plaintext HTML scan + * cannot inspect. It receives the same independent ZIP check as blob reports. */ +export async function inspectEmbeddedReport(raw: string, secrets: readonly string[]) { + const payloads = [...raw.matchAll(/data:application\/zip;base64,([A-Za-z0-9+/=]+)/g)]; + if (!payloads.length) return /playwrightReportBase64/.test(raw) ? "embedded report encoding could not be inspected" : null; + const temporary = await mkdtemp(path.join(os.tmpdir(), "runner-report-scan-")); + try { + for (const [index, payload] of payloads.entries()) { + if (payload[1]!.length > 128 * 1024 * 1024) return "embedded report exceeded its byte bound"; + const filename = path.join(temporary, `${index}.zip`); + await writeFile(filename, Buffer.from(payload[1]!, "base64"), { mode: 0o600 }); + const issue = await inspectZip(filename, secrets); if (issue) return issue; + } + return null; + } finally { await rm(temporary, { recursive: true, force: true }); } +} + export async function packageEvidence(input: { privateDir: string; uploadDir: string; @@ -155,6 +179,10 @@ export async function packageEvidence(input: { continue; } else if (TEXT_EXTENSIONS.has(extension)) { const raw = await readFile(source, "utf8"); + if (extension === ".html") { + const issue = await inspectEmbeddedReport(raw, input.secrets); + if (issue) { leaks.push({ file: relative, reason: issue }); continue; } + } const parsed = extension === ".json" ? JSON.parse(raw) : null; const leak = extension === ".json" diff --git a/tests/runner-e2e/harness-env.ts b/tests/runner-e2e/harness-env.ts index 8a8db216d3d..db0e3b7aae5 100644 --- a/tests/runner-e2e/harness-env.ts +++ b/tests/runner-e2e/harness-env.ts @@ -1,4 +1,5 @@ import { scrubChatPrivateEnvironment } from "./chat-private-inputs.js"; +import { PRIVATE_DIAGNOSTICS_ENV } from "./private-diagnostics.js"; import { installedDaytonaPluginKeys } from "./installed-daytona-plugin.js"; import { QUALIFIED_ACPX_PROFILES } from "../../packages/paperclip-runner/src/drivers/acpx/qualified-profiles.js"; import { assertInstalledCliSelection, installedCliKeys, usesInstalledCli } from "./installed-cli.js"; @@ -198,6 +199,7 @@ export function buildPaperclipServerEnvironment( overrides: NodeJS.ProcessEnv = {}, ): NodeJS.ProcessEnv { const result = scrubChatPrivateEnvironment({ ...source }, [], true); + delete result[PRIVATE_DIAGNOSTICS_ENV]; for (const key of Object.keys(result)) { if (PROVIDER_SECRET_KEY.test(key)) delete result[key]; } diff --git a/tests/runner-e2e/launch.ts b/tests/runner-e2e/launch.ts index 814e311f7e5..56f89e0831d 100644 --- a/tests/runner-e2e/launch.ts +++ b/tests/runner-e2e/launch.ts @@ -1,5 +1,7 @@ import { stagePublishedChatInstall, ChatInstalledCleanupError } from "./chat-installed-lane.js"; import { stageChatPrivateInputs, CHAT_PRIVATE_INPUTS_ENV } from "./chat-private-inputs.js"; +import { createPrivateDiagnosticStream, diagnosticSecrets, PRIVATE_DIAGNOSTICS_ENV, readPrivateDiagnosticSecrets, stagePrivateDiagnostics } from "./private-diagnostics.js"; +import { sanitizePrivateReporterArtifacts } from "./private-reporter-artifacts.js"; import { findChatCatalogCase } from "./chat-catalog-registry.js"; import { recoverWorkerTiming } from "./worker-timing.js"; import { openEvalOperations, type EvalOperations } from "./operations.js"; @@ -284,6 +286,7 @@ async function runProcess( logPath: string, completionPaths: readonly string[], interactive: boolean, + secrets: () => readonly string[], ) { const log = createWriteStream(logPath, { flags: "a", mode: 0o600 }); const child = spawn(invocation.command, invocation.args, { @@ -296,19 +299,17 @@ async function runProcess( activeProcessGroups.add(child.pid); const processOwner = createProcessTreeOwner(child); let outputTail = ""; - const recordOutput = (chunk: Buffer, destination: NodeJS.WriteStream) => { - destination.write(chunk); - log.write(chunk); - outputTail += chunk.toString("utf8"); + const recordOutput = (safe: string, destination: NodeJS.WriteStream) => { + destination.write(safe); + log.write(safe); + outputTail += safe; if (outputTail.length > 1024 * 1024) outputTail = outputTail.slice(-1024 * 1024); }; - child.stdout?.on("data", (chunk: Buffer) => - recordOutput(chunk, process.stdout), - ); - child.stderr?.on("data", (chunk: Buffer) => - recordOutput(chunk, process.stderr), - ); + const stdout = createPrivateDiagnosticStream(secrets, safe => recordOutput(safe, process.stdout)); + const stderr = createPrivateDiagnosticStream(secrets, safe => recordOutput(safe, process.stderr)); + child.stdout?.on("data", (chunk: Buffer) => stdout.write(chunk)); + child.stderr?.on("data", (chunk: Buffer) => stderr.write(chunk)); let childSettled = false; let postResultStallError: string | null = null; let boundedCleanup: Promise | undefined; @@ -336,12 +337,7 @@ async function runProcess( const diagnostic = await processTreeDiagnostic(child.pid!); if (childSettled) return; postResultStallError = `Playwright remained alive for ${completedResultExitGraceMs}ms after every result was written`; - recordOutput( - Buffer.from( - `\n${postResultStallError}; forcing bounded cleanup. ${diagnostic.summary}\n`, - ), - process.stderr, - ); + stderr.write(Buffer.from(`\n${postResultStallError}; forcing bounded cleanup. ${diagnostic.summary}\n`)); stopChildTree(diagnostic); }, }); @@ -351,10 +347,7 @@ async function runProcess( if (childSettled || postResultStallError) return; postResultStallError = "Completed-result exit guard failed during bounded inspection"; - recordOutput( - Buffer.from(`\n${postResultStallError}; forcing cleanup.\n`), - process.stderr, - ); + stderr.write(Buffer.from(`\n${postResultStallError}; forcing cleanup.\n`)); stopChildTree(); }); }, 500) @@ -401,6 +394,7 @@ async function runProcess( activeProcessCleanup.delete(child.pid); activeProcessTerminators.delete(child.pid); } + stdout.end(); stderr.end(); await new Promise((resolve) => log.end(resolve)); return { exitCode, @@ -409,6 +403,8 @@ async function runProcess( processCleanupError, spawnError, outputTail, + privateDiagnosticFailure: stdout.failed() || stderr.failed(), + privateDiagnosticLeak: stdout.redactedSecret() || stderr.redactedSecret(), }; } @@ -562,8 +558,9 @@ async function runAttempt(input: { attemptSecrets = credentials; const needsPrivateInputs = selectedDescriptor && ((selectedDescriptor.requiredCredentials?.length ?? 0) > 0 || selectedDescriptor.externalInput); if (needsPrivateInputs && executions.length !== 1) throw new Error("Private case inputs require one explicitly selected execution"); - if (needsPrivateInputs) await mkdir(selectedPrivateInputsDirectory, { mode: 0o700 }); + await mkdir(selectedPrivateInputsDirectory, { mode: 0o700 }); const privateInputs = needsPrivateInputs ? stageChatPrivateInputs({ execution, descriptor: selectedDescriptor, privateDir: selectedPrivateInputsDirectory, env: process.env, secrets: credentials }) : undefined; + const diagnosticRegistry = stagePrivateDiagnostics(temporaryRoot, executions.map(row => row.id), credentials); if (installedCli) await writeFile(path.join(privateDir, "installed-cli-admission.json"), `${JSON.stringify(installedCli, null, 2)}\n`, { mode: 0o600 }); if (installedPlugin) await writeFile(path.join(privateDir, "installed-daytona-plugin-admission.json"), `${JSON.stringify(installedPlugin, null, 2)}\n`, { mode: 0o600 }); const runnerBinary = installedCli ? undefined : resolvePaperclipRunnerBinaryForHarness( @@ -573,6 +570,7 @@ async function runAttempt(input: { const childEnv: NodeJS.ProcessEnv = { ...buildRunnerE2EProcessEnvironment(sourceEnvironment, executions), [CHAT_PRIVATE_INPUTS_ENV]: privateInputs?.filename, + [PRIVATE_DIAGNOSTICS_ENV]: diagnosticRegistry, PATH: providerPath, PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: JSON.stringify( executions.map((candidate) => candidate.id), @@ -646,7 +644,9 @@ async function runAttempt(input: { path.join(privateDir, "cases", candidate.task.id, "result.json"), ), options.ui || options.debug, + () => readPrivateDiagnosticSecrets(diagnosticRegistry, temporaryRoot), ); + credentials.splice(0, credentials.length, ...diagnosticSecrets(readPrivateDiagnosticSecrets(diagnosticRegistry, temporaryRoot))); processCleanupFailed = processResult.processCleanupError !== null; if (options.installedStartupOnly) { const proofDir = path.join(resultsRoot, campaignId, "installed-startup-only"); @@ -703,7 +703,9 @@ async function runAttempt(input: { } // Keep cleanup authority before evidence copying/publication can fail. rawCleanupResults.push({ cleanup: result.cleanup, synthetic: result === fallback, status: result.status }); - return enforceResultProcessIntegrity(result, processResult); + const checked = enforceResultProcessIntegrity(result, processResult); + return processResult.privateDiagnosticFailure ? { ...checked, status: "failed" as const, failureClass: checked.status === "passed" ? "secret_leak" as const : checked.failureClass, + error: `${checked.error ? `${checked.error}; ` : ""}Private diagnostic redaction was unavailable; raw output was withheld.` } : checked; }), ); let isolationError: unknown; @@ -760,6 +762,8 @@ async function runAttempt(input: { persistedStateError = error; } } + const reporterPrivacy = await sanitizePrivateReporterArtifacts(privateDir, credentials); + if (processResult.privateDiagnosticLeak) reporterPrivacy.push({ file: "playwright process stdout/stderr", disposition: "redacted", reason: "credential detected before diagnostic emission" }); for (const [index, candidate] of executions.entries()) { let result = { ...results[index], @@ -831,6 +835,13 @@ async function runAttempt(input: { ); await mkdir(casePrivateDir, { recursive: true }); await copySharedEvidence(privateDir, casePrivateDir); + if (reporterPrivacy.length) { + const original = { status: result.status, failureClass: result.failureClass, error: result.error }; + const snapshots = path.join(casePrivateDir, "snapshots"); await mkdir(snapshots, { recursive: true }); + await writeFile(path.join(snapshots, "diagnostic-privacy.json"), `${JSON.stringify(sanitizeJson({ schemaVersion: 1, originalResult: original, issues: reporterPrivacy }, credentials), null, 2)}\n`, { mode: 0o600 }); + result = { ...result, status: "failed", failureClass: result.status === "passed" ? "secret_leak" : result.failureClass, + error: `${result.error ? `${result.error}; ` : ""}Reporter privacy violation: ${reporterPrivacy.map(issue => `${issue.disposition} ${issue.file}`).join(", ")}` }; + } if (persistedStateError && persistenceCheckedExecutions.includes(candidate)) { const snapshots = path.join(casePrivateDir, "snapshots"); await mkdir(snapshots, { recursive: true }); diff --git a/tests/runner-e2e/private-diagnostics.test.ts b/tests/runner-e2e/private-diagnostics.test.ts new file mode 100644 index 00000000000..9f83329f0f2 --- /dev/null +++ b/tests/runner-e2e/private-diagnostics.test.ts @@ -0,0 +1,37 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, chmodSync, realpathSync } from "node:fs"; +import path from "node:path"; +import os from "node:os"; +import { createPrivateDiagnosticStream, diagnosticSecrets, PRIVATE_DIAGNOSTICS_ENV, readPrivateDiagnosticSecrets, stagePrivateDiagnostics, trackedPrivateSecrets } from "./private-diagnostics.js"; +import { buildPaperclipServerEnvironment } from "./harness-env.js"; +const roots: string[] = []; afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); +function registry() { const root = realpathSync(mkdtempSync(path.join(os.tmpdir(), "diagnostic-control-"))); roots.push(root); mkdirSync(path.join(root, "selected-private-inputs"), { mode: 0o700 }); return { root, file: stagePrivateDiagnostics(root, ["selected-case"], []) }; } +describe("private diagnostic output boundary", () => { + it("redacts secret and escaped variants across chunks, multiline PEM and UTF-8 without losing ordinary failure detail", () => { + const password = 'synthetic-credential-<&"-12345678'; + const pem = "-----BEGIN PRIVATE KEY-----\nU1lOVEhFVElDLVByaXZhdGUtS2V5LUxpbmUx\nU1lOVEhFVElDLVByaXZhdGUtS2V5LUxpbmUy\n-----END PRIVATE KEY-----"; + const outputs: string[] = []; const stream = createPrivateDiagnosticStream(() => [password, pem], value => outputs.push(value)); + const input = `HTTP503: π provider unavailable ${password}\n${pem}\nJSON ${JSON.stringify(password)}\nHTML ${password.replace(/&/g, "&").replace(/ { + const { root, file } = registry(); const secrets = trackedPrivateSecrets([], "selected-case", { [PRIVATE_DIAGNOSTICS_ENV]: file, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root }); + expect(buildPaperclipServerEnvironment({ [PRIVATE_DIAGNOSTICS_ENV]: file })[PRIVATE_DIAGNOSTICS_ENV]).toBeUndefined(); + const outputs: string[] = []; const stream = createPrivateDiagnosticStream(() => readPrivateDiagnosticSecrets(file, root), value => outputs.push(value)); + const cookie = "generated-session-canary-982726351"; secrets.push(cookie); + expect(readPrivateDiagnosticSecrets(file, root)).toContain(cookie); + stream.write(Buffer.from(`session ${cookie.slice(0, 9)}`)); stream.write(Buffer.from(cookie.slice(9))); stream.end(); + expect(outputs.join("")).not.toContain(cookie); expect(outputs.join("")).toContain("[REDACTED]"); + expect(readFileSync(file, "utf8")).toContain(cookie); rmSync(path.dirname(file), { recursive: true }); expect(() => readFileSync(file)).toThrow(); + }); + it("fails closed for case mismatch, unsafe permissions, missing registry, oversized lines and late writes", () => { + const { root, file } = registry(); + expect(() => trackedPrivateSecrets([], "different-case", { [PRIVATE_DIAGNOSTICS_ENV]: file, PAPERCLIP_RUNNER_E2E_TEMP_ROOT: root })).toThrow(/mismatch/); + chmodSync(file, 0o644); expect(() => readPrivateDiagnosticSecrets(file, root)).toThrow(/permissions/); chmodSync(file, 0o600); + const outputs: string[] = []; const stream = createPrivateDiagnosticStream(() => { throw new Error("private lookup failed"); }, value => outputs.push(value), 64); + stream.write(Buffer.from("sensitive\n")); stream.write(Buffer.alloc(100, "x")); stream.end(); stream.write(Buffer.from("late private bytes\n")); + expect(stream.failed()).toBe(true); expect(outputs.join("")).not.toContain("sensitive"); expect(outputs.join("")).not.toContain("late private bytes"); expect(outputs.join("")).toContain("size bound exceeded"); + }); +}); diff --git a/tests/runner-e2e/private-diagnostics.ts b/tests/runner-e2e/private-diagnostics.ts new file mode 100644 index 00000000000..0332bf4a1b5 --- /dev/null +++ b/tests/runner-e2e/private-diagnostics.ts @@ -0,0 +1,71 @@ +import { lstatSync, readFileSync, realpathSync, renameSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { StringDecoder } from "node:string_decoder"; +import { findSecretLeak, normalizedSecrets, redactText } from "./redaction.js"; +export const PRIVATE_DIAGNOSTICS_ENV = "PAPERCLIP_RUNNER_E2E_PRIVATE_DIAGNOSTICS"; +interface Envelope { version: 1; executionIds: string[]; secrets: string[] } +function checkedPath(filename: string, temporaryRoot: string) { + const parent = path.join(temporaryRoot, "selected-private-inputs"); + if (filename !== path.join(parent, "diagnostic-redaction.json") || realpathSync(parent) !== parent) throw new Error("Invalid private diagnostic registry path"); + const directory = lstatSync(parent); const file = lstatSync(filename); + if (!directory.isDirectory() || directory.mode & 0o077 || file.isSymbolicLink() || !file.isFile() || file.mode & 0o077 || file.size > 4 * 1024 * 1024 + || typeof process.getuid === "function" && (directory.uid !== process.getuid() || file.uid !== process.getuid())) throw new Error("Private diagnostic registry ownership or permissions are invalid"); +} +function read(filename: string, temporaryRoot: string): Envelope { + checkedPath(filename, temporaryRoot); + const value = JSON.parse(readFileSync(filename, "utf8")) as Envelope; + if (value.version !== 1 || !Array.isArray(value.executionIds) || !value.executionIds.every(id => typeof id === "string") || !Array.isArray(value.secrets) || !value.secrets.every(secret => typeof secret === "string")) throw new Error("Invalid private diagnostic registry"); + return value; +} +export function stagePrivateDiagnostics(temporaryRoot: string, executionIds: string[], secrets: readonly string[]) { + const filename = path.join(temporaryRoot, "selected-private-inputs", "diagnostic-redaction.json"); + writeFileSync(filename, JSON.stringify({ version: 1, executionIds, secrets: normalizedSecrets(secrets) } satisfies Envelope), { mode: 0o600, flag: "wx" }); + checkedPath(filename, temporaryRoot); return filename; +} +export function readPrivateDiagnosticSecrets(filename: string, temporaryRoot: string) { return read(filename, temporaryRoot).secrets; } +/** Existing case code continues to push into its private redaction sink. Each + * registration is durable before the subsequent instrumented operation starts; + * only the launcher/test host receives this private path, never the product. */ +export function trackedPrivateSecrets(initial: string[], executionId: string, env: NodeJS.ProcessEnv = process.env): string[] { + const filename = env[PRIVATE_DIAGNOSTICS_ENV]; if (!filename) return initial; + const temporaryRoot = env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; + if (!temporaryRoot) throw new Error("Private diagnostic registry requires an owned temporary root"); + const previous = read(filename, temporaryRoot); + if (!previous.executionIds.includes(executionId)) throw new Error("Private diagnostic registry execution mismatch"); + const persist = (values: string[]) => { + const current = read(filename, temporaryRoot); + const temporary = `${filename}.${process.pid}.next`; + writeFileSync(temporary, JSON.stringify({ ...current, secrets: normalizedSecrets([...current.secrets, ...values]) }), { mode: 0o600, flag: "wx" }); + renameSync(temporary, filename); + }; + persist(initial); + return new Proxy(initial, { set(target, property, value) { + if (property !== "length" && typeof value !== "string") throw new Error("Private diagnostic secret registration must be a string"); + const next = [...target]; Reflect.set(next, property, value); persist(next); + return Reflect.set(target, property, value); + } }); +} +export function diagnosticSecrets(values: readonly string[]): string[] { + const variants = normalizedSecrets(values).flatMap(value => [value, JSON.stringify(value).slice(1, -1), encodeURIComponent(value), + value.replace(/&/g, "&").replace(//g, ">").replace(/"/g, """).replace(/'/g, "'"), + ...value.split(/\r?\n/).filter(line => line.length >= 8)]); + return normalizedSecrets(variants); +} +/** No raw byte reaches a console or log. UTF-8 and secret boundaries may span + * chunks. Oversized unterminated lines are withheld whole, with a truthful + * omission marker; arbitrary memory growth never forces an unsafe flush. */ +export function createPrivateDiagnosticStream(secrets: () => readonly string[], emit: (safe: string) => void, maximumLine = 1024 * 1024) { + const decoder = new StringDecoder("utf8"); let pending = ""; let suppressing = false; let failed = false; let ended = false; let leaked = false; + const line = (value: string) => { + try { const values = diagnosticSecrets(secrets()); leaked ||= !!findSecretLeak(value, values); emit(redactText(value, values)); } + catch { failed = true; emit("[diagnostics withheld: private redaction unavailable]\n"); } + }; + const consume = (text: string) => { + for (const fragment of text.match(/[^\n]*\n|[^\n]+$/g) ?? []) { + if (!suppressing) pending += fragment; + if (pending.length > maximumLine) { pending = ""; suppressing = true; emit("[diagnostic line withheld: size bound exceeded]\n"); } + if (fragment.endsWith("\n")) { if (!suppressing) line(pending); pending = ""; suppressing = false; } + } + }; + return { write(chunk: Buffer) { if (!ended) consume(decoder.write(chunk)); }, end() { if (ended) return; ended = true; consume(decoder.end()); if (pending && !suppressing) line(pending); pending = ""; }, failed() { return failed; }, redactedSecret() { return leaked; } }; +} diff --git a/tests/runner-e2e/private-reporter-artifacts.test.ts b/tests/runner-e2e/private-reporter-artifacts.test.ts new file mode 100644 index 00000000000..7909bea149b --- /dev/null +++ b/tests/runner-e2e/private-reporter-artifacts.test.ts @@ -0,0 +1,42 @@ +import { afterEach, expect, it } from "vitest"; +import { execFileSync } from "node:child_process"; +import { mkdtemp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { sanitizePrivateReporterArtifacts } from "./private-reporter-artifacts.js"; +import { inspectEmbeddedReport, packageEvidence } from "./evidence.js"; +const roots: string[] = []; afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +it("redacts actual reporter sinks, rejects compressed HTML/blob secrets, and leaves independently observed product bytes unchanged", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "report-privacy-control-")); roots.push(root); + const canary = "synthetic-private-cookie-723958234"; + for (const directory of ["snapshots", "playwright-output", "html-report", "blob-report"]) await mkdir(path.join(root, directory)); + for (const relative of ["playwright.log", "junit.xml", "playwright-output/error-context.md"]) await writeFile(path.join(root, relative), `HTTP429: provider unavailable; failed fill ${canary}\n`, { mode: 0o600 }); + // This is independent oracle input, deliberately not a diagnostic sink. + await writeFile(path.join(root, "snapshots", "api-state.json"), JSON.stringify({ actualProductResponse: canary })); + const raw = path.join(root, "report.json"); await writeFile(raw, JSON.stringify({ error: `failed fill ${canary}` }), { mode: 0o600 }); + const zip = path.join(root, "blob-report", "report.zip"); execFileSync("zip", ["-q", zip, "report.json"], { cwd: root }); + const html = ``; + await writeFile(path.join(root, "html-report", "index.html"), html); + expect(await inspectEmbeddedReport(html, [canary])).toContain("exact secret"); + const issues = await sanitizePrivateReporterArtifacts(root, [canary]); + expect(issues.filter(issue => issue.disposition === "redacted")).toHaveLength(3); + expect(issues.filter(issue => issue.disposition === "omitted")).toHaveLength(2); + expect(JSON.stringify(issues)).not.toContain(canary); + for (const relative of ["playwright.log", "junit.xml", "playwright-output/error-context.md"]) { const safe = await readFile(path.join(root, relative), "utf8"); expect(safe).not.toContain(canary); expect(safe).toContain("HTTP429: provider unavailable"); } + expect(await readdir(path.join(root, "blob-report"))).toEqual([]); expect(await readdir(path.join(root, "html-report"))).toEqual([]); + expect(await readFile(path.join(root, "snapshots/api-state.json"), "utf8")).toContain(canary); + const packaged = await packageEvidence({ privateDir: root, uploadDir: path.join(root, "uploaded"), secrets: [canary], expectPassScreenshot: false }); + // Product leakage still fails independently, despite sanitized diagnostics. + expect(packaged.leaks).toContainEqual({ file: "snapshots/api-state.json", reason: "exact secret value" }); +}); +it("keeps clean inspectable reports, but refuses unknown embedded report encoding", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "report-clean-control-")); roots.push(root); + await mkdir(path.join(root, "html-report")); await writeFile(path.join(root, "html-report/index.html"), "ordinary clean report"); + expect(await sanitizePrivateReporterArtifacts(root, ["private-canary-6234"])).toEqual([]); + expect(await readFile(path.join(root, "html-report/index.html"), "utf8")).toContain("ordinary clean report"); + await writeFile(path.join(root, "clean.json"), JSON.stringify({ error: "HTTP503 provider unavailable" })); + const zip = path.join(root, "clean.zip"); execFileSync("zip", ["-q", zip, "clean.json"], { cwd: root }); + const embedded = `window.playwrightReportBase64="data:application/zip;base64,${(await readFile(zip)).toString("base64")}";`; + expect(await inspectEmbeddedReport(embedded, ["private-canary-6234"])).toBeNull(); + expect(await inspectEmbeddedReport('', [])).toContain("could not be inspected"); +}); diff --git a/tests/runner-e2e/private-reporter-artifacts.ts b/tests/runner-e2e/private-reporter-artifacts.ts new file mode 100644 index 00000000000..0aeba607216 --- /dev/null +++ b/tests/runner-e2e/private-reporter-artifacts.ts @@ -0,0 +1,39 @@ +import { lstat, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { inspectEmbeddedReport, inspectZip } from "./evidence.js"; +import { diagnosticSecrets } from "./private-diagnostics.js"; +import { findSecretLeak, redactText } from "./redaction.js"; +export interface ReporterPrivacyIssue { file: string; disposition: "redacted" | "omitted"; reason: string } +const sources = ["server.log", "playwright.log", "junit.xml", "html-report", "blob-report", "playwright-output"]; +const text = new Set([".log", ".txt", ".md", ".xml", ".html", ".json", ".js", ".css"]); +/** Only reporter/diagnostic sinks are touched, after independent product oracles + * finish. Product API snapshots and work-product bytes are never pre-sanitized. + * Uninspectable/leaking compressed reports are omitted rather than fabricated. */ +export async function sanitizePrivateReporterArtifacts(root: string, values: readonly string[]): Promise { + const secrets = diagnosticSecrets(values); const issues: ReporterPrivacyIssue[] = []; + const visit = async (relative: string): Promise => { + const filename = path.join(root, relative); const info = await lstat(filename).catch((error: NodeJS.ErrnoException) => { if (error.code === "ENOENT") return null; throw error; }); + if (!info) return; + const safeName = redactText(relative, secrets); + if (info.isSymbolicLink()) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter symlink cannot be inspected" }); return; } + if (info.isDirectory()) { for (const name of await readdir(filename)) await visit(path.join(relative, name)); return; } + if (!info.isFile()) return; + if (safeName !== relative) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "credential in reporter filename" }); return; } + const extension = path.extname(filename).toLowerCase(); + if (extension === ".zip") { + const reason = await inspectZip(filename, secrets); + if (reason) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason }); } + } else if (text.has(extension)) { + if (info.size > 128 * 1024 * 1024) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: "reporter text exceeds inspection bound" }); return; } + const raw = await readFile(filename, "utf8"); + const opaque = extension === ".html" ? await inspectEmbeddedReport(raw, secrets) : null; + if (opaque) { await rm(filename); issues.push({ file: safeName, disposition: "omitted", reason: opaque }); return; } + const leak = findSecretLeak(raw, secrets); + const safe = redactText(raw, secrets); + if (safe !== raw) await writeFile(filename, safe, { mode: 0o600 }); + if (leak) issues.push({ file: safeName, disposition: "redacted", reason: leak }); + } + }; + for (const source of sources) await visit(source); + return issues; +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 44d39704683..826b525b011 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,4 +1,5 @@ import { readChatPrivateInputs } from "./chat-private-inputs.js"; +import { trackedPrivateSecrets } from "./private-diagnostics.js"; import { findChatCatalogCase } from "./chat-catalog-registry.js"; import { runChatBoundaryFlow } from "./chat-boundary-flow.js"; import { chatCatalogScenarios } from "./chat-catalog-inventory.js"; @@ -627,7 +628,7 @@ for (const execution of executions) { verifyNativeInstructionPreflight(process.env[NATIVE_INSTRUCTION_PREFLIGHT_ENV]); } const credentials = credentialValues(); - const secrets = normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]); + const secrets = trackedPrivateSecrets(normalizedSecrets([...Object.values(credentials), ...EVAL_OBSERVABILITY_SECRET_NAMES.map(name => process.env[name]), process.env.PAPERCLIP_E2E_GATEWAY_TOKEN]), execution.id); const privateInputs = execution.suite.id === "chat-features" ? readChatPrivateInputs(execution.id, secrets) : undefined; const traceConfig = evalBraintrustConfig(process.env); const trace = traceConfig ? new EvalAttemptTrace(traceConfig, {