From 155dd1bc1c4b85ae107f6a0cd429720caa281367 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:56:13 -0600 Subject: [PATCH] fix(evals): reject redaction-required plain attachments --- tests/runner-e2e/evidence.ts | 2 +- .../private-framework-report.test.ts | 22 +++++++++++++++++-- tests/runner-e2e/private-framework-report.ts | 3 ++- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/evidence.ts b/tests/runner-e2e/evidence.ts index 4d8b097728c..3e138d5d69d 100644 --- a/tests/runner-e2e/evidence.ts +++ b/tests/runner-e2e/evidence.ts @@ -265,7 +265,7 @@ export async function packageEvidence(input: { : findSecretLeak(raw, input.secrets); if (leak) leaks.push({ file: relative, reason: leak }); if (registeredAttachments.has(relative.split(path.sep).join("/"))) { - if (JSON.stringify(sanitizeJson(parsed ?? raw, input.secrets)) !== JSON.stringify(parsed ?? raw)) throw new Error("Private framework attachment requires redaction"); + if (extension === ".json" ? JSON.stringify(sanitizeJson(parsed, input.secrets)) !== JSON.stringify(parsed) : redactText(raw, input.secrets) !== raw) throw new Error("Private framework attachment requires redaction"); if (!leak) { await copyFile(source, destination); files.push(relative); } continue; } diff --git a/tests/runner-e2e/private-framework-report.test.ts b/tests/runner-e2e/private-framework-report.test.ts index 55116881b2a..95b46f40d04 100644 --- a/tests/runner-e2e/private-framework-report.test.ts +++ b/tests/runner-e2e/private-framework-report.test.ts @@ -79,9 +79,9 @@ it("keeps the unchanged 256MiB packaging/archive caps with one canonical large a await promisify(execFile)("python3", ["-c", "import zipfile,sys;z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_STORED);f=z.open('synthetic.trace','w');[f.write(b'0'*1048576) for _ in range(140)];f.close();z.close()", zip], { timeout: 30000 }); const tests = path.join(f.root, "budget-tests"); await mkdir(tests); const require = createRequire(import.meta.url), pkg = require.resolve("@playwright/test/package.json"), entry = require.resolve("@playwright/test"); - await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>{throw new Error('The --token option requires a value');});});`); + await writeFile(path.join(tests, "failed.spec.cjs"), `const {test,expect}=require(${JSON.stringify(entry)});test('owned failed large attachment',async({},info)=>{await info.attach('owned trace',{path:${JSON.stringify(zip)},contentType:'application/zip'});await test.step('actual failed assertion',async()=>expect(1).toBe(2));});`); const config = path.join(tests, "config.cjs"); - await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, use: { trace: "retain-on-failure" }, outputDir: path.join(f.source, "playwright-output"), reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`); + await writeFile(config, `module.exports=${JSON.stringify({ testDir: tests, testMatch: "failed.spec.cjs", workers: 1, use: { trace: "retain-on-failure" }, metadata: { diagnostic: "The --token option requires a value" }, outputDir: path.join(f.source, "playwright-output"), reporter: [["json", { outputFile: path.join(f.source, "framework-report.json") }], ["html", { outputFolder: path.join(f.source, "html-report"), open: "never" }], ["blob", { outputDir: path.join(f.source, "blob-report") }]] })};`); await promisify(execFile)(process.execPath, [path.join(path.dirname(pkg), "cli.js"), "test", "--config", config], { cwd: tests, timeout: 30000, env: { PATH: process.env.PATH, CI: "1", NODE_OPTIONS: "", NODE_PATH: "" } }).then(() => { throw new Error("Original should fail"); }, error => { expect(error.code).toBe(1); }); await rm(path.join(f.cell, "playwright-output"), { recursive: true }); for (const folder of ["playwright-output", "html-report", "blob-report"]) await cp(path.join(f.source, folder), path.join(f.cell, folder), { recursive: true }); @@ -184,3 +184,21 @@ it("rejects sensitive JSON keys in registered metadata and JSON attachments with await expect(packageEvidence({ privateDir: f.cell, uploadDir: path.join(f.root, "upload"), secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } })).rejects.toThrow("requires redaction"); expect(await readFile(file, "utf8")).toBe('{"password":"unrecognized-private-value"}'); }); + +it("rejects unrecognized credentials in a decoded plain-text body", async () => { + const f = await fixture(), attachment = f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]! as any; + attachment.contentType = "text/plain"; attachment.body = Buffer.from('{"password":"opaque-value"}').toString("base64"); + await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report)); + await expect(copyPrivateFrameworkReport(f.source, f.cell, [])).rejects.toThrow(); +}); +it("rejects unrecognized command credentials in a registered plain-text file", async () => { + const f = await fixture(), file = path.join(f.cell, "snapshots/owned.log"); await mkdir(path.dirname(file), { recursive: true }); + await writeFile(file, "--token opaque-value"); + f.report.suites[0]!.specs[0]!.tests[0]!.results[0]!.attachments[0]!.path = file; + await writeFile(path.join(f.source, "framework-report.json"), JSON.stringify(f.report)); await copyPrivateFrameworkReport(f.source, f.cell, []); + const { packageEvidence } = await import("./evidence.js"); + const { inspectCellReportProducer, privateCellEvidenceOwner } = await import("./manual-private-report-assets.js"); + const id = "chat-features.runner-opencode.local.activity.agent-feedback"; + const owner = await privateCellEvidenceOwner({ root: f.root, private: true, runId: "123", attempt: "1", executionIds: [id], cellReportProducer: await inspectCellReportProducer() }, id); + await expect(packageEvidence({ privateDir: f.cell, uploadDir: path.join(f.root, "upload"), secrets: [], expectPassScreenshot: false, privateCell: { owner, layout: "original" } })).rejects.toThrow("requires redaction"); +}); diff --git a/tests/runner-e2e/private-framework-report.ts b/tests/runner-e2e/private-framework-report.ts index 8e5680bac72..16c5be60f88 100644 --- a/tests/runner-e2e/private-framework-report.ts +++ b/tests/runner-e2e/private-framework-report.ts @@ -3,7 +3,7 @@ import { createRequire } from "node:module"; import { createHash } from "node:crypto"; import { copyFile, lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; import path from "node:path"; -import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets, sanitizeJson } from "./redaction.js"; +import { assertSecretFree, findSecretLeak, findSecretLeakInJsonValues, normalizedSecrets, sanitizeJson, redactText } from "./redaction.js"; export const PRIVATE_FRAMEWORK_ORIGINAL_REPORT = "snapshots/playwright-framework-original.txt"; export const PRIVATE_FRAMEWORK_REPORT = "snapshots/playwright-framework.json"; @@ -60,6 +60,7 @@ export function inspectPrivateFrameworkMetadata(relative: string, raw: string, s if (typeof attachment.body !== "string" || !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(attachment.body)) fail(); const body = Buffer.from(attachment.body, "base64"), text = body.toString("utf8"); if (!["text/plain", "application/json"].includes(attachment.contentType) || !Buffer.from(text).equals(body) || findSecretLeak(body, expanded)) fail(); + if (attachment.contentType === "text/plain" && redactText(text, secrets) !== text) fail(); if (attachment.contentType === "application/json" && (findSecretLeak(JSON.stringify(JSON.parse(text)), expanded) || JSON.stringify(sanitizeJson(JSON.parse(text), secrets)) !== JSON.stringify(JSON.parse(text)))) fail(); } }