From 39a7a83e3d8185967f05d2d61c509da0b983fe3d Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 21:27:25 -0600 Subject: [PATCH] test(evals): retain bounded archive carrier rejection counts Co-Authored-By: Paperclip --- .../private-attempt-diagnostic.test.ts | 24 +++++++++++++++++++ .../runner-e2e/private-attempt-diagnostic.ts | 19 ++++++++++++++- .../private-evidence-budget.test.ts | 2 +- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/tests/runner-e2e/private-attempt-diagnostic.test.ts b/tests/runner-e2e/private-attempt-diagnostic.test.ts index 8336fff4bde..b405bcc384e 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.test.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.test.ts @@ -70,3 +70,27 @@ it("writes one closed variable to the actual owned runner environment file and r await expect(appendPrivateAttemptDiagnostic({ stage: "execution" }, { ...env, GITHUB_ENV: path.join(commands, "set_env_symlink") })).rejects.toThrow(); } finally { await rm(root, { recursive: true, force: true }); } }); + +it("classifies existing reporter and packaging carriers without exposing private filenames", () => { + const reason = "zip inspection exceeded its expanded-byte bound"; + const issues = [ + { file: "blob-report/report.zip", reason }, + { file: "playwright-output/private-test/trace.zip", reason }, + { file: "html-report/data/private-hash.zip", reason }, + { file: "html-report/index.html", reason: "zip inspection exceeded its time bound" }, + { file: "snapshots/private-path.json", reason: "exact secret value" }, + { file: "private-path", reason: "private-reason" }, + ]; + const diagnostic = privateAttemptDiagnostic({ stage: "package-original", observation: { privacy: { issues } }, + evidence: { leaks: [{ file: "playwright-output\\private-test\\trace.zip", reason }, { file: "private-path", reason }] } }); + expect(diagnostic.carrierReasons).toEqual([ + { source: "reporter", carrier: "blob-zip", reason, count: 1 }, + { source: "reporter", carrier: "playwright-trace-zip", reason, count: 1 }, + { source: "reporter", carrier: "html-trace-zip", reason, count: 1 }, + { source: "reporter", carrier: "embedded-html", reason: "zip inspection exceeded its time bound", count: 1 }, + { source: "reporter", carrier: "other", reason: "exact secret value", count: 1 }, + { source: "packaging", carrier: "playwright-trace-zip", reason, count: 1 }, + { source: "packaging", carrier: "other", reason, count: 1 }, + ]); + expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:test|hash|path|reason)/); +}); diff --git a/tests/runner-e2e/private-attempt-diagnostic.ts b/tests/runner-e2e/private-attempt-diagnostic.ts index 7c72c97937e..44068835035 100644 --- a/tests/runner-e2e/private-attempt-diagnostic.ts +++ b/tests/runner-e2e/private-attempt-diagnostic.ts @@ -33,6 +33,22 @@ function closedPackagingBudget(input: unknown, message: string) { export interface PrivateAttemptDiagnosticInput { stage: unknown; result?: unknown; error?: unknown; observation?: unknown; evidence?: unknown } const record = (value: unknown): Record => value && typeof value === "object" ? value as Record : {}; const member = (value: unknown, choices: string[]) => typeof value === "string" && choices.includes(value) ? value : "unknown"; +const reporterReasons = ["zip inspection exceeded its expanded-byte bound", "zip inspection exceeded its time bound", "exact secret value", "secret-shaped value"]; +function carrierReasons(input: unknown, source: "reporter" | "packaging") { + const findings = (Array.isArray(input) ? input : []).map(issue => { + const file = typeof record(issue).file === "string" ? record(issue).file.replace(/\\/g, "/") : ""; + const carrier = /^blob-report\/[^/]+\.zip$/.test(file) ? "blob-zip" + : /^playwright-output\/[^/]+\/trace\.zip$/.test(file) ? "playwright-trace-zip" + : /^html-report\/data\/[^/]+\.zip$/.test(file) ? "html-trace-zip" + : file === "html-report/index.html" ? "embedded-html" : "other"; + return { carrier, reason: record(issue).reason }; + }); + return ["blob-zip", "playwright-trace-zip", "html-trace-zip", "embedded-html", "other"].flatMap(carrier => + reporterReasons.flatMap(reason => { + const count = Math.min(findings.filter(issue => issue.carrier === carrier && issue.reason === reason).length, 1000); + return count ? [{ source, carrier, reason, count }] : []; + })); +} const failures = new Map([ ["Credential-contaminated original evidence cannot be retained", "credential-contamination"], ["Unsupported original evidence carrier cannot be retained", "unsupported-carrier"], @@ -107,7 +123,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) { phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified", privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")), reporter: Array.isArray(privacy.issues) ? Math.min(privacy.issues.length, 1000) : 0, - reporterReasons: Object.fromEntries(["zip inspection exceeded its expanded-byte bound", "zip inspection exceeded its time bound", "exact secret value", "secret-shaped value"].map(reason => + reporterReasons: Object.fromEntries(reporterReasons.map(reason => [reason, Array.isArray(privacy.issues) ? Math.min(privacy.issues.filter(issue => record(issue).reason === reason).length, 1000) : 0])), packaging: /^Secret leak in /.test(message) ? (/^Secret leak in snapshots\//.test(message) ? "snapshot" : /^Secret leak in .*\.json:/.test(message) ? "json" : "text") : "unknown" }, runStatusCounts: Object.fromEntries(["queued", "running", "succeeded", "failed", "cancelled", "timed_out"].map(label => @@ -119,6 +135,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) { held: status(runs.find(run => typeof state.activeRunId === "string" && run.id === state.activeRunId)), productGradePassed: record(envelope.grade).passed === true, sourceProgress: sourceProgress(envelope.sourceEvents), + carrierReasons: [...carrierReasons(privacy.issues, "reporter"), ...carrierReasons(leaks, "packaging")], carrier: leaks.some(leak => record(leak).reason === "zip inspection exceeded its expanded-byte bound") ? "zip-byte-bound" : leaks.some(leak => record(leak).reason === "zip inspection exceeded its time bound") ? "zip-time-bound" : leaks.length ? "other-finding" : "none", }; diff --git a/tests/runner-e2e/private-evidence-budget.test.ts b/tests/runner-e2e/private-evidence-budget.test.ts index f1cf8fdd2c3..afc21fc125d 100644 --- a/tests/runner-e2e/private-evidence-budget.test.ts +++ b/tests/runner-e2e/private-evidence-budget.test.ts @@ -30,7 +30,7 @@ it("retains the real ZIP expansion scanner and reports its distinct bound with i execFileSync("python3",["-c","import zipfile,sys; z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_DEFLATED); f=z.open('synthetic.txt','w'); block=b'0'*1048576; [f.write(block) for _ in range(257)]; f.close(); z.close()",zip],{stdio:"pipe",timeout:30000}); const evidence=await packageEvidence(f.input); expect(evidence.leaks).toMatchObject([{reason:"zip inspection exceeded its expanded-byte bound"}]); - expect(privateAttemptDiagnostic({stage:"seal-original",evidence})).toMatchObject({packagingBudget:{bound:"zip-expanded-bytes",inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified"}}); + expect(privateAttemptDiagnostic({stage:"seal-original",evidence})).toMatchObject({carrierReasons:[{source:"packaging",carrier:"blob-zip",reason:"zip inspection exceeded its expanded-byte bound",count:1}],packagingBudget:{bound:"zip-expanded-bytes",inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified"}}); expect(evidence.files).not.toContain("blob-report/oversized.zip"); }); it("a completed small scan reports exact closed metadata but cannot qualify an outcome without required carriers",async()=>{