test(evals): retain bounded archive carrier rejection counts

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/private-attempt-diagnostic.test.ts

@@ -70,3 +70,27 @@ it("writes one closed variable to the actual owned runner environment file and r
     await expect(appendPrivateAttemptDiagnostic({ stage: "execution" }, { ...env, GITHUB_ENV: path.join(commands, "set_env_symlink") })).rejects.toThrow();
   } finally { await rm(root, { recursive: true, force: true }); }
 });
+
+it("classifies existing reporter and packaging carriers without exposing private filenames", () => {
+  const reason = "zip inspection exceeded its expanded-byte bound";
+  const issues = [
+    { file: "blob-report/report.zip", reason },
+    { file: "playwright-output/private-test/trace.zip", reason },
+    { file: "html-report/data/private-hash.zip", reason },
+    { file: "html-report/index.html", reason: "zip inspection exceeded its time bound" },
+    { file: "snapshots/private-path.json", reason: "exact secret value" },
+    { file: "private-path", reason: "private-reason" },
+  ];
+  const diagnostic = privateAttemptDiagnostic({ stage: "package-original", observation: { privacy: { issues } },
+    evidence: { leaks: [{ file: "playwright-output\\private-test\\trace.zip", reason }, { file: "private-path", reason }] } });
+  expect(diagnostic.carrierReasons).toEqual([
+    { source: "reporter", carrier: "blob-zip", reason, count: 1 },
+    { source: "reporter", carrier: "playwright-trace-zip", reason, count: 1 },
+    { source: "reporter", carrier: "html-trace-zip", reason, count: 1 },
+    { source: "reporter", carrier: "embedded-html", reason: "zip inspection exceeded its time bound", count: 1 },
+    { source: "reporter", carrier: "other", reason: "exact secret value", count: 1 },
+    { source: "packaging", carrier: "playwright-trace-zip", reason, count: 1 },
+    { source: "packaging", carrier: "other", reason, count: 1 },
+  ]);
+  expect(JSON.stringify(diagnostic)).not.toMatch(/private-(?:test|hash|path|reason)/);
+});

tests/runner-e2e/private-attempt-diagnostic.ts

@@ -33,6 +33,22 @@ function closedPackagingBudget(input: unknown, message: string) {
 export interface PrivateAttemptDiagnosticInput { stage: unknown; result?: unknown; error?: unknown; observation?: unknown; evidence?: unknown }
 const record = (value: unknown): Record<string, any> => value && typeof value === "object" ? value as Record<string, any> : {};
 const member = (value: unknown, choices: string[]) => typeof value === "string" && choices.includes(value) ? value : "unknown";
+const reporterReasons = ["zip inspection exceeded its expanded-byte bound", "zip inspection exceeded its time bound", "exact secret value", "secret-shaped value"];
+function carrierReasons(input: unknown, source: "reporter" | "packaging") {
+  const findings = (Array.isArray(input) ? input : []).map(issue => {
+    const file = typeof record(issue).file === "string" ? record(issue).file.replace(/\\/g, "/") : "";
+    const carrier = /^blob-report\/[^/]+\.zip$/.test(file) ? "blob-zip"
+      : /^playwright-output\/[^/]+\/trace\.zip$/.test(file) ? "playwright-trace-zip"
+      : /^html-report\/data\/[^/]+\.zip$/.test(file) ? "html-trace-zip"
+      : file === "html-report/index.html" ? "embedded-html" : "other";
+    return { carrier, reason: record(issue).reason };
+  });
+  return ["blob-zip", "playwright-trace-zip", "html-trace-zip", "embedded-html", "other"].flatMap(carrier =>
+    reporterReasons.flatMap(reason => {
+      const count = Math.min(findings.filter(issue => issue.carrier === carrier && issue.reason === reason).length, 1000);
+      return count ? [{ source, carrier, reason, count }] : [];
+    }));
+}
 const failures = new Map([
   ["Credential-contaminated original evidence cannot be retained", "credential-contamination"],
   ["Unsupported original evidence carrier cannot be retained", "unsupported-carrier"],
@@ -107,7 +123,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) {
     phase: phases.find(([label]) => [...errors, message].some(error => error.includes(label!)))?.[1] ?? "unclassified",
     privacy: { persistedState: errors.some(error => error.startsWith("Secret leak in persisted ")),
       reporter: Array.isArray(privacy.issues) ? Math.min(privacy.issues.length, 1000) : 0,
-      reporterReasons: Object.fromEntries(["zip inspection exceeded its expanded-byte bound", "zip inspection exceeded its time bound", "exact secret value", "secret-shaped value"].map(reason =>
+      reporterReasons: Object.fromEntries(reporterReasons.map(reason =>
         [reason, Array.isArray(privacy.issues) ? Math.min(privacy.issues.filter(issue => record(issue).reason === reason).length, 1000) : 0])),
       packaging: /^Secret leak in /.test(message) ? (/^Secret leak in snapshots\//.test(message) ? "snapshot" : /^Secret leak in .*\.json:/.test(message) ? "json" : "text") : "unknown" },
     runStatusCounts: Object.fromEntries(["queued", "running", "succeeded", "failed", "cancelled", "timed_out"].map(label =>
@@ -119,6 +135,7 @@ export function privateAttemptDiagnostic(input: PrivateAttemptDiagnosticInput) {
     held: status(runs.find(run => typeof state.activeRunId === "string" && run.id === state.activeRunId)),
     productGradePassed: record(envelope.grade).passed === true,
     sourceProgress: sourceProgress(envelope.sourceEvents),
+    carrierReasons: [...carrierReasons(privacy.issues, "reporter"), ...carrierReasons(leaks, "packaging")],
     carrier: leaks.some(leak => record(leak).reason === "zip inspection exceeded its expanded-byte bound") ? "zip-byte-bound"
       : leaks.some(leak => record(leak).reason === "zip inspection exceeded its time bound") ? "zip-time-bound" : leaks.length ? "other-finding" : "none",
   };

tests/runner-e2e/private-evidence-budget.test.ts

@@ -30,7 +30,7 @@ it("retains the real ZIP expansion scanner and reports its distinct bound with i
  execFileSync("python3",["-c","import zipfile,sys; z=zipfile.ZipFile(sys.argv[1],'w',compression=zipfile.ZIP_DEFLATED); f=z.open('synthetic.txt','w'); block=b'0'*1048576; [f.write(block) for _ in range(257)]; f.close(); z.close()",zip],{stdio:"pipe",timeout:30000});
  const evidence=await packageEvidence(f.input);
  expect(evidence.leaks).toMatchObject([{reason:"zip inspection exceeded its expanded-byte bound"}]);
- expect(privateAttemptDiagnostic({stage:"seal-original",evidence})).toMatchObject({packagingBudget:{bound:"zip-expanded-bytes",inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified"}});
+ expect(privateAttemptDiagnostic({stage:"seal-original",evidence})).toMatchObject({carrierReasons:[{source:"packaging",carrier:"blob-zip",reason:"zip inspection exceeded its expanded-byte bound",count:1}],packagingBudget:{bound:"zip-expanded-bytes",inventoryComplete:true,scanFinished:true,requiredEvidenceComplete:false,qualification:"unqualified"}});
  expect(evidence.files).not.toContain("blob-report/oversized.zip");
 });
 it("a completed small scan reports exact closed metadata but cannot qualify an outcome without required carriers",async()=>{