fix(evals): withhold screenshots after original privacy findings

tests/runner-e2e/SECURITY.md

@@ -581,3 +581,10 @@ qualification prerequisite. This alternative does not qualify that path, create
 an instance administrator, bypass route authorization, or change retained failed
 outcomes. Passwords and cookies remain in the existing private session registry;
 the setup receipt contains only public identity, role, grant keys and statuses.
+
+A screenshot route marker is provisional. Any original secret finding, persisted
+state secret finding, privacy-driven reporter redaction/omission, or packaged
+evidence secret finding withholds all screenshots for that attempt. The launcher
+retains a closed withholding reason and removes public markers while preserving
+original private PNG bytes and hashes. Later cleanup labels and empty sanitized
+export scans cannot clear this decision; the publisher and gallery enforce it.

tests/runner-e2e/abnormal-finalization.ts

@@ -1,9 +1,10 @@
+import { withholdPrivateScreenshots } from "./screenshot-privacy.js";
 import type { RunnerE2EResult } from "./types.js";
 
 /** Evidence failure must retain the original startup/work verdict. A generated
  * fallback remains a failed harness result, never proof that a browser ran. */
 export function abnormalFinalizationResult(original: RunnerE2EResult, error: unknown): RunnerE2EResult {
   const detail = error instanceof Error ? error.message : String(error);
-  return { ...original, status: "failed", failureClass: original.status === "failed" ? original.failureClass : "permanent_infrastructure",
+  return { ...withholdPrivateScreenshots(original), status: "failed", failureClass: original.status === "failed" ? original.failureClass : "permanent_infrastructure",
     error: `${original.error ? `${original.error}; ` : ""}Abnormal attempt finalization: ${detail}` };
 }

tests/runner-e2e/dashboard.ts

@@ -1,3 +1,4 @@
+import { screenshotWithholdingNotice } from "./screenshot-privacy.js";
 import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, summarizeCampaignTiming, summarizeWorkerPhaseTiming } from "./campaign-timing.js";
 import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js";
 import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js";
@@ -83,6 +84,7 @@ function compactJson(value: unknown) {
 function resolveScreenshots(
   entry: RunnerDashboardEntry | undefined,
 ): ResolvedScreenshot[] {
+  if (entry && screenshotWithholdingNotice(entry.result)) return [];
   const declaredScreenshots = entry?.result.screenshots?.length
     ? entry.result.screenshots
     : entry
@@ -244,7 +246,8 @@ function renderCase(
     ${STARTUP_REUSE_LAYERS.map(layer => `<td>${html(reuse.layers[layer].state)} (${html(reuse.layers[layer].reason)})</td>`).join("")}
     <td>${html(reuse.providerLaunchCount === null ? "unknown" : reuse.providerLaunchCount)}</td>
   </tr>`).join("");
-  const gallery = screenshots.length
+  const withholdingNotice = entry ? screenshotWithholdingNotice(entry.result) : null;
+  const gallery = withholdingNotice ? `<p class="muted">${html(withholdingNotice)}</p>` : screenshots.length
     ? `<div class="gallery" aria-label="Screenshots for ${html(execution.id)}">${screenshots
         .map(
           (item) => `<button

tests/runner-e2e/history-publish.ts

@@ -1,3 +1,4 @@
+import { screenshotPrivacyReasons } from "./screenshot-privacy.js";
 import { validateCampaignAccounting, validateRetainedRunnerResult } from "./result-validation.js";
 import { validateEvalOperationsReport } from "./operations-report.js";
 import { validateCampaignFeedbackReview } from "./feedback-dispositions.js";
@@ -89,6 +90,7 @@ export function publicScreenshotPaths(campaign: RunnerE2ECampaign) {
   assertPublicCampaign(campaign);
   const screenshots = new Set<string>();
   for (const result of campaign.results) {
+    if (result.screenshotPrivacy?.withheld || screenshotPrivacyReasons(result).length) continue;
     const files =
       result.screenshots
         ?.filter(

tests/runner-e2e/history.test.ts

@@ -892,3 +892,17 @@ describe("historical publication security", () => {
     );
   });
 });
+
+// These PNGs are synthetic controls; no retained incident image is inspected.
+describe("privacy findings override screenshot route admission", () => {
+  it.each([
+    { failureClass: "secret_leak", screenshotPrivacy: undefined },
+    { failureClass: "cleanup_failure", screenshotPrivacy: { withheld: true, reasons: ["inner-secret-finding"] } },
+    { failureClass: "cleanup_failure", screenshotPrivacy: { withheld: true, reasons: ["reporter-evidence-withheld"] } },
+  ])("withholds marked screenshots despite terminal wrapper: %j", signal => {
+    const execution = runnerMatrix[0]!;
+    const original = { ...result(execution, "failed"), ...signal, screenshots: [{ id: "failure", label: "Synthetic failure", file: "failure.png", publication: "public-runner-fixture" }] } as RunnerE2EResult;
+    const campaign = buildRunnerCampaign({ campaignId: "privacy-withheld", generatedAt: "2026-08-28T00:01:00.000Z", expected: [execution.id], results: [original] });
+    expect([...publicScreenshotPaths(campaign)]).toEqual([]);
+  });
+});

tests/runner-e2e/launch.ts

@@ -1,3 +1,4 @@
+import { withholdPrivateScreenshots } from "./screenshot-privacy.js";
 import { copyHealthHelloDetail } from "./chat-readiness-detail.js";
 import { copyRetainedSourceSdk } from "./source-sdk-evidence.js";
 import { prepareChatRemoteHost } from "./chat-remote-host.js";
@@ -793,7 +794,7 @@ async function runAttempt(input: {
     const reporterPrivacy = await sanitizePrivateReporterArtifacts(privateDir, credentials);
     if (processResult.privateDiagnosticLeak) reporterPrivacy.push({ file: "playwright process stdout/stderr", disposition: "redacted", reason: "credential detected before diagnostic emission" });
     for (const [index, candidate] of executions.entries()) {
-      let result = {
+      let result: RunnerE2EResult = {
         ...results[index],
         accountingIdentity: { campaignId: accountingCampaignId, attemptId: accountingAttemptId },
       };
@@ -880,6 +881,10 @@ async function runAttempt(input: {
           originalResult: { status: original.status, failureClass: original.failureClass, error: original.error },
         }, credentials), null, 2)}\n`, { mode: 0o600 });
       }
+      result = withholdPrivateScreenshots(result, {
+        persistedStateSecretFinding: !!persistedStateFinding && persistenceCheckedExecutions.includes(candidate),
+        reporterEvidenceWithheld: reporterPrivacy.length > 0,
+      });
       await writeFile(
         resultPath,
         `${JSON.stringify(sanitizeJson(result, credentials), null, 2)}\n`,
@@ -892,6 +897,7 @@ async function runAttempt(input: {
         expectPassScreenshot: result.status === "passed",
       });
       if (evidence.leaks.length > 0 || evidence.missing.length > 0) {
+        result = withholdPrivateScreenshots(result, { packagedEvidenceSecretFinding: evidence.leaks.length > 0 });
         result = {
           ...result,
           status: "failed",

tests/runner-e2e/result-validation.ts

@@ -1,3 +1,4 @@
+import { SCREENSHOT_PRIVACY_REASONS } from "./screenshot-privacy.js";
 import { POSTGRES_IMAGE, POSTGRES_IMAGE_ID, POSTGRES_AMD64_IMAGE, POSTGRES_AMD64_IMAGE_IDS, reviewedPostgresReceiptIdentity } from "./docker-postgres.js";
 import { validateWorkerTiming } from "./worker-timing.js";
 import { connectionCheckpoints, connectionEvidencePasses, connectionRunSignals } from "./connection-evidence.js";
@@ -332,6 +333,7 @@ const fields = {
   matcherResults: optional(
     array(shape({ matcher, passed: boolean, detail: string })),
   ),
+  screenshotPrivacy: optional(closedShape({ withheld: (value, at) => { if (value !== true) invalid(at); }, reasons: (value, at) => { array(oneOf(...SCREENSHOT_PRIVACY_REASONS))(value, at); if (!(value as unknown[]).length) invalid(at); } })),
   screenshots: optional(
     array(
       shape({

tests/runner-e2e/runner.spec.ts

@@ -1,3 +1,4 @@
+import { withholdPrivateScreenshots } from "./screenshot-privacy.js";
 import { ordinaryChatCapture } from "./chat-screenshot-capture.js";
 import { checkSetupHealthCredential } from "./chat-setup-credential.js";
 import { SetupHealthAbort, runChatSetupHealth, captureSetupHealthProfile } from "./chat-setup-health.js";
@@ -703,6 +704,7 @@ for (const execution of executions) {
     const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
     let primaryError: unknown;
     let failureClassOverride: FailureClass | undefined;
+    let innerSecretFinding = false;
     let cleanup: RunnerE2EResult["cleanup"] = "not_started";
     const cleanupAssertions: CleanupAssertion[] = [];
     const beforeEnvironmentTeardownAssertions: CleanupAssertion[] = [];
@@ -3157,6 +3159,7 @@ for (const execution of executions) {
       }
     } catch (error) {
       primaryError = error;
+      innerSecretFinding ||= classifyFailure(error) === "secret_leak";
       if (execution.task.flow === "first_task" && !firstTaskEvidence && classifyFailure(error) === "candidate_failure") {
         failureClassOverride = "permanent_infrastructure";
       }
@@ -3165,6 +3168,7 @@ for (const execution of executions) {
       } catch (captureError) {
         const failureClass = classifyFailure(captureError);
         if (failureClass === "secret_leak") {
+          innerSecretFinding = true;
           failureClassOverride = "secret_leak";
           primaryError = new AggregateError(
             [primaryError, captureError],
@@ -3216,6 +3220,7 @@ for (const execution of executions) {
           `Browser diagnostics contained unsafe data: ${error instanceof Error ? error.message : String(error)}`,
         );
         failureClassOverride = "secret_leak";
+        innerSecretFinding = true;
       }
       browserBootstrap.dispose();
       if (healthAbort) { cleanup = "failed"; failureClassOverride = "cleanup_failure"; }
@@ -3290,6 +3295,7 @@ for (const execution of executions) {
             ? classifyFailure(primaryError)
             : undefined;
           const cleanupFailureClass = classifyFailure(error);
+          innerSecretFinding ||= priorFailureClass === "secret_leak" || cleanupFailureClass === "secret_leak";
           failureClassOverride =
             priorFailureClass === "secret_leak"
               ? priorFailureClass
@@ -3426,10 +3432,10 @@ for (const execution of executions) {
         screenshots,
         cleanup,
       };
-      const result: RunnerE2EResult = {
+      const result: RunnerE2EResult = withholdPrivateScreenshots({
         ...resultWithoutBilling,
         billing: summarizeExecutionBilling(resultWithoutBilling),
-      };
+      }, { innerSecretFinding });
       if (firstTaskEvidence) {
         await writeSanitizedJson(snapshotsDir, "first-task-result-private.json", result, secrets);
       }

tests/runner-e2e/screenshot-privacy.test.ts

@@ -0,0 +1,84 @@
+import { execFileSync } from "node:child_process";
+import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, expect, it } from "vitest";
+import { abnormalFinalizationResult } from "./abnormal-finalization.js";
+import { runnerMatrix } from "./catalog.js";
+import { regenerateRunnerDashboard } from "./dashboard-regenerate.js";
+import { packageEvidence } from "./evidence.js";
+import { publicScreenshotPaths, prunePrivateHistoryEvidence } from "./history-publish.js";
+import { buildRunnerCampaign } from "./history.js";
+import { sanitizePrivateReporterArtifacts } from "./private-reporter-artifacts.js";
+import { validateRetainedRunnerResult } from "./result-validation.js";
+import { screenshotWithholdingNotice, withholdPrivateScreenshots } from "./screenshot-privacy.js";
+import type { RunnerE2EResult } from "./types.js";
+const roots: string[] = [];
+afterEach(async () => { await Promise.all(roots.splice(0).map(root => rm(root, { recursive: true, force: true }))); });
+const png = Buffer.from("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII=", "base64");
+function result(): RunnerE2EResult {
+  const e = runnerMatrix.find(row => row.suite.id === "chat-features" && row.profile.id === "runner-opencode" && row.environment.id === "local")!;
+  return { schema: "paperclip.runner-e2e.result/v2", executionId: e.id, suiteId: e.suite.id, suiteDefinitionHash: e.suiteDefinitionHash,
+    attempt: 1, status: "failed", failureClass: "candidate_failure", profileId: e.profile.id, environmentId: e.environment.id, caseId: e.task.id,
+    provider: e.profile.provider, model: e.profile.model, runtimeMode: e.profile.expectedRuntimeMode, startedAt: "2026-10-28T00:00:00.000Z",
+    finishedAt: "2026-10-28T00:00:01.000Z", durationMs: 1000, cleanup: "passed",
+    screenshots: [{ id: "failure", label: "Synthetic image", file: "failure.png", publication: "public-runner-fixture", sha256: createHash("sha256").update(png).digest("hex") }] };
+}
+it.each(["inner", "persisted", "reporter", "packaged"] as const)("retains %s privacy custody through cleanup and repeated clean finalization", finding => {
+  const original = result();
+  const held = withholdPrivateScreenshots(original, { innerSecretFinding: finding === "inner", persistedStateSecretFinding: finding === "persisted",
+    reporterEvidenceWithheld: finding === "reporter", packagedEvidenceSecretFinding: finding === "packaged" });
+  const wrapped = { ...abnormalFinalizationResult(held, new Error("packaging failed")), failureClass: "cleanup_failure" as const, cleanup: "failed" as const };
+  const final = withholdPrivateScreenshots(wrapped);
+  expect(final.screenshotPrivacy?.reasons).toHaveLength(1);
+  expect(final.screenshotPrivacy).toEqual(held.screenshotPrivacy);
+  expect(final.screenshots?.[0]).toEqual({ id: "failure", label: "Synthetic image", file: "failure.png", sha256: original.screenshots![0]!.sha256 });
+  expect(original.screenshots![0]!.publication).toBe("public-runner-fixture");
+  expect(final.status).toBe("failed"); expect(final.cleanup).toBe("failed");
+  expect(screenshotWithholdingNotice(final)).toContain("original images remain private");
+  expect(() => validateRetainedRunnerResult(final)).not.toThrow();
+});
+it("keeps clean behavior-failure screenshots publishable and rejects forged withholding metadata", () => {
+  const original = result();
+  expect(withholdPrivateScreenshots(original)).toBe(original);
+  expect(screenshotWithholdingNotice(original)).toBeNull();
+  const campaign = buildRunnerCampaign({ campaignId: "clean", generatedAt: original.finishedAt, expected: [original.executionId], results: [original] });
+  expect([...publicScreenshotPaths(campaign)]).toHaveLength(1);
+  for (const screenshotPrivacy of [{ withheld: false, reasons: ["inner-secret-finding"] }, { withheld: true, reasons: [] }, { withheld: true, reasons: ["arbitrary private text"] }])
+    expect(() => validateRetainedRunnerResult({ ...original, screenshotPrivacy })).toThrow("screenshotPrivacy");
+});
+it.each(["redacted", "omitted"] as const)("withholds synthetic PNG after actual reporter %s even when packaged scans are empty", async disposition => {
+  const root = await mkdtemp(path.join(os.tmpdir(), "screenshot-privacy-")); roots.push(root);
+  const privateDir = path.join(root, "private"), packaged = path.join(root, "packaged");
+  await mkdir(path.join(privateDir, "playwright-output"), { recursive: true });
+  await writeFile(path.join(privateDir, "failure.png"), png);
+  const secret = "synthetic-reporter-canary-only-2345";
+  if (disposition === "redacted") await writeFile(path.join(privateDir, "playwright-output", "error-context.md"), `Synthetic field ${secret}`);
+  else {
+    await mkdir(path.join(privateDir, "blob-report"));
+    await writeFile(path.join(root, "synthetic-report.txt"), secret);
+    execFileSync("zip", ["-q", path.join(privateDir, "blob-report", "report.zip"), "synthetic-report.txt"], { cwd: root });
+  }
+  const issues = await sanitizePrivateReporterArtifacts(privateDir, [secret]);
+  expect(issues).toMatchObject([{ disposition, reason: "exact secret value" }]);
+  const original = result(), held = withholdPrivateScreenshots(original, { reporterEvidenceWithheld: issues.length > 0 });
+  const final = { ...held, failureClass: "cleanup_failure" as const, cleanup: "failed" as const };
+  await writeFile(path.join(privateDir, "result.json"), JSON.stringify(final));
+  expect(await packageEvidence({ privateDir, uploadDir: packaged, secrets: [secret], expectPassScreenshot: false })).toMatchObject({ leaks: [], missing: [] });
+  const stage = path.join(root, "public-stage"), base = `evidence/${final.executionId}/attempt-1`;
+  await mkdir(path.join(stage, "evidence"), { recursive: true });
+  await cp(packaged, path.join(stage, base), { recursive: true });
+  const campaign = buildRunnerCampaign({ campaignId: "privacy-synthetic", generatedAt: final.finishedAt, expected: [final.executionId], results: [final] });
+  await writeFile(path.join(stage, "normalized-results.json"), JSON.stringify(campaign));
+  const allowed = publicScreenshotPaths(campaign);
+  expect([...allowed]).toEqual([]);
+  await prunePrivateHistoryEvidence(stage, allowed);
+  await expect(readFile(path.join(stage, base, "failure.png"))).rejects.toThrow();
+  expect(await readFile(path.join(privateDir, "failure.png"))).toEqual(png);
+  expect(await readFile(path.join(packaged, "failure.png"))).toEqual(png);
+  await regenerateRunnerDashboard({ bundle: stage });
+  const html = await readFile(path.join(stage, "index.html"), "utf8");
+  expect(html).toContain("Screenshots withheld after a privacy finding");
+  expect(html).not.toContain(`data-gallery-href="${base}/failure.png"`);
+});

tests/runner-e2e/screenshot-privacy.ts

@@ -0,0 +1,38 @@
+import type { RunnerE2EResult } from "./types.js";
+
+export const SCREENSHOT_PRIVACY_REASONS = [
+  "inner-secret-finding", "persisted-state-secret-finding", "reporter-evidence-withheld", "packaged-evidence-secret-finding",
+] as const;
+export type ScreenshotPrivacyReason = typeof SCREENSHOT_PRIVACY_REASONS[number];
+export interface ScreenshotPrivacy { withheld: true; reasons: ScreenshotPrivacyReason[] }
+
+/** Sticky attempt-level privacy custody survives later cleanup/packaging labels.
+ * A clean sanitized export cannot certify the original visual evidence. */
+export function screenshotPrivacyReasons(result: Pick<RunnerE2EResult, "failureClass" | "screenshotPrivacy">): ScreenshotPrivacyReason[] {
+  return [...new Set([
+    ...(result.screenshotPrivacy?.reasons ?? []),
+    ...(result.failureClass === "secret_leak" && !result.screenshotPrivacy ? ["inner-secret-finding" as const] : []),
+  ])];
+}
+
+export function withholdPrivateScreenshots(result: RunnerE2EResult, findings: {
+  innerSecretFinding?: boolean; persistedStateSecretFinding?: boolean; reporterEvidenceWithheld?: boolean; packagedEvidenceSecretFinding?: boolean;
+} = {}): RunnerE2EResult {
+  const reasons = [...new Set([
+    ...screenshotPrivacyReasons(result),
+    ...(findings.innerSecretFinding ? ["inner-secret-finding" as const] : []),
+    ...(findings.persistedStateSecretFinding ? ["persisted-state-secret-finding" as const] : []),
+    ...(findings.reporterEvidenceWithheld ? ["reporter-evidence-withheld" as const] : []),
+    ...(findings.packagedEvidenceSecretFinding ? ["packaged-evidence-secret-finding" as const] : []),
+  ])];
+  if (!reasons.length) return result;
+  return { ...result, screenshotPrivacy: { withheld: true, reasons },
+    // Removing only the publication marker also protects older trusted
+    // publishers. Original filenames, hashes and private PNG bytes remain.
+    screenshots: result.screenshots?.map(({ publication: _, ...screenshot }) => screenshot) };
+}
+
+export function screenshotWithholdingNotice(result: Pick<RunnerE2EResult, "failureClass" | "screenshotPrivacy">): string | null {
+  const reasons = screenshotPrivacyReasons(result);
+  return reasons.length || result.screenshotPrivacy?.withheld ? `Screenshots withheld after a privacy finding (${reasons.join(", ")}); original images remain private.` : null;
+}

tests/runner-e2e/types.ts

@@ -356,6 +356,7 @@ export interface RunnerE2EResult {
     passed: boolean;
     detail: string;
   }>;
+  screenshotPrivacy?: import("./screenshot-privacy.js").ScreenshotPrivacy;
   screenshots?: Array<{
     id: string;
     label: string;