From 945674fffef7381e790e9c74173018faee7f563b Mon Sep 17 00:00:00 2001
From: Dotta
Date: Tue, 27 Oct 2026 16:06:44 -0500
Subject: [PATCH] test(evals): retain ordinary catalog chat screenshots
privately
---
tests/runner-e2e/SECURITY.md | 5 +++
.../chat-screenshot-capture.test.ts | 39 +++++++++++++++++++
tests/runner-e2e/chat-screenshot-capture.ts | 13 +++++++
tests/runner-e2e/runner.spec.ts | 3 +-
4 files changed, 59 insertions(+), 1 deletion(-)
create mode 100644 tests/runner-e2e/chat-screenshot-capture.test.ts
create mode 100644 tests/runner-e2e/chat-screenshot-capture.ts
diff --git a/tests/runner-e2e/SECURITY.md b/tests/runner-e2e/SECURITY.md
index 30e2f2bc1a9..b7e7d31829e 100644
--- a/tests/runner-e2e/SECURITY.md
+++ b/tests/runner-e2e/SECURITY.md
@@ -292,6 +292,11 @@ latest pointers are mutable, and S3 versioning makes those updates recoverable.
## Public evidence boundary
+Ordinary `chat-features` catalog captures remain private Actions evidence, including
+the required genuine final-state PNG. They receive no public screenshot marker;
+canonical agent slugs and legitimate result navigation do not expand the public
+route allowlist. Pre-chat boundary and other suite capture guards remain unchanged.
+
CloudFront and GitHub Pages are public. Fixture identifiers, timing, token
usage, costs, normalized results, and trusted runner PNG screenshots are
expected public data. Each
diff --git a/tests/runner-e2e/chat-screenshot-capture.test.ts b/tests/runner-e2e/chat-screenshot-capture.test.ts
new file mode 100644
index 00000000000..1e5c4ad33a6
--- /dev/null
+++ b/tests/runner-e2e/chat-screenshot-capture.test.ts
@@ -0,0 +1,39 @@
+import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import path from "node:path";
+import os from "node:os";
+import { expect, it, vi } from "vitest";
+import { ordinaryChatCapture } from "./chat-screenshot-capture.js";
+import { packageEvidence } from "./evidence.js";
+import { isHistoricalBundlePathAllowed } from "./history-publish.js";
+import { isPublicRunnerScreenshotRoute } from "./screenshot-policy.js";
+const execFileAsync = promisify(execFile);
+it("retains catalog result and mandatory final-state captures privately without invoking the public route/marker callback", async () => {
+ const privateCapture = vi.fn(async () => {}), publicCapture = vi.fn(async () => { throw new Error("Public route guard would reject the canonical Morgan slug"); });
+ const capture = ordinaryChatCapture("chat-features", privateCapture, publicCapture);
+ await capture("chat-catalog-result", "Verified result", "chat-catalog-result.png");
+ await capture("final-state", "Verified final state", "final-state.png");
+ expect(privateCapture.mock.calls).toEqual([["chat-catalog-result", "chat-catalog-result.png"], ["final-state", "final-state.png"]]);
+ expect(publicCapture).not.toHaveBeenCalled();
+ expect(isPublicRunnerScreenshotRoute("http://127.0.0.1:3100/RUN/chats/morgan", { issuePrefix: "RUN", issueId: "source", issueIdentifier: "RUN-3", chatAgentId: "actual-agent-uuid" })).toBe(false);
+});
+it("preserves the exact public capture callback and its rejection for all other ordinary chat suites", async () => {
+ const privateCapture = vi.fn(async () => {}), publicCapture = vi.fn(async () => { throw new Error("Existing public guard rejection"); });
+ const capture = ordinaryChatCapture("chat", privateCapture, publicCapture);
+ expect(capture).toBe(publicCapture); await expect(capture("final", "Task", "final-state.png")).rejects.toThrow("Existing public guard rejection"); expect(privateCapture).not.toHaveBeenCalled();
+});
+it("keeps the required final-state file in real packaged Actions evidence while public history excludes its unmarked path", async () => {
+ const root = await mkdtemp(path.join(os.tmpdir(), "private-chat-capture-")), upload = `${root}-upload`;
+ try {
+ for (const dir of ["snapshots", "html-report", "blob-report"]) await mkdir(path.join(root, dir));
+ for (const [file, content] of Object.entries({ "result.json": "{}", "junit.xml": "", "snapshots/fixtures.json": "{}", "snapshots/api-state.json": "{}", "html-report/index.html": "", "blob-report/receipt.txt": "synthetic unit fixture" })) await writeFile(path.join(root, file), content);
+ await execFileAsync("zip", ["-q", "report.zip", "receipt.txt"], { cwd: path.join(root, "blob-report") }); await rm(path.join(root, "blob-report/receipt.txt"));
+ const bytes = Buffer.from("unit PNG custody fixture");
+ const capture = ordinaryChatCapture("chat-features", async (_id, file) => { await writeFile(path.join(root, file), bytes); }, async () => { throw new Error("Must remain private"); });
+ await capture("final-state", "Real callback supplies the actual page screenshot", "final-state.png");
+ const packaged = await packageEvidence({ privateDir: root, uploadDir: upload, secrets: [], expectPassScreenshot: true });
+ expect(packaged.missing).toEqual([]); expect(await readFile(path.join(upload, "final-state.png"))).toEqual(bytes);
+ expect(isHistoricalBundlePathAllowed("evidence/chat-features.runner-opencode.local.agent-chat.reopen-conversation/attempt-1/final-state.png", false, new Set())).toBe(false);
+ } finally { await rm(root, { recursive: true, force: true }); await rm(upload, { recursive: true, force: true }); }
+});
diff --git a/tests/runner-e2e/chat-screenshot-capture.ts b/tests/runner-e2e/chat-screenshot-capture.ts
new file mode 100644
index 00000000000..d4ff602814c
--- /dev/null
+++ b/tests/runner-e2e/chat-screenshot-capture.ts
@@ -0,0 +1,13 @@
+type Capture = (id: string, label: string, file: string) => Promise;
+/** Ordinary catalog chats retain genuine screenshots as private Actions evidence.
+ * Canonical agent slugs and legitimate result navigation are not public routes.
+ * All other suites still use their existing reviewed public capture guard. */
+export function ordinaryChatCapture(
+ suiteId: string,
+ capturePrivate: (id: string, file: string) => Promise,
+ captureReviewedPublic: Capture,
+): Capture {
+ return suiteId === "chat-features"
+ ? (id, _label, file) => capturePrivate(id, file)
+ : captureReviewedPublic;
+}
diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts
index 54996ec3ece..cc255af7e23 100644
--- a/tests/runner-e2e/runner.spec.ts
+++ b/tests/runner-e2e/runner.spec.ts
@@ -1,3 +1,4 @@
+import { ordinaryChatCapture } from "./chat-screenshot-capture.js";
import { checkSetupHealthCredential } from "./chat-setup-credential.js";
import { SetupHealthAbort, runChatSetupHealth, captureSetupHealthProfile } from "./chat-setup-health.js";
import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js";
@@ -1355,7 +1356,7 @@ for (const execution of executions) {
page, api, fixtures, execution, nonce, workspacePath,
restart: () => restartChatServer(page, () => restartIsolatedPaperclipServer({ api, requestId: `chat-${nonce}`, deadlineAt: startedAtMs + deadlineMs })),
observe: (chatIssue, chatRuns) => { issue = chatIssue; selectedRuns = chatRuns; },
- capture: captureScreenshot,
+ capture: ordinaryChatCapture(execution.suite.id, capturePrivateScreenshot, captureScreenshot),
evidence: completionEvidence,
check: (id, passed, detail) => matcherResults.push({ matcher: { kind: "json_path", path: `chat.${id}`, expected: true }, passed, detail }),
});