test(evals): retain genuine setup health pass evidence

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-setup-health.test.ts

@@ -7,7 +7,7 @@ import { afterEach, describe, expect, it, vi } from "vitest";
 import { runnerMatrix, runnerSuites } from "./catalog.js";
 import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js";
 import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js";
-import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE, gradeSetupHealth, runChatSetupHealth, requestSetupHealthStop, stopSetupHealthIfRequested, SetupHealthAbort } from "./chat-setup-health.js";
+import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE, gradeSetupHealth, runChatSetupHealth, requestSetupHealthStop, stopSetupHealthIfRequested, SetupHealthAbort, captureSetupHealthProfile } from "./chat-setup-health.js";
 const valid = (): Parameters<typeof gradeSetupHealth>[0] => ({ ready: true, agentCount: 1, configMatches: true, issues: 0, conversations: 0, comments: 0, humanComments: 0, runs: 0,
   probes: [{ schema: "paperclip.chat-readiness-probe/v1", companyId: "company", sequence: 1, status: "pass", thrown: false,
     checks: [{ code: "opencode_hello_probe_passed", level: "info", category: "unknown" as const }], errorCategory: "unknown" as const }] });
@@ -66,9 +66,11 @@ it.each([[false, false], [true, false], [false, true]])("uses only readiness GET
   if (failed || lateProbe) await expect(operation).rejects.toThrow("Setup health failed"); else await operation;
   expect(post).toHaveBeenCalledTimes(1);
   expect(get.mock.calls.every(([route]) => !route.includes("test-environment") && !route.includes("/chats/"))).toBe(true);
-  expect(evidence).toHaveBeenCalledTimes(2);
+  expect(evidence).toHaveBeenCalledTimes(3);
   expect((evidence.mock.calls[1]![1] as { state: { probes: unknown[] } }).state.probes).toHaveLength(lateProbe ? 2 : 1);
   expect(evidence.mock.calls[1]![1]).toMatchObject({ canonicalQueries: 0, state: { issues: 0, conversations: 0, runs: 0 } });
+  expect(evidence.mock.calls[2]).toEqual(["api-state.json", expect.objectContaining({ scope: "diagnostic-outside-244", capturePhase: "after-public-pause",
+    observed: expect.objectContaining({ issues: 0, conversations: 0, runs: 0, configMatches: true }) })]);
 });
 
 it("only an exact owned health stop request invokes the existing cleanup callback", async () => {
@@ -100,3 +102,24 @@ it("pause failure requests owned stop before extra oracle reads, even if initial
   expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(true);
   expect(stop).toHaveBeenCalledTimes(1);
 });
+
+it("captures the actual scoped readonly profile only after its visible identity loads", async () => {
+  let current = "", visible = false;
+  const waitFor = vi.fn(async () => { visible = true; });
+  const heading = vi.fn(() => ({ waitFor }));
+  const page = { goto: vi.fn(async (url: string) => { current = url; }), url: () => current,
+    getByRole: vi.fn(() => ({ getByRole: heading })) };
+  const get = vi.fn(async (route: string) => route.includes("/companies/") ? { id: "company", issuePrefix: "RUN" } : { id: "agent", companyId: "company", name: "CEO", urlKey: "ceo" });
+  const capturePrivate = vi.fn(async () => { expect(visible).toBe(true); });
+  const input = { executionId: CHAT_SETUP_HEALTH_ID, api: { baseURL: "http://127.0.0.1:3100", get } as unknown as RunnerApi,
+    page: page as unknown as import("@playwright/test").Page, companyId: "company", agentId: "agent", capturePrivate };
+  await captureSetupHealthProfile(input);
+  expect(page.goto).toHaveBeenCalledWith("http://127.0.0.1:3100/RUN/agents/ceo/overview", { waitUntil: "domcontentloaded" });
+  expect(heading).toHaveBeenCalledWith("heading", { name: "CEO", exact: true });
+  expect(capturePrivate).toHaveBeenCalledExactlyOnceWith("setup-health-final-state", "final-state.png");
+  capturePrivate.mockClear();
+  await expect(captureSetupHealthProfile({ ...input, executionId: "chat-features.runner-opencode.local.create-backlog" })).rejects.toThrow("exact diagnostic cell");
+  get.mockResolvedValue({ id: "foreign", issuePrefix: "OTHER" });
+  await expect(captureSetupHealthProfile(input)).rejects.toThrow("resource mismatch");
+  expect(capturePrivate).not.toHaveBeenCalled();
+});

tests/runner-e2e/chat-setup-health.ts

@@ -1,3 +1,4 @@
+import type { Page } from "@playwright/test";
 import { createHash } from "node:crypto";
 import { constants, readFileSync } from "node:fs";
 import { open, realpath, stat, writeFile } from "node:fs/promises";
@@ -104,6 +105,30 @@ export async function runChatSetupHealth(input: { api: RunnerApi; fixtures: Live
     await input.evidence("chat-setup-health.json", { schema: "paperclip.chat-setup-health/v1", scope: "diagnostic-outside-244", companyId: f.company.id,
       agentId: f.agent.id, state, checks, canonicalQueries: state.humanComments,
       setupProbeUsage: "One ordinary automatic lifecycle hello is expected. Provider tokens and cost are unavailable outside heartbeat accounting; never reported as zero." });
+    await input.evidence("api-state.json", { scope: "diagnostic-outside-244", capturePhase: "after-public-pause",
+      companyId: f.company.id, agentId: f.agent.id, observed: state, checks });
     if (checks.some(check => !check.passed)) throw new Error("Setup health failed; inspect retained closed readiness diagnostics");
   }
 }
+
+/** Genuine readonly diagnostic UI evidence, never published as a task screenshot. */
+export async function captureSetupHealthProfile(input: {
+  executionId: string; api: RunnerApi; page: Page; companyId: string; agentId: string;
+  capturePrivate(id: string, filename: string): Promise<void>;
+}) {
+  if (input.executionId !== CHAT_SETUP_HEALTH_ID) throw new Error("Setup health evidence requires its exact diagnostic cell");
+  const [company, agent] = await Promise.all([
+    input.api.get<{ id: string; issuePrefix: string }>(`/api/companies/${input.companyId}`),
+    input.api.get<{ id: string; companyId: string; name: string; urlKey: string }>(`/api/agents/${input.agentId}`),
+  ]);
+  if (company.id !== input.companyId || agent.id !== input.agentId || agent.companyId !== company.id
+    || !company.issuePrefix || !agent.name || !agent.urlKey) throw new Error("Setup health evidence resource mismatch");
+  const origin = new URL(input.api.baseURL);
+  if (origin.protocol !== "http:" || origin.hostname !== "127.0.0.1" || origin.username || origin.password)
+    throw new Error("Setup health evidence requires the owned loopback app");
+  const url = new URL(`/${encodeURIComponent(company.issuePrefix)}/agents/${encodeURIComponent(agent.urlKey)}/overview`, origin).href;
+  await input.page.goto(url, { waitUntil: "domcontentloaded" });
+  await input.page.getByRole("main").getByRole("heading", { name: agent.name, exact: true }).waitFor({ state: "visible" });
+  if (input.page.url() !== url) throw new Error("Setup health evidence route mismatch");
+  await input.capturePrivate("setup-health-final-state", "final-state.png");
+}

tests/runner-e2e/runner.spec.ts

@@ -1,5 +1,5 @@
 import { checkSetupHealthCredential } from "./chat-setup-credential.js";
-import { SetupHealthAbort, runChatSetupHealth } from "./chat-setup-health.js";
+import { SetupHealthAbort, runChatSetupHealth, captureSetupHealthProfile } from "./chat-setup-health.js";
 import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js";
 import { verifyInvitationSourceProof } from "./chat-catalog-domains/identity-invite-source-proof.js";
 import { readChatPrivateInputs, chatPrivateSetupBlockers, chatPrivateInputRequirements } from "./chat-private-inputs.js";
@@ -1345,6 +1345,8 @@ for (const execution of executions) {
         selectedRuns = contextIntegrity.runs as unknown as RunRecord[];
       } else if (execution.task.flow === "setup_health") {
         await runChatSetupHealth({ api, fixtures, temporaryRoot: temporaryRoot!, evidence: completionEvidence });
+        await captureSetupHealthProfile({ executionId: execution.id, api, page,
+          companyId: fixtures.company.id, agentId: fixtures.agent.id, capturePrivate: capturePrivateScreenshot });
         matcherResults = [{ matcher: { kind: "json_path", path: "setupHealth.closed-checks", expected: true }, passed: true,
           detail: "One genuine ordinary setup hello, exact saved configuration, zero chat/task work." }];
       } else if (execution.task.flow === "agent_chat") {