From 1c72e2d74e77f4761d64a5bf571684fa6ccb2311 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 14:49:31 -0500 Subject: [PATCH] test(evals): retain genuine setup health pass evidence Co-Authored-By: Paperclip --- tests/runner-e2e/chat-setup-health.test.ts | 27 ++++++++++++++++++++-- tests/runner-e2e/chat-setup-health.ts | 25 ++++++++++++++++++++ tests/runner-e2e/runner.spec.ts | 4 +++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/tests/runner-e2e/chat-setup-health.test.ts b/tests/runner-e2e/chat-setup-health.test.ts index 00770afdf04..86d1bbed57a 100644 --- a/tests/runner-e2e/chat-setup-health.test.ts +++ b/tests/runner-e2e/chat-setup-health.test.ts @@ -7,7 +7,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { runnerMatrix, runnerSuites } from "./catalog.js"; import { parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; import { CHAT_OPENCODE_MODEL } from "./chat-runtime-profile.js"; -import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE, gradeSetupHealth, runChatSetupHealth, requestSetupHealthStop, stopSetupHealthIfRequested, SetupHealthAbort } from "./chat-setup-health.js"; +import { CHAT_SETUP_HEALTH_ID, CHAT_SETUP_HEALTH_SUITE, gradeSetupHealth, runChatSetupHealth, requestSetupHealthStop, stopSetupHealthIfRequested, SetupHealthAbort, captureSetupHealthProfile } from "./chat-setup-health.js"; const valid = (): Parameters[0] => ({ ready: true, agentCount: 1, configMatches: true, issues: 0, conversations: 0, comments: 0, humanComments: 0, runs: 0, probes: [{ schema: "paperclip.chat-readiness-probe/v1", companyId: "company", sequence: 1, status: "pass", thrown: false, checks: [{ code: "opencode_hello_probe_passed", level: "info", category: "unknown" as const }], errorCategory: "unknown" as const }] }); @@ -66,9 +66,11 @@ it.each([[false, false], [true, false], [false, true]])("uses only readiness GET if (failed || lateProbe) await expect(operation).rejects.toThrow("Setup health failed"); else await operation; expect(post).toHaveBeenCalledTimes(1); expect(get.mock.calls.every(([route]) => !route.includes("test-environment") && !route.includes("/chats/"))).toBe(true); - expect(evidence).toHaveBeenCalledTimes(2); + expect(evidence).toHaveBeenCalledTimes(3); expect((evidence.mock.calls[1]![1] as { state: { probes: unknown[] } }).state.probes).toHaveLength(lateProbe ? 2 : 1); expect(evidence.mock.calls[1]![1]).toMatchObject({ canonicalQueries: 0, state: { issues: 0, conversations: 0, runs: 0 } }); + expect(evidence.mock.calls[2]).toEqual(["api-state.json", expect.objectContaining({ scope: "diagnostic-outside-244", capturePhase: "after-public-pause", + observed: expect.objectContaining({ issues: 0, conversations: 0, runs: 0, configMatches: true }) })]); }); it("only an exact owned health stop request invokes the existing cleanup callback", async () => { @@ -100,3 +102,24 @@ it("pause failure requests owned stop before extra oracle reads, even if initial expect(await stopSetupHealthIfRequested({ executionIds: [CHAT_SETUP_HEALTH_ID], temporaryRoot, stop })).toBe(true); expect(stop).toHaveBeenCalledTimes(1); }); + +it("captures the actual scoped readonly profile only after its visible identity loads", async () => { + let current = "", visible = false; + const waitFor = vi.fn(async () => { visible = true; }); + const heading = vi.fn(() => ({ waitFor })); + const page = { goto: vi.fn(async (url: string) => { current = url; }), url: () => current, + getByRole: vi.fn(() => ({ getByRole: heading })) }; + const get = vi.fn(async (route: string) => route.includes("/companies/") ? { id: "company", issuePrefix: "RUN" } : { id: "agent", companyId: "company", name: "CEO", urlKey: "ceo" }); + const capturePrivate = vi.fn(async () => { expect(visible).toBe(true); }); + const input = { executionId: CHAT_SETUP_HEALTH_ID, api: { baseURL: "http://127.0.0.1:3100", get } as unknown as RunnerApi, + page: page as unknown as import("@playwright/test").Page, companyId: "company", agentId: "agent", capturePrivate }; + await captureSetupHealthProfile(input); + expect(page.goto).toHaveBeenCalledWith("http://127.0.0.1:3100/RUN/agents/ceo/overview", { waitUntil: "domcontentloaded" }); + expect(heading).toHaveBeenCalledWith("heading", { name: "CEO", exact: true }); + expect(capturePrivate).toHaveBeenCalledExactlyOnceWith("setup-health-final-state", "final-state.png"); + capturePrivate.mockClear(); + await expect(captureSetupHealthProfile({ ...input, executionId: "chat-features.runner-opencode.local.create-backlog" })).rejects.toThrow("exact diagnostic cell"); + get.mockResolvedValue({ id: "foreign", issuePrefix: "OTHER" }); + await expect(captureSetupHealthProfile(input)).rejects.toThrow("resource mismatch"); + expect(capturePrivate).not.toHaveBeenCalled(); +}); diff --git a/tests/runner-e2e/chat-setup-health.ts b/tests/runner-e2e/chat-setup-health.ts index 6393ad904b5..0a28bab2cef 100644 --- a/tests/runner-e2e/chat-setup-health.ts +++ b/tests/runner-e2e/chat-setup-health.ts @@ -1,3 +1,4 @@ +import type { Page } from "@playwright/test"; import { createHash } from "node:crypto"; import { constants, readFileSync } from "node:fs"; import { open, realpath, stat, writeFile } from "node:fs/promises"; @@ -104,6 +105,30 @@ export async function runChatSetupHealth(input: { api: RunnerApi; fixtures: Live await input.evidence("chat-setup-health.json", { schema: "paperclip.chat-setup-health/v1", scope: "diagnostic-outside-244", companyId: f.company.id, agentId: f.agent.id, state, checks, canonicalQueries: state.humanComments, setupProbeUsage: "One ordinary automatic lifecycle hello is expected. Provider tokens and cost are unavailable outside heartbeat accounting; never reported as zero." }); + await input.evidence("api-state.json", { scope: "diagnostic-outside-244", capturePhase: "after-public-pause", + companyId: f.company.id, agentId: f.agent.id, observed: state, checks }); if (checks.some(check => !check.passed)) throw new Error("Setup health failed; inspect retained closed readiness diagnostics"); } } + +/** Genuine readonly diagnostic UI evidence, never published as a task screenshot. */ +export async function captureSetupHealthProfile(input: { + executionId: string; api: RunnerApi; page: Page; companyId: string; agentId: string; + capturePrivate(id: string, filename: string): Promise; +}) { + if (input.executionId !== CHAT_SETUP_HEALTH_ID) throw new Error("Setup health evidence requires its exact diagnostic cell"); + const [company, agent] = await Promise.all([ + input.api.get<{ id: string; issuePrefix: string }>(`/api/companies/${input.companyId}`), + input.api.get<{ id: string; companyId: string; name: string; urlKey: string }>(`/api/agents/${input.agentId}`), + ]); + if (company.id !== input.companyId || agent.id !== input.agentId || agent.companyId !== company.id + || !company.issuePrefix || !agent.name || !agent.urlKey) throw new Error("Setup health evidence resource mismatch"); + const origin = new URL(input.api.baseURL); + if (origin.protocol !== "http:" || origin.hostname !== "127.0.0.1" || origin.username || origin.password) + throw new Error("Setup health evidence requires the owned loopback app"); + const url = new URL(`/${encodeURIComponent(company.issuePrefix)}/agents/${encodeURIComponent(agent.urlKey)}/overview`, origin).href; + await input.page.goto(url, { waitUntil: "domcontentloaded" }); + await input.page.getByRole("main").getByRole("heading", { name: agent.name, exact: true }).waitFor({ state: "visible" }); + if (input.page.url() !== url) throw new Error("Setup health evidence route mismatch"); + await input.capturePrivate("setup-health-final-state", "final-state.png"); +} diff --git a/tests/runner-e2e/runner.spec.ts b/tests/runner-e2e/runner.spec.ts index 0d1177b1846..54996ec3ece 100644 --- a/tests/runner-e2e/runner.spec.ts +++ b/tests/runner-e2e/runner.spec.ts @@ -1,5 +1,5 @@ import { checkSetupHealthCredential } from "./chat-setup-credential.js"; -import { SetupHealthAbort, runChatSetupHealth } from "./chat-setup-health.js"; +import { SetupHealthAbort, runChatSetupHealth, captureSetupHealthProfile } from "./chat-setup-health.js"; import { bindInvitationEnrollmentPrerequisite } from "./chat-catalog-domains/identity-invite-prerequisite.js"; import { verifyInvitationSourceProof } from "./chat-catalog-domains/identity-invite-source-proof.js"; import { readChatPrivateInputs, chatPrivateSetupBlockers, chatPrivateInputRequirements } from "./chat-private-inputs.js"; @@ -1345,6 +1345,8 @@ for (const execution of executions) { selectedRuns = contextIntegrity.runs as unknown as RunRecord[]; } else if (execution.task.flow === "setup_health") { await runChatSetupHealth({ api, fixtures, temporaryRoot: temporaryRoot!, evidence: completionEvidence }); + await captureSetupHealthProfile({ executionId: execution.id, api, page, + companyId: fixtures.company.id, agentId: fixtures.agent.id, capturePrivate: capturePrivateScreenshot }); matcherResults = [{ matcher: { kind: "json_path", path: "setupHealth.closed-checks", expected: true }, passed: true, detail: "One genuine ordinary setup hello, exact saved configuration, zero chat/task work." }]; } else if (execution.task.flow === "agent_chat") {