From 0a76dbde9b87ad031d630b6c8b1601abc9e9ef86 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 2 Nov 2026 23:55:00 -0600 Subject: [PATCH] test(evals): inspect announcements on normal company surfaces Co-Authored-By: Paperclip --- .../identity-announcement-surface.test.ts | 58 +++++++++++++++++++ .../identity-navigation-boundaries.ts | 14 ++++- 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-announcement-surface.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-announcement-surface.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-announcement-surface.test.ts new file mode 100644 index 00000000000..b1ff7ec71d2 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-announcement-surface.test.ts @@ -0,0 +1,58 @@ +import { expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { CHAT_ANNOUNCEMENT_MANIFEST } from "../chat-announcement-fixture.js"; +vi.mock("@playwright/test", () => ({ expect: (node: { visible(): boolean; count(): number }) => ({ + toBeVisible: async () => { if (!node.visible()) throw Error("Requested normal surface is not visible"); }, + toHaveCount: async (count: number) => { if (node.count() !== count) throw Error("Unexpected visible surface count"); }, +}) })); +vi.mock("./identity-auth.js", () => ({ createAuthenticatedPerson: async ({ browser }: { browser: { person: unknown } }) => browser.person, enrollPrerequisitePerson: async () => {} })); +vi.mock("./identity-support.js", async importOriginal => ({ ...await importOriginal(), waitForReply: async () => {}, probeAgentPermission: async () => 403 })); +import { navigationBoundaryIdentityCases } from "./identity-navigation-boundaries.js"; +function fixture(fault?: "modal" | "unsettled projects" | "missing secondary publication" | "dismissal lost on reload" | "modal after reload" | "unsettled projects after reload") { + const publication = CHAT_ANNOUNCEMENT_MANIFEST.announcement; + let current = "/RUN/chats/source", dismissed = false, reloaded = false; + const visits: string[] = []; + function page(observer = false) { + let path = current; + const node = (visible: () => boolean) => ({ visible, count: () => Number(visible()), getByRole: (role: string, options: { name: string }) => role === "button" ? { click: async () => { expect(options.name).toBe("Dismiss announcement"); dismissed = true; } } : node(visible) }); + const modal = () => path === "/BIR/dashboard" || fault === "modal" || (reloaded && fault === "modal after reload"); + const p = { + url: () => path, + context: () => ({ browser: () => ({ person }) }), + goto: async (value: string) => { path = value; current = value; visits.push(value); }, + reload: async () => { visits.push("reload:" + path); reloaded = true; if (fault === "dismissal lost on reload") dismissed = false; }, + getByText: (text: string) => node(() => text === "No projects yet." && path === "/BIR/projects" && fault !== "unsettled projects" && !(reloaded && fault === "unsettled projects after reload") && !modal()), + getByRole: (role: string) => role === "dialog" ? node(modal) : node(() => !modal() && (observer || !dismissed) && !(path.startsWith("/BIR/") && fault === "missing secondary publication")), + waitForResponse: async () => ({ status: () => 204 }), close: async () => {}, + }; + return p; + } + const person = { id: "riley", dispose: async () => {}, context: { newPage: async () => page(true) }, request: async (_path: string, method?: string) => ({ status: method === "POST" ? 404 : 200, json: async () => publication, body: { cancel: async () => {} } }) }; + const ctx = { input: { nonce: "owned", fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } }, privateSecrets: [], page: page(), api: { + get: async (path: string) => path === "/api/auth/get-session" ? { user: { id: "alex" } } : path === "/api/announcements/current" ? (dismissed ? null : publication) : Promise.reject(Error("Unexpected read")), + post: async (path: string) => { expect(path).toBe("/api/companies"); return { id: "birch", issuePrefix: "BIR" }; }, delete: async () => {}, + }, evidence: async () => {} } } as unknown as ChatCatalogContext; + return { ctx, visits, definition: navigationBoundaryIdentityCases[1]! }; +} +it("the actual dismissal setup uses a settled ordinary second-company surface, then preserves real reload and independent-person proof", async () => { + const f = fixture(); const state = await f.definition.setup(f.ctx); + expect(state.secondaryVisible).toBe(true); expect(state.invalidAuditCompanyStatus).toBe(404); + await f.definition.act!(f.ctx, state); + expect(f.visits.filter(x => x.startsWith("/BIR/"))).toEqual(["/BIR/projects", "/BIR/projects"]); + expect(f.visits).toContain("reload:/BIR/projects"); + expect(state.secondaryAfterReloadHidden).toBe(true); expect(state.observerStillVisible).toBe(true); expect(state.agentDismissStatus).toBe(403); +}); +it.each(["modal", "unsettled projects", "missing secondary publication"] as const)("setup refuses %s rather than pretending the announcement was visible", async fault => { + const f = fixture(fault); await expect(f.definition.setup(f.ctx)).rejects.toThrow(); +}); +it("a publication reappearing after the real secondary reload still fails", async () => { + const f = fixture("dismissal lost on reload"), state = await f.definition.setup(f.ctx); + await expect(f.definition.act!(f.ctx, state)).rejects.toThrow("Unexpected visible surface count"); + expect(state.secondaryAfterReloadHidden).not.toBe(true); +}); + +it.each(["modal after reload", "unsettled projects after reload"] as const)("does not mistake %s for a persisted dismissal", async fault => { + const f = fixture(fault), state = await f.definition.setup(f.ctx); + await expect(f.definition.act!(f.ctx, state)).rejects.toThrow(); + expect(state.secondaryAfterReloadHidden).not.toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts b/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts index f80e90eb6bd..76783899c28 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-navigation-boundaries.ts @@ -50,6 +50,14 @@ export function gradeNavigationBoundary(value: unknown) { return c.result(); } const sources = ["chat-catalog-domains/identity-navigation-boundaries.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-private-browser.ts", "chat-catalog-domains/identity-private-browser-worker.ts", "chat-catalog-domains/identity-wizard.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/navigation-preferences.yaml"]; +// An empty company's dashboard correctly opens first-agent onboarding, which hides +// optional announcements. Use its normal Projects page for this personal preference. +async function openAnnouncementCompanySurface(ctx: ChatCatalogContext, issuePrefix: string, reload = false) { + await ctx.input.page.goto(`/${issuePrefix}/projects`, { waitUntil: "domcontentloaded" }); + if (reload) await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByText("No projects yet.", { exact: true })).toBeVisible(); + await expect(ctx.input.page.getByRole("dialog")).toHaveCount(0); +} export const navigationBoundaryIdentityCases: readonly ChatCatalogCase[] = [ identityDefinition("navigation-preferences.broken-link", async ctx => { const source = await openStandingChat(ctx); const p = await project(ctx); @@ -106,7 +114,7 @@ export const navigationBoundaryIdentityCases: readonly ChatCatalogCase[] = [ const invalidAuditCompanyStatus = invalid.status; await invalid.body?.cancel(); const source = ctx.input.page.url(); await expect(ctx.input.page.getByRole("complementary", { name: "Paperclip announcements" }).getByRole("heading", { name: publication.title, exact: true })).toBeVisible(); - await ctx.input.page.goto(`/${otherCompany.issuePrefix}/dashboard`, { waitUntil: "domcontentloaded" }); + await openAnnouncementCompanySurface(ctx, otherCompany.issuePrefix); await expect(ctx.input.page.getByRole("complementary", { name: "Paperclip announcements" }).getByRole("heading", { name: publication.title, exact: true })).toBeVisible(); const secondary = await ctx.input.api.get("/api/announcements/current"); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); @@ -114,7 +122,7 @@ export const navigationBoundaryIdentityCases: readonly ChatCatalogCase[] = [ observerBeforePublicationId: observerPublication?.id, invalidAuditCompanyStatus }; } catch (error) { try { await observer.dispose(); } finally { observers.delete(ctx); const id = otherCompanies.get(ctx); if (id) await ctx.input.api.delete(`/api/companies/${id}`, { allowNotFound: true }); otherCompanies.delete(ctx); } throw error; } }, gradeNavigationBoundary, { - sourceFiles: [...sources, "chat-announcement-fixture.ts", "harness-env.ts", "../../packages/shared/src/announcements.ts", "../../server/src/services/announcement-feed.ts", "../../server/src/routes/announcements.ts"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.announcement-dismissal.v2", + sourceFiles: [...sources, "chat-announcement-fixture.ts", "harness-env.ts", "../../packages/shared/src/announcements.ts", "../../server/src/services/announcement-feed.ts", "../../server/src/routes/announcements.ts", "../../ui/src/pages/Projects.tsx", "../../ui/src/components/AnnouncementWell.tsx"], authentication: "human-session", oracleVersion: "paperclip.chat-catalog.announcement-dismissal.v2", async act(ctx, state) { await waitForReply(ctx); const publication = state.publication as Publication; const source = ctx.input.page.url(); state.beforeHumanPublicationId = (await ctx.input.api.get("/api/announcements/current"))?.id; @@ -123,7 +131,7 @@ export const navigationBoundaryIdentityCases: readonly ChatCatalogCase[] = [ await ctx.input.page.getByRole("complementary", { name: "Paperclip announcements" }).getByRole("button", { name: "Dismiss announcement", exact: true }).click(); state.dismissRequestStatus = (await response).status(); state.afterHumanPublication = await ctx.input.api.get("/api/announcements/current"); const other = state.otherCompany as Entity & { issuePrefix: string }; - await ctx.input.page.goto(`/${other.issuePrefix}/dashboard`, { waitUntil: "domcontentloaded" }); await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await openAnnouncementCompanySurface(ctx, other.issuePrefix, true); await expect(ctx.input.page.getByRole("complementary", { name: "Paperclip announcements" })).toHaveCount(0); state.secondaryAfterReloadHidden = true; const observer = observers.get(ctx)!; const receipt = await observer.request("/api/announcements/current"); state.observerAfterPublicationId = ((await receipt.json()) as Publication | null)?.id; const page = await observer.context.newPage();