From d6f00a813626f65f8fcbcecdf0c809829a21379d Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 16:10:07 -0500 Subject: [PATCH] test(evals): use genuine human origin for disabled chat probe --- .../runtime-chat-disabled.test.ts | 35 +++++++++++++++++-- .../runtime-chat-disabled.ts | 18 +++++++--- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts index 80b8bf6c295..66db0603377 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.test.ts @@ -1,11 +1,40 @@ -import { it, expect } from "vitest"; -import { gradeDisabledChat } from "./runtime-chat-disabled.js"; +import { createAuthenticatedPerson } from "./identity-auth.js"; +import type { RunnerApi } from "../api.js"; +import type { Browser } from "@playwright/test"; +import { it, expect, vi, afterEach } from "vitest"; +import { gradeDisabledChat, probeDisabledChatAsPerson } from "./runtime-chat-disabled.js"; function valid() { return { companyId: "company", agentId: "morgan", personId: "actual-human", canonicalIntent: "Open a chat with Morgan.", agentChatEnabled: false, unavailableVisible: true, composerAbsent: true, supportedOperatorStepVisible: true, profileAccessible: true, agentSettingsStatus: 403, - createChatStatus: 404, gateAfterProbe: false, beforeTaskIds: [], afterTaskIds: [] as string[], chatIds: [] as string[], runIds: [] as string[] }; } + createChatStatus: 404, createChatErrorCategory: "agent-chat-disabled", createChatProbePersonId: "actual-human", gateAfterProbe: false, beforeTaskIds: [], afterTaskIds: [] as string[], chatIds: [] as string[], runIds: [] as string[] }; } it("accepts the actual unavailable Morgan Chat surface with no admission", () => expect(gradeDisabledChat(valid()).passed).toBe(true)); it("rejects an unrelated disabled conference gate while Agent Chat is enabled", () => { const o = valid(); o.agentChatEnabled = true; expect(gradeDisabledChat(o).passed).toBe(false); }); it("rejects missing canonical unavailable context", () => expect(gradeDisabledChat(undefined).passed).toBe(false)); it("rejects a disguised task created while Chat is disabled", () => { const o = valid(); o.afterTaskIds = ["workaround"]; expect(gradeDisabledChat(o).passed).toBe(false); }); it("rejects an ordinary agent enabling the instance setting", () => { const o = valid(); o.agentSettingsStatus = 200; o.gateAfterProbe = true; expect(gradeDisabledChat(o).passed).toBe(false); }); it("rejects native work started by unavailable navigation", () => { const o = valid(); o.runIds = ["unrequested"]; expect(gradeDisabledChat(o).passed).toBe(false); }); + +afterEach(() => vi.unstubAllGlobals()); +it("requires the actual disabled-route denial, not a missing-origin denial or another person's probe", () => { + for (const change of [{ createChatStatus: 403 }, { createChatStatus: 200 }, { createChatStatus: 500 }, { createChatErrorCategory: "trusted-origin-required" }, { createChatErrorCategory: "unrecognized-response" }, { createChatProbePersonId: "different-human" }]) expect(gradeDisabledChat({ ...valid(), ...change }).passed).toBe(false); +}); +it("uses the existing real-person wrapper with its session cookie and trusted origin, without board bearer substitution", async () => { + const origin = "http://127.0.0.1:3100", captured: Array<{ url: string; method: string; headers: Headers }> = []; + vi.stubGlobal("fetch", vi.fn(async (input: URL | string, options: RequestInit = {}) => { + const url = new URL(String(input)), headers = new Headers(options.headers); + if (url.pathname === "/api/auth/sign-up/email") return new Response(JSON.stringify({ user: { id: "actual-human", name: "Alex" } }), { status: 200, headers: { "set-cookie": "fixture-session=unit-private-session; Path=/; HttpOnly" } }); + if (url.pathname === "/api/auth/sign-out") return new Response("{}", { status: 200 }); + captured.push({ url: url.href, method: options.method ?? "GET", headers }); + const correct = headers.get("origin") === origin && headers.get("cookie") === "fixture-session=unit-private-session" && !headers.has("authorization"); + return new Response(JSON.stringify({ error: correct ? "Agent Chat is disabled" : "Board mutation requires trusted browser origin" }), { status: correct ? 404 : 403 }); + })); + const context = { setStorageState: vi.fn(async () => {}), close: vi.fn(async () => {}) }; + const person = await createAuthenticatedPerson({ api: { baseURL: origin, get: async () => ({ deploymentMode: "authenticated" }) } as unknown as RunnerApi, + browser: { newContext: async () => context } as unknown as Browser, secrets: [], name: "Alex" }); + try { + expect(await probeDisabledChatAsPerson(person, "owned-company", "owned-agent")).toEqual({ createChatStatus: 404, createChatErrorCategory: "agent-chat-disabled", createChatProbePersonId: "actual-human" }); + expect(captured).toHaveLength(1); expect(captured[0]!.url).toBe(`${origin}/api/companies/owned-company/chats/owned-agent`); expect(captured[0]!.method).toBe("POST"); + expect(context.setStorageState).toHaveBeenCalledOnce(); + const missingOrigin = await probeDisabledChatAsPerson({ id: person.id, request: async path => fetch(new URL(path, origin), { method: "POST", headers: { Cookie: "fixture-session=unit-private-session" } }) }, "owned-company", "owned-agent"); + expect(missingOrigin).toMatchObject({ createChatStatus: 403, createChatErrorCategory: "trusted-origin-required" }); expect(gradeDisabledChat({ ...valid(), ...missingOrigin }).passed).toBe(false); + } finally { await person.dispose(); } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts index 37f82ad557e..baa00e79788 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-chat-disabled.ts @@ -10,7 +10,17 @@ type Person = Awaited>; const owned = new WeakMap(); interface DisabledObservation { companyId: string; agentId: string; personId: string; canonicalIntent: string; agentChatEnabled: boolean; unavailableVisible: boolean; composerAbsent: boolean; supportedOperatorStepVisible: boolean; profileAccessible: boolean; - agentSettingsStatus: number; createChatStatus: number; gateAfterProbe: boolean; beforeTaskIds: string[]; afterTaskIds: string[]; chatIds: string[]; runIds: string[] } + agentSettingsStatus: number; createChatStatus: number; createChatErrorCategory: string; createChatProbePersonId: string; gateAfterProbe: boolean; beforeTaskIds: string[]; afterTaskIds: string[]; chatIds: string[]; runIds: string[] } +/** Use the same genuine person's existing private same-origin session wrapper. + * Return only a closed error category, never the raw response or credential. */ +export async function probeDisabledChatAsPerson(person: Pick, companyId: string, agentId: string) { + const response = await person.request(`/api/companies/${encodeURIComponent(companyId)}/chats/${encodeURIComponent(agentId)}`, "POST", {}); + const status = response.status; + const body = await response.json().catch(() => undefined) as { error?: unknown } | undefined; + const errorCategory = body?.error === "Agent Chat is disabled" ? "agent-chat-disabled" + : body?.error === "Board mutation requires trusted browser origin" ? "trusted-origin-required" : "unrecognized-response"; + return { createChatStatus: status, createChatErrorCategory: errorCategory, createChatProbePersonId: person.id }; +} export function gradeDisabledChat(value: unknown) { const c = checks(); const o = value as DisabledObservation | undefined; c.add("actual-disabled-canonical-context", !!o && !!o.companyId && !!o.agentId && !!o.personId && o.personId !== "local-board" @@ -19,7 +29,7 @@ export function gradeDisabledChat(value: unknown) { c.add("clear-supported-human-path", !!o && o.supportedOperatorStepVisible && o.profileAccessible, "The unavailable surface identifies Experimental settings as the human management step, and the actual Morgan profile remains usable."); c.add("ordinary-agent-cannot-change-instance-gate", !!o && o.agentSettingsStatus === 403 && o.gateAfterProbe === false - && o.createChatStatus === 404, "The real ordinary agent cannot enable instance Chat, and the public create-chat path enforces the same disabled gate."); + && o.createChatStatus === 404 && o.createChatErrorCategory === "agent-chat-disabled" && o.createChatProbePersonId === o.personId, "The real ordinary agent cannot enable instance Chat, and the public create-chat path enforces the same disabled gate."); c.add("no-disguised-task-or-admission", !!o && Array.isArray(o.beforeTaskIds) && Array.isArray(o.afterTaskIds) && JSON.stringify(o.beforeTaskIds) === JSON.stringify(o.afterTaskIds) && Array.isArray(o.chatIds) && !o.chatIds.length && Array.isArray(o.runIds) && !o.runIds.length, @@ -63,7 +73,7 @@ export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{ const denied = await fetch(new URL("/api/instance/settings/experimental", api.baseURL), { method: "PATCH", redirect: "error", signal: AbortSignal.timeout(15_000), headers: { Authorization: `Bearer ${key.token}`, "content-type": "application/json" }, body: JSON.stringify({ enableAgentChat: true }) }); const agentSettingsStatus = denied.status; await denied.body?.cancel(); - const rejected = await api.request.post(`${root}/chats/${fixtures.agent.id}`, { data: {} }); const createChatStatus = rejected.status(); await rejected.dispose(); + const createChatProbe = await probeDisabledChatAsPerson(person, fixtures.company.id, fixtures.agent.id); const gateAfterProbe = (await api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental")).enableAgentChat; await page.goto(`/${fixtures.company.issuePrefix}/agents/${fixtures.agent.id}`, { waitUntil: "domcontentloaded" }); await page.reload({ waitUntil: "domcontentloaded" }); await expect(page.getByRole("heading", { name: "Morgan", exact: true }).first()).toBeVisible(); @@ -72,7 +82,7 @@ export const runtimeDisabledChatCases: readonly IdentityBoundaryCase[] = [{ ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds }); const observation: DisabledObservation = { companyId: fixtures.company.id, agentId: fixtures.agent.id, personId: person.id, canonicalIntent: ctx.scenario.query, agentChatEnabled: settings.enableAgentChat, unavailableVisible, composerAbsent: true, supportedOperatorStepVisible, profileAccessible: true, - agentSettingsStatus, createChatStatus, gateAfterProbe, beforeTaskIds, afterTaskIds: tasks.map(row => row.id).sort(), + agentSettingsStatus, ...createChatProbe, gateAfterProbe, beforeTaskIds, afterTaskIds: tasks.map(row => row.id).sort(), chatIds: tasks.filter(row => row.conversationAgentId).map(row => row.id), runIds }; await ctx.evidence("runtime-disabled-chat-boundary.json", observation); return observation; },