From 0f95be3f07b6396291b73f4704ef55a0eb9a482f Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 18:48:58 -0500 Subject: [PATCH] test(evals): author real expired-session browser boundary --- .../chat-catalog-domains/identity-auth.ts | 20 +++- .../identity-sign-in.test.ts | 24 +++++ .../chat-catalog-domains/identity-sign-in.ts | 96 +++++++++++++++++++ 3 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-sign-in.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-auth.ts b/tests/runner-e2e/chat-catalog-domains/identity-auth.ts index a52b8379dd8..a288aef0f99 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-auth.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-auth.ts @@ -1,5 +1,5 @@ import { randomBytes, randomUUID } from "node:crypto"; -import type { Browser, BrowserContext } from "@playwright/test"; +import { expect, type Browser, type BrowserContext, type Page } from "@playwright/test"; import type { RunnerApi } from "../api.js"; /** Credentials remain in this closure and the attempt redactor, never case state/evidence. @@ -39,6 +39,24 @@ export async function createAuthenticatedPerson(input: { ...(data === undefined ? {} : { "content-type": "application/json" }) }, ...(data === undefined ? {} : { body: JSON.stringify(data) }) }); } return { id: user.id, name: user.name, context, request, + bindApiSession(api: RunnerApi) { + if (api.baseURL !== input.api.baseURL || !cookie) throw new Error("Public human API binding requires the same authenticated instance"); + api.setBrowserSession(cookie); + }, + /** Server revocation leaves the old browser cookie in place so the next + * navigation exercises an actually invalid session rather than a fake UI flag. */ + async expireSession() { await auth("sign-out", {}); cookie = ""; }, + async signInThroughBrowser(page: Page, returnPath: string) { + if (page.context() !== context || !returnPath.startsWith("/") || returnPath.startsWith("//")) throw new Error("Human sign-in target must be the person's same-origin browser"); + await page.getByLabel("Email", { exact: true }).fill(email); + await page.getByLabel("Password", { exact: true }).fill(password); + await page.getByRole("button", { name: "Sign In", exact: true }).click(); + await expect(page).toHaveURL(new URL(returnPath, input.api.baseURL).href); + const cookies = await context.cookies(input.api.baseURL); + cookie = cookies.map(value => `${value.name}=${value.value}`).join("; "); + for (const value of cookies) input.secrets.push(value.value); + if (!cookie) throw new Error("Human browser sign-in returned no session"); + }, async signOut() { await auth("sign-out", {}); cookie = ""; await context.clearCookies(); }, async signIn() { await auth("sign-in/email", { email, password }); await installSession(context); }, async dispose() { try { if (cookie) await auth("sign-out", {}); } finally { cookie = ""; await context.close(); } }, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-sign-in.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-sign-in.test.ts new file mode 100644 index 00000000000..c54a73493fb --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-sign-in.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { gradeSignInBoundary, signInIdentityBoundaryCases } from "./identity-sign-in.js"; +function valid() { return { personId: "alex", returnedPersonId: "alex", companyId: "acorn", conversationId: "standing-chat", returnedConversationId: "standing-chat", + beforeComposerVisible: true, expiredReadStatus: 401, expiredComposerAbsent: true, configuredSignInVisible: true, + safeReturnPath: "/ACO/chats/morgan", intendedReturnPath: "/ACO/chats/morgan", restoredComposerVisible: true, + nonmemberCompanyStatus: 403, nonmemberConversationStatus: 404, nonmemberBrowserBlocked: true, + beforeCommentIds: [], afterCommentIds: [], runIds: [] }; } +describe("real sign-in boundary acceptance", () => { + it("accepts complete real auth+same-resource+audience evidence", () => { expect(gradeSignInBoundary(valid()).passed).toBe(true); }); + it.each([ + { expiredReadStatus: 200 }, { returnedPersonId: "riley" }, { returnedConversationId: "new-conversation" }, + { safeReturnPath: "https://other.test/steal" }, { nonmemberConversationStatus: 200 }, { nonmemberBrowserBlocked: false }, + { runIds: ["accidental-run"] }, { afterCommentIds: ["sent-expired-intent"] }, { restoredComposerVisible: false }, + ])("rejects incomplete or wrong boundary proof %j", change => { expect(gradeSignInBoundary({ ...valid(), ...change }).passed).toBe(false); }); + it("fails closed on a sign-in-screen-only claim or missing evidence", () => { + expect(gradeSignInBoundary({ configuredSignInVisible: true }).passed).toBe(false); + expect(gradeSignInBoundary(undefined).passed).toBe(false); + }); + it("declares the pre-chat journey independently with zero query/run admissions", () => { + expect(signInIdentityBoundaryCases[0]).toMatchObject({ scenarioId: "access.sign-in", journeyKind: "browser-boundary", boundaryTarget: "authenticated-harness", + authentication: "human-session", expectedRunCount: 0, expectedHumanMessageCount: 0 }); + expect(signInIdentityBoundaryCases[0]!.coverageGaps ?? []).toEqual([]); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts b/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts new file mode 100644 index 00000000000..3558d396cbc --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-sign-in.ts @@ -0,0 +1,96 @@ +import { mkdir } from "node:fs/promises"; +import path from "node:path"; +import { expect } from "@playwright/test"; +import { RunnerApi } from "../api.js"; +import { setupLiveFixtures, type LiveFixtureValues } from "../live-fixtures.js"; +import type { IdentityBoundaryCase, IdentityBoundaryContext } from "./identity-boundary-contract.js"; +import { checks } from "./internal-support.js"; +import { createAuthenticatedPerson } from "./identity-auth.js"; + +type Person = Awaited>; +interface Owned { person: Person; api: RunnerApi; other?: Person; companyId?: string; fixtures?: LiveFixtureValues } +const owned = new WeakMap(); +interface SignInEvidence { + personId: string; returnedPersonId: string; companyId: string; conversationId: string; returnedConversationId: string; + beforeComposerVisible: boolean; expiredReadStatus: number; expiredComposerAbsent: boolean; + configuredSignInVisible: boolean; safeReturnPath: string; intendedReturnPath: string; restoredComposerVisible: boolean; + nonmemberCompanyStatus: number; nonmemberConversationStatus: number; nonmemberBrowserBlocked: boolean; + beforeCommentIds: string[]; afterCommentIds: string[]; runIds: string[]; +} +export function gradeSignInBoundary(value: unknown) { + const c = checks(); const o = value as SignInEvidence | undefined; + c.add("real-invalidated-human-session", !!o && !!o.personId && o.personId !== "local-board" && o.beforeComposerVisible + && o.expiredReadStatus === 401 && o.expiredComposerAbsent, "A previously usable real human session is revoked by the server; the expired browser cannot read or compose in its conversation."); + c.add("configured-safe-sign-in-return", !!o && o.configuredSignInVisible && typeof o.safeReturnPath === "string" && o.safeReturnPath === o.intendedReturnPath + && o.safeReturnPath.startsWith("/") && !o.safeReturnPath.startsWith("//") && o.returnedPersonId === o.personId + && o.returnedConversationId === o.conversationId && o.restoredComposerVisible, + "The ordinary configured sign-in form authenticates the same person and returns to the same real company conversation."); + c.add("authentication-retains-resource-checks", !!o && [403, 404].includes(o.nonmemberCompanyStatus) + && [403, 404].includes(o.nonmemberConversationStatus) && o.nonmemberBrowserBlocked, + "A different genuinely signed-in nonmember remains unable to access either the company or the original conversation."); + c.add("no-expired-query-or-dispatch", !!o && Array.isArray(o.runIds) && o.runIds.length === 0 + && Array.isArray(o.beforeCommentIds) && Array.isArray(o.afterCommentIds) + && JSON.stringify(o.beforeCommentIds) === JSON.stringify(o.afterCommentIds), + "The boundary retains unsent intent: no native run or extra human/agent comment is admitted before or after authentication."); + return c.result(); +} +export const signInIdentityBoundaryCases: readonly IdentityBoundaryCase[] = [{ + scenarioId: "access.sign-in", journeyKind: "browser-boundary", boundaryTarget: "authenticated-harness", authentication: "human-session", + oracleVersion: "paperclip.chat-catalog.identity.v1", expectedRunCount: 0, expectedHumanMessageCount: 0, turnTimeoutMs: 120_000, attemptTimeoutMs: 300_000, + sourceFiles: ["chat-catalog-domains/identity-sign-in.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/identity-boundary-contract.ts", "chat-catalog-domains/internal-support.ts", "live-fixtures.ts", "chat-scenarios/access.yaml"], + grade: gradeSignInBoundary, + async run(ctx) { + const browser = ctx.page.context().browser(); if (!browser) throw new Error("Real human boundary needs a browser"); + const person = await createAuthenticatedPerson({ api: ctx.api, browser, secrets: ctx.privateSecrets, name: "Alex" }); + const api = new RunnerApi(person.context.request); person.bindApiSession(api); const resources: Owned = { person, api }; owned.set(ctx, resources); + // The isolated installation's first real account follows the public operator + // claim, exactly as the authenticated Product E2E harness does. This does not + // grant any later observer or agent an instance-admin identity. + await api.post("/api/bootstrap/claim"); + await api.patch("/api/instance/settings/experimental", { enableAgentChat: true }); + const workspacePath = path.join(ctx.privateDir, "sign-in-workspace"); await mkdir(workspacePath, { recursive: true, mode: 0o700 }); + const fixtures = await setupLiveFixtures({ api, execution: ctx.execution, executionNonce: ctx.nonce, workspacePath, credentials: ctx.credentials, + async onCompanyCreated(id) { resources.companyId = id; ctx.observeResources({ companyId: id, runIds: [] }); } }); + resources.fixtures = fixtures; + const root = `/api/companies/${fixtures.company.id}`; + const conversation = await api.post<{ id: string }>(`${root}/chats/${fixtures.agent.id}`); + ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds: [] }); + const intendedReturnPath = `/${fixtures.company.issuePrefix}/chats/${fixtures.agent.id}`; + const page = await person.context.newPage(); + await page.goto(intendedReturnPath, { waitUntil: "domcontentloaded" }); + await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + const beforeCommentIds = (await api.get>(`/api/issues/${conversation.id}/comments`)).map(row => row.id).sort(); + await person.expireSession(); + const expired = await person.context.request.get(`${api.baseURL}/api/issues/${conversation.id}`); const expiredReadStatus = expired.status(); await expired.dispose(); + await page.reload({ waitUntil: "domcontentloaded" }); + await expect(page.getByRole("heading", { name: "Sign in to Paperclip", exact: true })).toBeVisible(); + const safeReturnPath = new URL(page.url()).searchParams.get("next") ?? ""; + const expiredComposerAbsent = await page.getByTestId("task-chat-composer-input").count() === 0; + await person.signInThroughBrowser(page, intendedReturnPath); person.bindApiSession(api); + await expect(page.getByTestId("task-chat-composer-input")).toBeVisible(); + const returned = await api.get<{ user: { id: string } }>("/api/auth/get-session"); + const same = await api.get<{ id: string }>(`${root}/chats/${fixtures.agent.id}`); + const other = await createAuthenticatedPerson({ api, browser, secrets: ctx.privateSecrets, name: "Unenrolled Riley" }); resources.other = other; + const deniedCompany = await other.request(root); const nonmemberCompanyStatus = deniedCompany.status; await deniedCompany.body?.cancel(); + const deniedConversation = await other.request(`/api/issues/${conversation.id}`); const nonmemberConversationStatus = deniedConversation.status; await deniedConversation.body?.cancel(); + const otherPage = await other.context.newPage(); await otherPage.goto(intendedReturnPath, { waitUntil: "domcontentloaded" }); + await expect(otherPage.getByRole("heading", { name: "No organization access", exact: true })).toBeVisible(); + const runIds = (await api.get>(`${root}/heartbeat-runs?limit=100`)).map(row => row.id); + ctx.observeResources({ companyId: fixtures.company.id, agentId: fixtures.agent.id, runIds }); + const observation: SignInEvidence = { personId: person.id, returnedPersonId: returned.user.id, companyId: fixtures.company.id, conversationId: conversation.id, + returnedConversationId: same.id, beforeComposerVisible: true, expiredReadStatus, expiredComposerAbsent, configuredSignInVisible: true, + safeReturnPath, intendedReturnPath, restoredComposerVisible: true, nonmemberCompanyStatus, nonmemberConversationStatus, nonmemberBrowserBlocked: true, + beforeCommentIds, afterCommentIds: (await api.get>(`/api/issues/${conversation.id}/comments`)).map(row => row.id).sort(), runIds }; + await ctx.evidence("identity-real-sign-in.json", observation); return observation; + }, + async cleanup(ctx) { + const resources = owned.get(ctx); if (!resources) return; + try { + // Reauthenticate privately if an earlier phase stopped with its session + // expired; cleanup must not rely on the positive sign-in assertion. + await resources.person.signIn(); resources.person.bindApiSession(resources.api); + await resources.fixtures?.teardown(); + if (resources.companyId) await resources.api.delete(`/api/companies/${resources.companyId}`, { allowNotFound: true }); + } finally { try { await resources.other?.dispose(); } finally { await resources.person.dispose(); owned.delete(ctx); } } + }, +}];