diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md index 5e2c05192c7..ce71547f4a8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md @@ -260,3 +260,29 @@ calibration. A visible real form rejection is retained as a nonpassing product observation; the harness does not pre-reject an actual rotating-token format or bypass that form. Slack's documented rotating access-token format is described at https://docs.slack.dev/authentication/using-token-rotation/. + +The named human Notion setup cases (`notion-missing`, `return-from-login`, and +`save-and-resume`) use the real legacy catalog OAuth UI and official Notion MCP, +not a supplied bearer connection as a substitute. In addition to the independent +REST workspace/pages/facts above, they require `workspaceName`, +`oauthBrowserSessionEnv` (a selected namespaced reference containing bounded +Playwright storage JSON for the authorized synthetic Notion human), and +`oauthHumanEmailEnv` (that human's private email reference). Only official secure +Notion-domain cookies/storage are accepted. A qualified session must expose the +actual account email, selected workspace, and authorization controls; missing or +changed controls fail explicitly. No password, OTP mailbox, or generic browser +execution fallback is inferred. Notion's [official MCP flow](https://developers.notion.com/guides/mcp/get-started-with-mcp) +grants the selected workspace; a private actual `notion-get-self` call verifies +that workspace after callback, separately from the model's required launch read. + +These cases require the selected owned HTTPS-origin receipt and actual public +sign-in of the same generated fixture person. They do not launch a tunnel or copy +loopback cookies. Full-page provider navigation/consent runs in the finite +standalone private child, with no Test tracing/video/report hooks. The callback +must actually return to the original issue and resolve the exact pending request +for its addressee. Save-and-resume uses actual **Continue to Notion** to persist a +personal draft before provider consent, leaves through supported navigation, +reloads that same draft, then requires its eventual authorization to reuse the +same connection ID. No literal Save & exit control or API-created draft is +invented. All of this remains authored/unmeasured until a reserved execution has +a real authorized disposable Notion session, workspace/pages, and owned origin. diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts index 04c0affc5b1..3129f01023c 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts @@ -20,12 +20,12 @@ export const externalRequirements: Record + typeof s === "string" && /^[a-zA-Z0-9_-]{1,100}$/.test(s); +const notionHost = (host: string) => + /^(?:[a-z0-9-]+\.)*notion\.(?:so|com)$/.test(host); +export function notionStorageMatches( + storage: PublicSignInStorage, + host: (h: string) => boolean, +) { + return ( + !!storage && + Array.isArray(storage.cookies) && + storage.cookies.length > 0 && + storage.cookies.length <= 100 && + storage.cookies.every( + (c) => + !!c && + typeof c.name === "string" && + !!c.name && + typeof c.value === "string" && + c.value.length > 0 && + c.value.length <= 65536 && + c.secure === true && + host(c.domain.replace(/^\./, "")) && + c.path.startsWith("/"), + ) && + Array.isArray(storage.origins) && + storage.origins.length <= 10 && + storage.origins.every((o) => { + try { + const u = new URL(o.origin); + return ( + u.origin === o.origin && + u.protocol === "https:" && + !u.port && + host(u.hostname) && + Array.isArray(o.localStorage) && + o.localStorage.length <= 200 && + o.localStorage.every( + (x) => + typeof x.name === "string" && + typeof x.value === "string" && + x.value.length <= 262144, + ) + ); + } catch { + return false; + } + }) + ); +} +export function validNotionOAuthInput(input: NotionOAuthInput) { + try { + const u = new URL(input.publicOrigin); + return ( + ["save-draft", "authorize"].includes(input.operation) && + u.origin === input.publicOrigin && + u.protocol === "https:" && + !u.username && + !u.password && + !u.port && + [input.companyId, input.humanId, input.agentId, input.prefix].every(id) && + (!input.intentId || id(input.intentId)) && + !!input.intentId === !!input.issueId && + (!input.issueId || id(input.issueId)) && + (!input.draftId || id(input.draftId)) && + /^[a-f0-9-]{32,36}$/i.test(input.workspaceId) && + typeof input.workspaceName === "string" && + input.workspaceName.length > 0 && + input.workspaceName.length <= 200 && + !/[\r\n\0]/.test(input.workspaceName) && + /^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(input.providerEmail) && + input.providerEmail.length <= 254 && + notionStorageMatches(input.notionSession, notionHost) && + notionStorageMatches(input.publicSession, (h) => h === u.hostname) + ); + } catch { + return false; + } +} +/** A self-tool receipt is read privately; no account names, emails or token + * material enters the parent. Match ONLY the documented workspace object. */ +export function notionSelfWorkspaceMatches( + value: unknown, + workspaceId: string, +): boolean { + let v = value; + for (let n = 0; n < 4; n++) { + if (!v || typeof v !== "object") return false; + const r = v as Record; + const workspace = r.workspace as { id?: unknown } | undefined; + const bot = r.bot as { workspace_id?: unknown } | undefined; + if (r.type === "bot" && bot && typeof bot.workspace_id === "string") + return ( + bot.workspace_id.replaceAll("-", "").toLowerCase() === + workspaceId.replaceAll("-", "").toLowerCase() + ); + if (workspace && typeof workspace.id === "string") + return ( + workspace.id.replaceAll("-", "").toLowerCase() === + workspaceId.replaceAll("-", "").toLowerCase() + ); + if (r.result) { + v = r.result; + continue; + } + if ( + Array.isArray(r.content) && + r.content.length === 1 && + r.content[0]?.type === "text" && + typeof r.content[0]?.text === "string" + ) { + try { + v = JSON.parse(r.content[0].text); + continue; + } catch { + return false; + } + } + return false; + } + return false; +} +export function validNotionOAuthReceipt( + value: NotionOAuthReceipt, + input: NotionOAuthInput, +) { + const keys = + "callbackReturned,connectionId,consentCompleted,credentialAbsentAtHandoff,draftSurvivedReload,humanId,intentId,personalDraft,processIsolation,providerHandoff,providerHumanMatched,resumedSameDraft,testRunnerAttached,tracingEnabled,videoEnabled,workspaceMatched"; + return ( + !!value && + Object.keys(value).sort().join(",") === keys && + id(value.connectionId) && + (!input.draftId || input.draftId === value.connectionId) && + value.humanId === input.humanId && + value.intentId === (input.intentId ?? null) && + value.personalDraft === true && + value.providerHandoff === true && + value.credentialAbsentAtHandoff === true && + value.resumedSameDraft === !!input.draftId && + value.processIsolation === true && + value.testRunnerAttached === false && + value.tracingEnabled === false && + value.videoEnabled === false && + (input.operation === "save-draft" + ? value.draftSurvivedReload === true && + !value.consentCompleted && + !value.callbackReturned && + !value.workspaceMatched && + !value.providerHumanMatched + : value.consentCompleted === true && + value.callbackReturned === true && + value.workspaceMatched === true && + value.providerHumanMatched === true) + ); +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts new file mode 100644 index 00000000000..dea1da8298c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts @@ -0,0 +1,360 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { chromium, test } from "@playwright/test"; +import type { + ToolConnection, + ToolCatalogEntry, + ToolConnectionTestCallResult, + ConnectionGrantsResponse, +} from "../../../packages/shared/src/types/tool-access.js"; +import { + validNotionOAuthInput, + notionSelfWorkspaceMatches, + type NotionOAuthInput, + type NotionOAuthReceipt, +} from "./external-notion-oauth-contract.js"; +/** Finite real Notion OAuth operation. Never imported into Playwright Test; + * private redirects/session/API bodies cannot enter its step/report hooks. */ +async function main(): Promise { + let attached = false; + try { + test.info(); + attached = true; + } catch { + /* standalone */ + } + if (attached) throw new Error("Private Notion OAuth must run outside Test"); + const input = JSON.parse( + await readFile(process.argv[2]!, "utf8"), + ) as NotionOAuthInput & { executablePath?: string; channel?: string }; + if (!validNotionOAuthInput(input)) + throw new Error("Invalid private Notion request"); + const browser = await chromium.launch({ + headless: true, + ...(input.executablePath ? { executablePath: input.executablePath } : {}), + ...(input.channel ? { channel: input.channel } : {}), + }); + try { + const storage = path.join(path.dirname(process.argv[2]!), "storage.json"); + await writeFile( + storage, + JSON.stringify({ + cookies: [ + ...input.publicSession.cookies, + ...input.notionSession.cookies, + ], + origins: [ + ...input.publicSession.origins, + ...input.notionSession.origins, + ], + }), + { mode: 0o600 }, + ); + const context = await browser.newContext({ storageState: storage }); + const page = await context.newPage(); + page.setDefaultTimeout(20000); + const api = async (pathname: string): Promise => { + const response = await context.request.get(input.publicOrigin + pathname); + if (!response.ok()) throw new Error("Private public-state read failed"); + return (await response.json()) as T; + }; + const session = await api<{ user: { id: string } }>( + "/api/auth/get-session", + ); + if (session.user?.id !== input.humanId) + throw new Error( + "Public Notion setup person differs from addressed human", + ); + const root = `/api/companies/${input.companyId}/tools`; + if (input.intentId) { + const cards = await api< + Array<{ + id: string; + status: string; + addresseeUserId?: string; + payload?: { serviceSlug: string; requestingAgentId: string }; + }> + >(`/api/issues/${input.issueId}/interactions`); + const card = cards.find((c) => c.id === input.intentId); + if ( + !card || + card.status !== "pending" || + card.addresseeUserId !== input.humanId || + card.payload?.serviceSlug !== "notion" || + card.payload.requestingAgentId !== input.agentId + ) + throw new Error( + "Notion full-page handoff lost its actual addressed source intent", + ); + } + const before = ( + await api<{ connections: ToolConnection[] }>(root + "/connections") + ).connections; + const params = new URLSearchParams({ source: "notion", stage: "setup" }); + if (input.intentId) params.set("intent", input.intentId); + if (input.draftId) params.set("resume", input.draftId); + const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`; + await page.goto(setupURL, { waitUntil: "domcontentloaded" }); + await page + .getByRole("heading", { + name: input.draftId + ? "Finish connecting Notion" + : "Connect Notion to Paperclip", + exact: true, + }) + .waitFor(); + if (!input.draftId) { + await page.getByRole("button", { name: "Change", exact: true }).click(); + await page.getByRole("radio", { name: "Just me", exact: true }).click(); + } + const started = page.waitForResponse( + (r) => + r.url().startsWith(input.publicOrigin + root + "/apps/connect") && + r.request().method() === "POST", + ); + const navigated = page.waitForURL((u) => u.origin !== input.publicOrigin, { + timeout: 90000, + }); + // Source UI starts OAuth directly from a durable resume without connectApp. + if (input.draftId) void started.catch(() => undefined); + await page + .getByRole("button", { + name: input.draftId ? "Finish with Notion" : "Continue to Notion", + exact: true, + }) + .click(); + let connectionId = input.draftId; + if (!connectionId) { + const response = await started; + if (!response.ok()) + throw new Error("Actual Notion draft creation failed"); + connectionId = ((await response.json()) as { connectionId: string }) + .connectionId; + } + if (!connectionId) throw new Error("Actual Notion draft ID missing"); + await navigated; + const provider = new URL(page.url()); + if ( + provider.protocol !== "https:" || + !/^(?:[a-z0-9-]+\.)*notion\.(?:so|com)$/.test(provider.hostname) + ) + throw new Error("Notion handoff escaped official provider"); + const draft = await api( + `/api/tool-connections/${connectionId}`, + ); + const grantState = await api( + `/api/tool-connections/${connectionId}/grants`, + ); + const personal = grantState.grants.filter( + (g) => g.kind === "user" && g.subjectUserId === input.humanId, + ); + if ( + draft.companyId !== input.companyId || + draft.credentialPolicy !== "per_user" || + draft.authKind !== "oauth" || + draft.status !== "draft" || + draft.credentialSecretRefs.length || + draft.credentialRefs?.length || + personal.length !== 1 || + before.some((c) => c.id === connectionId) !== !!input.draftId + ) + throw new Error( + "Notion handoff did not preserve one actual credential-free personal draft", + ); + const receipt: NotionOAuthReceipt = { + connectionId, + humanId: input.humanId, + intentId: input.intentId ?? null, + personalDraft: true, + providerHandoff: true, + credentialAbsentAtHandoff: true, + resumedSameDraft: !!input.draftId, + draftSurvivedReload: false, + consentCompleted: false, + callbackReturned: false, + workspaceMatched: false, + providerHumanMatched: false, + processIsolation: true, + testRunnerAttached: false, + tracingEnabled: false, + videoEnabled: false, + }; + if (input.operation === "save-draft") { + // Leave before any provider consent; supported setup navigation and actual + // reload must hydrate the original durable choices, without a fake draft. + const resume = new URL(setupURL); + resume.searchParams.set("resume", connectionId); + await page.goto(`${input.publicOrigin}/${input.prefix}/apps`, { + waitUntil: "domcontentloaded", + }); + await page.goto(resume.href, { waitUntil: "domcontentloaded" }); + await page.reload(); + await page + .getByRole("button", { name: "Finish with Notion", exact: true }) + .waitFor(); + await page.getByRole("button", { name: "Change", exact: true }).click(); + if ( + !(await page + .getByRole("radio", { name: "Just me", exact: true }) + .isChecked()) + ) + throw new Error("Reloaded Notion draft lost personal identity choice"); + const current = await api( + `/api/tool-connections/${connectionId}`, + ); + const rows = ( + await api<{ connections: ToolConnection[] }>(root + "/connections") + ).connections; + if ( + current.status !== "draft" || + current.credentialPolicy !== "per_user" || + current.credentialSecretRefs.length || + current.credentialRefs?.length || + rows.length !== before.length + (input.draftId ? 0 : 1) + ) + throw new Error( + "Saved Notion draft became usable or duplicated before credentials", + ); + receipt.draftSurvivedReload = true; + return receipt; + } + // Provider UI identity is checked before consent; a session/display name + // alone never proves the workspace that was actually granted. The private + // get-self call below supplies independent exact workspace authority. + await page.getByText(input.providerEmail, { exact: true }).waitFor(); + const workspace = page.getByRole("combobox"); + if ((await workspace.count()) === 1) { + if (await workspace.evaluate((e) => e.tagName === "SELECT")) + await workspace.selectOption({ label: input.workspaceName }); + else { + await workspace.click(); + await page + .getByRole("option", { name: input.workspaceName, exact: true }) + .click(); + } + } + await page + .getByText(input.workspaceName, { exact: true }) + .first() + .waitFor(); + receipt.providerHumanMatched = true; + let originalConversationReturned = false; + page.on("framenavigated", (frame) => { + if (frame !== page.mainFrame() || !input.issueId) return; + const u = new URL(frame.url()); + if ( + u.origin === input.publicOrigin && + [ + `/issues/${input.issueId}`, + `/${input.prefix}/issues/${input.issueId}`, + ].includes(u.pathname) + ) + originalConversationReturned = true; + }); + await page + .getByRole("button", { name: /^(Allow access|Authorize|Allow)$/i }) + .click(); + await page.waitForURL((u) => u.origin === input.publicOrigin, { + timeout: 90000, + }); + if (input.issueId) { + for (let n = 0; n < 40 && !originalConversationReturned; n++) + await new Promise((r) => setTimeout(r, 250)); + if (!originalConversationReturned) + throw new Error( + "Notion callback did not actually return to its original source conversation", + ); + const cards = await api< + Array<{ + id: string; + status: string; + resolvedByUserId?: string; + result?: { outcome: string; connectionId: string }; + }> + >(`/api/issues/${input.issueId}/interactions`); + const card = cards.find((c) => c.id === input.intentId); + if ( + !card || + card.status === "pending" || + card.resolvedByUserId !== input.humanId || + card.result?.outcome !== "connected" || + card.result.connectionId !== connectionId + ) + throw new Error( + "Notion callback did not resolve the exact original addressed conversation request", + ); + } + receipt.callbackReturned = true; + let connected: ToolConnection | undefined; + for (let n = 0; n < 40; n++) { + connected = await api( + `/api/tool-connections/${connectionId}`, + ); + if (connected.status === "active" && connected.healthStatus === "ok") + break; + await new Promise((r) => setTimeout(r, 500)); + } + if ( + !connected || + connected.companyId !== input.companyId || + connected.status !== "active" || + connected.healthStatus !== "ok" || + connected.credentialPolicy !== "per_user" + ) + throw new Error( + "Real Notion callback did not activate the same personal draft", + ); + const catalog = ( + await api<{ catalog: ToolCatalogEntry[] }>( + `/api/tool-connections/${connectionId}/catalog`, + ) + ).catalog; + const self = catalog.filter( + (e) => + e.toolName === "notion-get-self" && + e.isReadOnly && + e.status === "active", + ); + if (self.length !== 1) + throw new Error("Official Notion workspace identity tool is unavailable"); + const testCall = await context.request.post( + `${input.publicOrigin}/api/tool-connections/${connectionId}/test-calls`, + { + data: { + agentId: input.agentId, + toolName: self[0]!.toolName, + parameters: {}, + }, + }, + ); + if (!testCall.ok()) + throw new Error("Independent Notion get-self could not execute"); + const result = (await testCall.json()) as ToolConnectionTestCallResult; + if ( + result.decision !== "allowed" || + result.error || + !notionSelfWorkspaceMatches(result.result, input.workspaceId) + ) + throw new Error( + "Actual granted Notion workspace differs from selected personal fixture", + ); + if ( + (await api<{ user: { id: string } }>("/api/auth/get-session")).user.id !== + input.humanId + ) + throw new Error("Notion callback changed the signed-in person"); + receipt.workspaceMatched = true; + receipt.consentCompleted = true; + return receipt; + } finally { + await browser.close(); + } +} +main() + .then((value) => process.stdout.write(JSON.stringify(value))) + .catch(() => { + process.stderr.write( + "Private Notion OAuth operation failed; provider/session/redirect details withheld\n", + ); + process.exitCode = 1; + }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts new file mode 100644 index 00000000000..9f4c3d49d87 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts @@ -0,0 +1,181 @@ +import { mkdtemp, writeFile, rm, chmod } from "node:fs/promises"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { chromium } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { signInPublicFixturePerson } from "../public-mcp-flow.js"; +import { + readValidatedPublicOriginReceipt, + PUBLIC_ORIGIN_RECEIPT_ENV, +} from "../chat-public-origin.js"; +import { + selectedExternalBinding, + selectedExternalCredential, +} from "./external-fixtures.js"; +import { + validNotionOAuthInput, + validNotionOAuthReceipt, + type NotionOAuthInput, + type NotionOAuthReceipt, +} from "./external-notion-oauth-contract.js"; +/** Selected actual human/provider session only; no generic navigation or private + * credential injection into the instrumented Test browser. */ +export async function privateNotionOAuth( + ctx: ChatCatalogContext, + options: Pick, +) { + const binding = selectedExternalBinding(ctx), + p = binding.prerequisites; + const secrets = ctx.input.privateSecrets; + if (!secrets) + throw new Error( + "Private Notion OAuth requires the owning attempt redaction sink", + ); + const sessionJson = selectedExternalCredential( + ctx, + binding, + "oauthBrowserSessionEnv", + ); + const providerEmail = selectedExternalCredential( + ctx, + binding, + "oauthHumanEmailEnv", + ); + secrets.push(sessionJson, providerEmail); + if (Buffer.byteLength(sessionJson) > 2 * 1024 * 1024) + throw new Error( + "Setup blocker: bounded selected Notion human session is required", + ); + let notionSession: NotionOAuthInput["notionSession"]; + try { + notionSession = JSON.parse(sessionJson); + } catch { + throw new Error( + "Setup blocker: selected private Notion human session JSON is invalid", + ); + } + for (const cookie of notionSession?.cookies ?? []) + if (typeof cookie.value === "string") secrets.push(cookie.value); + for (const origin of notionSession?.origins ?? []) + for (const item of origin.localStorage ?? []) + if (typeof item.value === "string" && item.value) + secrets.push(item.value); + await signInPublicFixturePerson(ctx.input.api, ctx.input.page.context()); + const receiptPath = process.env[PUBLIC_ORIGIN_RECEIPT_ENV], + attemptId = process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID; + const executions = JSON.parse( + process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]", + ) as string[]; + if (!receiptPath || !attemptId || executions.length !== 1) + throw new Error( + "Setup blocker: selected Notion attempt requires its owned validated public origin", + ); + const origin = await readValidatedPublicOriginReceipt({ + receiptPath, + attemptId, + executionId: executions[0]!, + backendOrigin: ctx.input.api.baseURL, + }); + const publicStorage = await ctx.input.page.context().storageState(); + const host = new URL(origin.publicOrigin).hostname; + const publicSession = { + cookies: publicStorage.cookies.filter( + (c) => c.domain.replace(/^\./, "") === host, + ), + origins: publicStorage.origins.filter( + (o) => o.origin === origin.publicOrigin, + ), + }; + const human = await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ); + const input: NotionOAuthInput = { + ...options, + ...(options.intentId ? { issueId: ctx.issue()?.id } : {}), + publicOrigin: origin.publicOrigin, + companyId: ctx.input.fixtures.company.id, + prefix: ctx.input.fixtures.company.issuePrefix ?? "", + humanId: human.user.id, + agentId: ctx.input.fixtures.agent.id, + workspaceId: p.workspaceId!, + workspaceName: p.workspaceName!, + providerEmail, + notionSession, + publicSession, + }; + if (!validNotionOAuthInput(input)) + throw new Error( + "Setup blocker: Notion human/workspace/session prerequisite is outside the finite named-provider contract", + ); + const directory = await mkdtemp( + path.join(tmpdir(), "paperclip-notion-oauth-"), + ); + await chmod(directory, 0o700); + try { + const request = path.join(directory, "request.json"), + channel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL; + await writeFile( + request, + JSON.stringify({ + ...input, + executablePath: + process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ?? + (channel ? undefined : chromium.executablePath()), + channel, + }), + { mode: 0o600 }, + ); + let stdout: string; + try { + stdout = ( + await promisify(execFile)( + process.execPath, + [ + "--import", + fileURLToPath( + new URL( + "../../../server/node_modules/tsx/dist/loader.mjs", + import.meta.url, + ), + ), + fileURLToPath( + new URL("./external-notion-oauth-worker.ts", import.meta.url), + ), + request, + ], + { + timeout: 240000, + maxBuffer: 16384, + env: { + PATH: process.env.PATH ?? "", + HOME: directory, + TMPDIR: directory, + NODE_ENV: "test", + DO_NOT_TRACK: "1", + }, + }, + ) + ).stdout; + } catch { + throw new Error( + "Private named Notion setup failed; session/provider/redirect details withheld", + ); + } + let receipt: NotionOAuthReceipt; + try { + receipt = JSON.parse(stdout); + } catch { + throw new Error("Private Notion setup did not return its closed receipt"); + } + if (!validNotionOAuthReceipt(receipt, input)) + throw new Error( + "Private Notion setup receipt failed its exact identity/draft/isolation contract", + ); + return receipt; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts new file mode 100644 index 00000000000..d0e56c65b27 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts @@ -0,0 +1,330 @@ +import { describe, expect, it } from "vitest"; +import { + gradeNotionSetup, + notionSetupExternalCases, +} from "./external-notion-setup.js"; +import { + validNotionOAuthInput, + validNotionOAuthReceipt, + notionSelfWorkspaceMatches, + type NotionOAuthInput, + type NotionOAuthReceipt, +} from "./external-notion-oauth-contract.js"; +import type { ExternalObservation } from "./external-support.js"; +const workspaceId = "11111111-1111-4111-8111-111111111111", + pageId = "22222222-2222-4222-8222-222222222222"; +const storage = (domain: string) => ({ + cookies: [ + { + name: "session", + value: "synthetic-private-session", + domain, + path: "/", + secure: true, + }, + ], + origins: [], +}); +const input: NotionOAuthInput = { + operation: "authorize", + publicOrigin: "https://owned.example.test", + companyId: "company", + prefix: "ACORN", + humanId: "person", + agentId: "ceo", + intentId: "intent", + issueId: "source", + workspaceId, + workspaceName: "Synthetic Launch", + providerEmail: "synthetic@example.test", + publicSession: storage("owned.example.test"), + notionSession: storage(".notion.so"), +}; +function receipt(): NotionOAuthReceipt { + return { + connectionId: "connection", + humanId: "person", + intentId: "intent", + personalDraft: true, + credentialAbsentAtHandoff: true, + providerHandoff: true, + resumedSameDraft: false, + draftSurvivedReload: false, + consentCompleted: true, + callbackReturned: true, + workspaceMatched: true, + providerHumanMatched: true, + processIsolation: true, + testRunnerAttached: false, + tracingEnabled: false, + videoEnabled: false, + }; +} +function valid(id: string): ExternalObservation { + return { + scenarioId: id, + companyId: "company", + leadId: "ceo", + origin: "http://127.0.0.1:3100", + state: { + intentId: "intent", + initialRunId: "initial", + humanId: "person", + pendingBeforeHuman: true, + cardVisible: true, + pendingWithoutCredential: true, + oauthReceipt: receipt(), + connectionId: "connection", + samePersonalActive: true, + currentHumanPreserved: true, + originalSourcePreserved: true, + sourceIssueId: "source", + readEntryIds: ["read"], + decisions: [ + { + runId: "continuation", + decisions: [ + { + invocation: { + runId: "continuation", + agentId: "ceo", + issueId: "source", + connectionId: "connection", + catalogEntryId: "read", + status: "succeeded", + }, + }, + ], + }, + ], + pages: [ + { + id: pageId, + editedAt: "2026-10-26T08:00:00.000Z", + fact: { + subjectAliases: ["launch", "release"], + valueAliases: ["November 8", "8 November"], + }, + }, + ], + pagesUnchanged: true, + }, + reply: "As of Oct 26, 2026, the release is now set for 8 November.", + phaseReplies: [ + { + runId: "initial", + body: "Please connect your personal Notion with the secure setup card; I cannot read it until you authorize.", + }, + { + runId: "continuation", + commentId: "comment", + body: "As of Oct 26, 2026, the release is now set for 8 November.", + }, + ], + connections: [{ id: "connection" }], + interactions: [ + { + id: "intent", + kind: "connection_intent", + sourceRunId: "initial", + addresseeUserId: "person", + resolvedByUserId: "person", + result: { outcome: "connected", connectionId: "connection" }, + payload: { serviceSlug: "notion" }, + }, + ], + profiles: [], + secrets: [], + endpoints: [], + activity: [], + links: [`https://www.notion.so/Launch-${pageId}?source=chat`], + runs: [ + { id: "initial", status: "succeeded" }, + { id: "continuation", status: "succeeded" }, + ], + humanQueries: [], + } as unknown as ExternalObservation; +} +describe("named Notion human setup controls", () => { + it("accepts actual personal callback plus a new source-bound body read", () => { + for (const id of [ + "connection-setup.notion-missing", + "connection-setup.return-from-login", + ]) + expect(gradeNotionSetup(valid(id)).passed).toBe(true); + }); + it("accepts actual leave/reload/resume of the same credential-free draft", () => { + const o = valid("connection-setup.save-and-resume"); + o.state.savedDraft = { + ...receipt(), + intentId: null, + consentCompleted: false, + callbackReturned: false, + workspaceMatched: false, + providerHumanMatched: false, + draftSurvivedReload: true, + }; + o.state.savedBeforeCanonical = true; + (o.state.oauthReceipt as NotionOAuthReceipt).resumedSameDraft = true; + expect(gradeNotionSetup(o).passed).toBe(true); + }); + const wrong: Array<[string, (o: ExternalObservation) => void]> = [ + [ + "wrong addressed human", + (o) => { + o.state.humanId = "other"; + }, + ], + [ + "premature completed summary", + (o) => { + o.phaseReplies[0]!.body = + "I've already reviewed Notion. The launch is November 8."; + }, + ], + [ + "provider refusal alone", + (o) => { + o.state.oauthReceipt = undefined; + o.state.decisions = []; + o.phaseReplies[1]!.body = "Notion is unavailable; I could not read it."; + }, + ], + [ + "wrong granted workspace", + (o) => { + (o.state.oauthReceipt as NotionOAuthReceipt).workspaceMatched = false; + }, + ], + [ + "another callback user", + (o) => { + (o.state.oauthReceipt as NotionOAuthReceipt).humanId = "other"; + }, + ], + [ + "duplicate connection", + (o) => { + o.connections.push({ id: "duplicate" }); + }, + ], + [ + "oracle-only read", + (o) => { + o.state.decisions = []; + }, + ], + [ + "read on wrong source", + (o) => { + (o.state.decisions as any[])[0].decisions[0].invocation.issueId = + "unrelated"; + }, + ], + [ + "stale or title-only summary", + (o) => { + o.phaseReplies[1]!.body = "The launch page title looks promising."; + }, + ], + [ + "wrong update date", + (o) => { + o.phaseReplies[1]!.body = "On Oct 25, 2026 the launch is November 8."; + }, + ], + [ + "wrong page identity", + (o) => { + o.links = ["https://www.notion.so/33333333333343338333333333333333"]; + }, + ], + ]; + for (const [name, mutate] of wrong) + it(`rejects ${name}`, () => { + const o = valid("connection-setup.notion-missing"); + mutate(o); + expect(gradeNotionSetup(o).passed).toBe(false); + }); + it("rejects a fabricated or replaced resume draft", () => { + const o = valid("connection-setup.save-and-resume"); + o.state.savedDraft = { ...receipt(), connectionId: "other" }; + o.state.savedBeforeCanonical = true; + expect(gradeNotionSetup(o).passed).toBe(false); + }); + it("keeps exact concrete callbacks and phase bounds", () => { + expect(notionSetupExternalCases.map((c) => c.scenarioId)).toHaveLength(3); + for (const c of notionSetupExternalCases) { + expect( + [c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every( + (f) => typeof f === "function", + ), + ).toBe(true); + expect(c.expectedRunCount).toBe(2); + expect(c.expectedHumanMessageCount).toBe(1); + expect(c.coverageGaps).toBeUndefined(); + } + }); + it("accepts only isolated selected official provider/public sessions", () => { + expect(validNotionOAuthInput(input)).toBe(true); + for (const change of [ + { publicOrigin: "http://owned.example.test" }, + { publicSession: storage("127.0.0.1") }, + { notionSession: storage("notion.so.attacker.test") }, + { workspaceName: "Synthetic\nLaunch" }, + { providerEmail: "not-an-email" }, + ]) + expect(validNotionOAuthInput({ ...input, ...change })).toBe(false); + }); + it("requires actual same-user closed callback receipt", () => { + expect(validNotionOAuthReceipt(receipt(), input)).toBe(true); + for (const change of [ + { callbackReturned: false }, + { processIsolation: false }, + { humanId: "other" }, + { intentId: "wrong" }, + { providerHumanMatched: false }, + { workspaceMatched: false }, + { extra: "private" }, + ]) + expect( + validNotionOAuthReceipt( + { ...receipt(), ...change } as NotionOAuthReceipt, + input, + ), + ).toBe(false); + }); + it("matches only actual workspace authority, never a coincidental nested ID", () => { + expect( + notionSelfWorkspaceMatches( + { + result: { + content: [ + { + type: "text", + text: JSON.stringify({ + type: "bot", + bot: { workspace_id: workspaceId }, + }), + }, + ], + }, + }, + workspaceId, + ), + ).toBe(true); + expect( + notionSelfWorkspaceMatches( + { workspace: { id: workspaceId.replaceAll("-", "") } }, + workspaceId, + ), + ).toBe(true); + for (const result of [ + { unrelated: { id: workspaceId } }, + { workspace: { id: pageId } }, + { type: "bot", bot: {} }, + { content: [{ type: "text", text: "Notion connected" }] }, + ]) + expect(notionSelfWorkspaceMatches(result, workspaceId)).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts new file mode 100644 index 00000000000..b8b8756a6ad --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts @@ -0,0 +1,442 @@ +import { expect } from "@playwright/test"; +import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js"; +import type { + ToolConnection, + ConnectionGrantsResponse, + ToolRunDecisionLookup, + ToolProfileEffectiveSummary, +} from "../../../packages/shared/src/types/tool-access.js"; +import { pollUntil } from "../api.js"; +import { externalRequirements } from "./external-fixtures.js"; +import { privateNotionOAuth } from "./external-notion-oauth.js"; +import type { NotionOAuthReceipt } from "./external-notion-oauth-contract.js"; +import { + readNotionFixturePages, + type NotionPageOracle, +} from "./external-notion-provider.js"; +import { + notionDateMentioned, + notionFactSpan, + notionPageIdentityMatches, +} from "./external-notion-semantics.js"; +import { + checks, + definition, + externalObservation, + observeExternal, + root, + verifyRenderedReply, +} from "./external-support.js"; + +export function gradeNotionSetup(value: unknown) { + const c = checks(), + o = externalObservation(value); + c.add( + "actual-personal-notion-setup-evidence", + !!o, + "Actual named-provider setup, pending request and provider-native read evidence are required.", + ); + if (!o) return c.result(); + const card = o.interactions.find( + (row) => row.id === o.state.intentId, + ) as unknown as ConnectionIntentInteraction | undefined; + const receipt = o.state.oauthReceipt as NotionOAuthReceipt | undefined; + const initial = o.phaseReplies + .filter((row) => row.runId === o.state.initialRunId) + .map((row) => row.body) + .join("\n"); + const continuation = o.phaseReplies + .filter((row) => row.runId !== o.state.initialRunId) + .map((row) => row.body) + .join("\n"); + const pages = o.state.pages as NotionPageOracle[] | undefined, + recent = pages?.[0], + older = pages?.[1]; + c.add( + "single-addressed-pending-request", + !!card && + o.interactions.filter((row) => row.kind === "connection_intent") + .length === 1 && + card.sourceRunId === o.state.initialRunId && + card.addresseeUserId === o.state.humanId && + o.state.pendingBeforeHuman === true && + o.state.cardVisible === true && + o.state.pendingWithoutCredential === true, + "The original model's addressed request remains pending without credentials; no hidden replacement or premature resolution.", + ); + c.add( + "no-premature-unseen-summary", + !!recent && + !notionFactSpan(initial, recent.fact) && + (!older || !notionFactSpan(initial, older.fact)) && + !/\b(?:I(?: have|'ve|’ve)? (?:already )?(?:read|reviewed|summarized)|here(?: is|'s|’s) (?:the |your )?(?:latest|launch summary))/i.test( + initial, + ), + "The setup reply cannot claim research or substitute unseen page facts before human authorization.", + ); + c.add( + "actual-human-provider-consent", + !!receipt && + receipt.humanId === o.state.humanId && + receipt.intentId === card?.id && + receipt.personalDraft === true && + receipt.credentialAbsentAtHandoff === true && + receipt.providerHandoff === true && + receipt.consentCompleted === true && + receipt.callbackReturned === true && + receipt.workspaceMatched === true && + receipt.providerHumanMatched === true && + receipt.processIsolation === true && + receipt.testRunnerAttached === false && + receipt.tracingEnabled === false && + receipt.videoEnabled === false, + "The addressed fixture person actually completes full-page named Notion consent and returns; private get-self independently proves the selected workspace.", + ); + c.add( + "same-usable-personal-connection", + !!card && + card.result?.outcome === "connected" && + card.result.connectionId === receipt?.connectionId && + card.resolvedByUserId === o.state.humanId && + !card.resolvedByAgentId && + o.state.connectionId === receipt?.connectionId && + o.connections.length === 1 && + o.state.samePersonalActive === true && + o.state.currentHumanPreserved === true && + o.state.originalSourcePreserved === true, + "Only the original personal draft becomes usable; the actual callback resolves the same request/user/conversation with one account.", + ); + const decisions = o.state.decisions as ToolRunDecisionLookup[] | undefined; + c.add( + "new-canonical-continuation-read", + !!decisions?.some( + (row) => + row.runId !== o.state.initialRunId && + o.runs.some( + (run) => run.id === row.runId && run.status === "succeeded", + ) && + row.decisions.some( + (d) => + d.invocation.runId === row.runId && + d.invocation.agentId === o.leadId && + d.invocation.issueId === o.state.sourceIssueId && + d.invocation.connectionId === o.state.connectionId && + d.invocation.status === "succeeded" && + (o.state.readEntryIds as string[] | undefined)?.includes( + d.invocation.catalogEntryId ?? "", + ), + ), + ), + "A new native continuation actually reads through the newly authorized Notion connection; the independent get-self/oracle read is insufficient.", + ); + c.add( + "current-body-date-and-source", + !!recent && + !!notionFactSpan(continuation, recent.fact) && + notionDateMentioned(continuation, recent.editedAt) && + o.links.some((link) => notionPageIdentityMatches(link, recent.id)) && + (!older || + !notionFactSpan(continuation, older.fact) || + /older|background|previous|earlier/i.test(continuation)) && + o.state.pagesUnchanged === true, + "The continuation summarizes the actual recent body fact with its real date/source, distinguishing older background.", + ); + if (o.scenarioId === "connection-setup.save-and-resume") { + const saved = o.state.savedDraft as NotionOAuthReceipt | undefined; + c.add( + "real-human-saved-uncredentialed-draft", + !!saved && + saved.personalDraft === true && + saved.credentialAbsentAtHandoff === true && + saved.providerHandoff === true && + saved.draftSurvivedReload === true && + !saved.consentCompleted && + !saved.callbackReturned && + saved.intentId === null && + saved.connectionId === receipt?.connectionId && + receipt?.resumedSameDraft === true && + o.state.savedBeforeCanonical === true, + "Before the canonical query, actual Continue to Notion saves personal choices; the human leaves before credentials, reloads, and later authorizes that same durable draft.", + ); + } + return c.result(); +} +const sourceFiles = [ + "chat-catalog-domains/external-notion-setup.ts", + "chat-catalog-domains/external-notion-oauth.ts", + "chat-catalog-domains/external-notion-oauth-worker.ts", + "chat-catalog-domains/external-notion-oauth-contract.ts", + "chat-catalog-domains/external-notion-provider.ts", + "chat-catalog-domains/external-notion-semantics.ts", + "chat-catalog-domains/external-public-signin.ts", + "chat-catalog-domains/external-public-signin-worker.ts", + "chat-catalog-domains/external-public-signin-contract.ts", + "public-mcp-flow.ts", + "chat-public-origin.ts", + "chat-catalog-domains/external-fixtures.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/connection-setup.yaml", + "../../ui/src/features/connections/ConnectionSetupFlow.tsx", + "../../ui/src/features/connections/ConnectionIntentInteractionBody.tsx", + "../../server/src/routes/tool-access.ts", + "../../server/src/services/tool-access/oauth/direct-callback.ts", + "../../scripts/smoke/notion-generic-live.mjs", +]; +export const notionSetupExternalCases = [ + "connection-setup.notion-missing", + "connection-setup.return-from-login", + "connection-setup.save-and-resume", +].map((id) => + definition( + id, + async (ctx) => { + const connections = ( + await ctx.input.api.get<{ connections: ToolConnection[] }>( + `${root(ctx)}/tools/connections`, + ) + ).connections; + if (connections.length) + throw new Error( + "No-connection Notion setup requires a fresh disposable company", + ); + const pages = await readNotionFixturePages(ctx); + const humanId = ( + await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ) + ).user.id; + const state: Record = { + pages, + humanId, + baselineConnectionIds: [], + savedBeforeCanonical: false, + }; + if (id === "connection-setup.save-and-resume") { + try { + state.savedDraft = await privateNotionOAuth(ctx, { + operation: "save-draft", + }); + state.savedBeforeCanonical = true; + } catch (error) { + // Every connection in this newly owned disposable company was created by + // this one finite UI operation. Do not leave an unreturned private draft. + for (const row of ( + await ctx.input.api.get<{ connections: ToolConnection[] }>( + `${root(ctx)}/tools/connections`, + ) + ).connections) + await ctx.input.api.delete(`/api/tool-connections/${row.id}`, { + allowNotFound: true, + }); + throw error; + } + } + return state; + }, + gradeNotionSetup, + { + authentication: "human-session", + externalInput: externalRequirements[id], + expectedRunCount: 2, + minimumRunCount: 2, + expectedHumanMessageCount: 1, + turnTimeoutMs: 120000, + attemptTimeoutMs: 900000, + sourceFiles, + async act(ctx, state) { + const rows = await pollUntil({ + label: "actual addressed pending Notion request", + deadlineAt: Date.now() + 150000, + load: async () => { + await ctx.refreshIssue(); + return ctx.issue() + ? ctx.input.api.get( + `/api/issues/${ctx.issue()!.id}/interactions`, + ) + : []; + }, + accept: (rows) => + rows.some( + (row) => + row.kind === "connection_intent" && + row.status === "pending" && + row.payload.serviceSlug === "notion", + ), + }); + const pending = rows.filter( + (row) => + row.kind === "connection_intent" && + row.status === "pending" && + row.payload.serviceSlug === "notion", + ); + if ( + pending.length !== 1 || + pending[0]!.addresseeUserId !== state.humanId || + pending[0]!.payload.requestingAgentId !== ctx.input.fixtures.agent.id + ) + throw new Error( + "Notion request is not uniquely addressed to the actual fixture person/requesting CEO", + ); + const card = pending[0]!; + state.intentId = card.id; + state.initialRunId = card.sourceRunId; + state.sourceIssueId = ctx.issue()!.id; + state.pendingBeforeHuman = true; + await expect( + ctx.input.page + .getByRole("button", { + name: /^(Connect|Connect \/ Use existing)$/, + exact: true, + }) + .first(), + ).toBeVisible(); + state.cardVisible = true; + const before = ( + await ctx.input.api.get<{ connections: ToolConnection[] }>( + `${root(ctx)}/tools/connections`, + ) + ).connections; + const saved = state.savedDraft as NotionOAuthReceipt | undefined; + state.pendingWithoutCredential = + before.length === (saved ? 1 : 0) && + before.every( + (row) => + row.id === saved?.connectionId && + row.status === "draft" && + row.credentialPolicy === "per_user" && + !row.credentialSecretRefs.length && + !row.credentialRefs?.length, + ); + const current = await observeExternal(ctx, state); + const initial = current.phaseReplies + .filter((row) => row.runId === card.sourceRunId) + .map((row) => row.body) + .join("\n"); + const pages = state.pages as NotionPageOracle[]; + if ( + !state.pendingWithoutCredential || + !initial.trim() || + pages.some((page) => notionFactSpan(initial, page.fact)) + ) + throw new Error( + "Notion request was prematurely satisfied or already exposed unseen fixture facts", + ); + // The visible addressed card's supported full-page setup host preserves the + // exact same intent. No API connection or completion substitutes for UI. + state.oauthReceipt = await privateNotionOAuth(ctx, { + operation: "authorize", + intentId: card.id, + ...(saved ? { draftId: saved.connectionId } : {}), + }); + state.connectionId = ( + state.oauthReceipt as NotionOAuthReceipt + ).connectionId; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + const o = await observeExternal(ctx, state), + receipt = state.oauthReceipt as NotionOAuthReceipt | undefined; + if (receipt) { + const connection = await ctx.input.api.get( + `/api/tool-connections/${receipt.connectionId}`, + ); + const grants = await ctx.input.api.get( + `/api/tool-connections/${receipt.connectionId}/grants`, + ); + state.samePersonalActive = + connection.companyId === ctx.input.fixtures.company.id && + connection.status === "active" && + connection.healthStatus === "ok" && + connection.credentialPolicy === "per_user" && + grants.grants.filter( + (g) => + g.kind === "user" && + g.subjectUserId === state.humanId && + g.status === "active", + ).length === 1 && + !grants.grants.some( + (g) => g.kind === "organization" && g.status === "active", + ); + const effective = + await ctx.input.api.get( + `${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`, + ); + const catalog = await ctx.input.api.get<{ + catalog: Array<{ id: string; isReadOnly: boolean }>; + }>(`/api/tool-connections/${receipt.connectionId}/catalog`); + state.readEntryIds = catalog.catalog + .filter( + (e) => + e.isReadOnly && + effective.allowedTools.some((t) => t.id === e.id), + ) + .map((e) => e.id); + } + state.decisions = await Promise.all( + (await ctx.allRuns()).map((run) => + ctx.input.api.get( + `${root(ctx)}/tools/runs/${run.id}/decisions`, + ), + ), + ); + state.currentHumanPreserved = + ( + await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ) + ).user.id === state.humanId; + state.originalSourcePreserved = ctx.issue()?.id === state.sourceIssueId; + const current = await readNotionFixturePages(ctx), + before = state.pages as NotionPageOracle[]; + state.pagesUnchanged = + current.length === before.length && + current.every( + (p, n) => + p.id === before[n]?.id && + p.editedAt === before[n]?.editedAt && + p.marker === before[n]?.marker, + ); + return { ...o, state }; + }, + async verifyResult(ctx, state, value) { + await verifyRenderedReply(ctx, value, String(state.initialRunId)); + const o = externalObservation(value)!, + page = (state.pages as NotionPageOracle[])[0]!; + const href = o.links.find((link) => + notionPageIdentityMatches(link, page.id), + ); + if (!href) + throw new Error( + "Actual current Notion source link absent from resumed summary", + ); + await expect( + ctx.input.page + .locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`) + .first(), + ).toBeVisible(); + await ctx.input.evidence("chat-catalog-notion-human-setup.json", { + scenarioId: ctx.scenario.id, + intentId: state.intentId, + sourceIssueId: state.sourceIssueId, + initialRunId: state.initialRunId, + connectionId: state.connectionId, + oauth: state.oauthReceipt, + draft: state.savedDraft, + currentHumanPreserved: state.currentHumanPreserved, + originalSourcePreserved: state.originalSourcePreserved, + }); + }, + async cleanup(ctx) { + for (const row of ( + await ctx.input.api.get<{ connections: ToolConnection[] }>( + `${root(ctx)}/tools/connections`, + ) + ).connections) + await ctx.input.api.delete(`/api/tool-connections/${row.id}`, { + allowNotFound: true, + }); + }, + }, + ), +);