diff --git a/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts b/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts index 6eed3b60cb4..8f52e6d508e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts @@ -18,6 +18,9 @@ export function gradeViewerConsent(value: unknown) { && connections?.length === 1 && connections[0]?.companyId === o.companyId && connections[0]?.scopes.includes("paperclip:read") && !connections[0]?.scopes.includes("paperclip:write") && o.state.tokenScopesReadOnly === true, "The actual signed-in viewer sees the intended company and requested write scope but cannot accept writes, including via the public consent API."); + c.add("explicit-consenting-person-boundary", o.state.phaseBoundary === "operator-chat-to-viewer-browser-consent" && typeof o.state.chatCallerId === "string" + && o.state.chatCallerId !== o.state.viewerId && o.state.viewerSessionBefore === o.state.viewerId && o.state.viewerSessionAtConsent === o.state.viewerId && o.state.viewerSessionAfter === o.state.viewerId, + "This tests viewer authority in the explicitly separate consent phase; the operator's canonical chat is not represented as viewer chat coverage."); c.add("only-explicit-human-grant", o.state.viewerGrantId === connections?.[0]?.id && o.state.humanConsented === true && o.activity.some(row => row.action === "mcp.connection_authorized" && row.entityId === o.state.viewerGrantId && row.actorType === "user" && row.actorId === o.state.viewerId), "Only the invited viewer's actual browser consent authorizes this grant, with independent actor attribution."); @@ -34,10 +37,11 @@ export const assistantConsentExternalCases = [definition("assistant-connections. if (!browser || !secrets) throw new Error("Viewer consent requires an isolated signed-in person and private redactor"); const viewer = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets, name: "Viewer Casey" }); people.set(ctx, viewer); try { + const caller = await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session"); await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, viewer, secrets, "viewer"); const marker = `Viewer allowed fixture ${randomBytes(12).toString("hex")}`; const task = await ctx.input.api.post<{ id: string; title: string; description: string }>(`${root(ctx)}/issues`, { title: "Viewer consent sample task", description: marker, status: "backlog" }); - return { viewerId: viewer.id, taskId: task.id, taskBefore: task, marker, forbiddenTitle: `Forbidden viewer task ${ctx.input.nonce}` }; + return { viewerId: viewer.id, chatCallerId: caller.user.id, phaseBoundary: "operator-chat-to-viewer-browser-consent", taskId: task.id, taskBefore: task, marker, forbiddenTitle: `Forbidden viewer task ${ctx.input.nonce}` }; } catch (error) { people.delete(ctx); await viewer.dispose(); throw error; } }, gradeViewerConsent, { authentication: "human-session", @@ -46,9 +50,12 @@ export const assistantConsentExternalCases = [definition("assistant-connections. await waitForReply(ctx); const viewer = people.get(ctx); if (!viewer || !ctx.input.privateSecrets) throw new Error("Viewer consent fixture missing"); const page = await viewer.context.newPage(); try { + async function signedInPerson() { const response = await page.context().request.get(`${ctx.input.api.baseURL}/api/auth/get-session`); if (!response.ok()) throw new Error("Viewer browser session check failed"); return (await response.json() as { user: { id: string } }).user.id; } + state.viewerSessionBefore = await signedInPerson(); const connection = await connect(viewer.context, { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, page, ctx.input.privateSecrets, false, { requestedWrite: true, async inspectConsent(consent) { + state.viewerSessionAtConsent = await signedInPerson(); await expect(consent.getByRole("radio", { name: ctx.input.fixtures.company.name, exact: true })).toBeChecked(); state.companyVisible = true; await expect(consent.getByText("Your role in this organization is read-only.", { exact: true })).toBeVisible(); state.readOnlyRoleVisible = true; const writes = consent.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true }); @@ -62,6 +69,8 @@ export const assistantConsentExternalCases = [definition("assistant-connections. }, }); state.humanConsented = true; + state.viewerSessionAfter = await signedInPerson(); + await ctx.input.evidence("chat-catalog-viewer-consent-boundary.json", { phaseBoundary: state.phaseBoundary, chatCallerId: state.chatCallerId, consentingViewerId: state.viewerId, viewerSessionBefore: state.viewerSessionBefore, viewerSessionAtConsent: state.viewerSessionAtConsent, viewerSessionAfter: state.viewerSessionAfter }); state.tokenScopesReadOnly = connection.tokens.scope.split(/\s+/).includes("paperclip:read") && !connection.tokens.scope.split(/\s+/).includes("paperclip:write"); const client = await mcp(connection.tokens); try { diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index a02348c56b8..8f3d8541031 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -14,3 +14,4 @@ export { gatewayClientExternalCases } from "./external-gateway-client.js"; export { pluginInstallExternalCases } from "./external-plugin-install.js"; export { mcpFailureExternalCases } from "./external-mcp-failure.js"; export { assistantConsentExternalCases } from "./external-assistant-consent.js"; +export { notionReadExternalCases } from "./external-notion-read.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md index 166addb8bc3..de08ceec8dd 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md @@ -11,13 +11,18 @@ of credential environment variables belong there. The file schema is: "providers": { "notion": { "disposable": true, - "resourceIds": ["authorized-disposable-workspace-id"], - "credentialEnv": ["PAPERCLIP_CHAT_EVAL_NOTION_AUTH"], + "resourceIds": ["authorized-disposable-workspace-id", "synthetic-launch-update-page-id", "synthetic-background-page-id"], + "credentialEnv": ["PAPERCLIP_CHAT_EVAL_NOTION_API", "PAPERCLIP_CHAT_EVAL_NOTION_MCP"], "prerequisites": { "workspaceId": "authorized-disposable-workspace-id", "recentPageId": "synthetic-launch-update-page-id", "olderPageId": "synthetic-background-page-id", - "mcpUrl": "https://mcp.notion.com/mcp" + "mcpUrl": "https://mcp.notion.com/mcp", + "apiVersion": "2022-06-28", + "apiTokenEnv": "PAPERCLIP_CHAT_EVAL_NOTION_API", + "mcpTokenEnv": "PAPERCLIP_CHAT_EVAL_NOTION_MCP", + "recentMarker": "approved synthetic current launch fact", + "olderMarker": "approved synthetic older background fact" } } } @@ -64,6 +69,20 @@ The remaining catalog prose is unchanged. The fixture creates a company-owned en connection, verifies current repository discovery, and deletes both on cleanup. Ambient `gh` authentication and personal credentials are never adopted. +The named `notion-connected` case requires separate references for the official +Notion REST oracle and official Notion MCP authorization. The API version is an +explicit coordinator choice; its validity is not inferred from the example. +Workspace and both page UUIDs must occur in `resourceIds`. At execution, the +official [token bot-user endpoint](https://developers.notion.com/reference/get-self) +must report the selected workspace, and both allowlisted page bodies must contain +their distinct approved synthetic facts with the current page actually newer. +Only minimal page/source/date projections and workspace-match booleans enter +case observations. Raw account receipts, names and email remain private. +The model must independently perform a successful MCP read in its own source run; +the oracle's REST reads cannot satisfy that requirement. No generic or synthetic +MCP host is accepted as a named Notion proof. Missing bindings remain executable +setup blockers; source-reference availability is not provider validity evidence. + Synthetic notes apps, MCP gateways, local plugins, team packages, local encrypted secrets and test endpoints do not require this file. They use real public Paperclip APIs and controlled fixture implementations; they cannot count as live diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts index 2a5894bdc24..b394fee835a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts @@ -14,7 +14,7 @@ export interface ExternalFixtureConfig { schemaVersion: 1; providers: Partial = { "projects.add-repository": { provider: "github", fields: ["siteRepositoryId", "siteRepositoryFullName", "tokenEnv"] }, "projects.replace-repository": { provider: "github", fields: ["oldRepositoryId", "oldRepositoryFullName", "newRepositoryId", "newRepositoryFullName", "tokenEnv"] }, - "connection-setup.notion-connected": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl"] }, + "connection-setup.notion-connected": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker"] }, "connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "recentPageId", "oauthBrowserSession"] }, "connection-setup.notion-wrong-account": { provider: "notion", fields: ["sharedWorkspaceId", "personalWorkspaceId", "personalPageId", "oauthBrowserSession"] }, "connection-setup.notion-agent-unassigned": { provider: "notion", fields: ["workspaceId", "recentPageId", "mcpUrl"] }, diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-provider.test.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-provider.test.ts new file mode 100644 index 00000000000..3ee87173c9e --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-provider.test.ts @@ -0,0 +1,39 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { readNotionFixturePages } from "./external-notion-provider.js"; +const workspaceId = "11111111-1111-4111-8111-111111111111", recent = "22222222-2222-4222-8222-222222222222", older = "33333333-3333-4333-8333-333333333333"; +function context() { + const binding = { disposable: true, resourceIds: [workspaceId, recent, older], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_API", "PAPERCLIP_CHAT_EVAL_NOTION_MCP"], prerequisites: { + workspaceId, recentPageId: recent, olderPageId: older, mcpUrl: "https://mcp.notion.com/mcp", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_API", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_MCP", recentMarker: "Current fixture fact", olderMarker: "Earlier fixture fact", + } }; + return { scenario: { id: "connection-setup.notion-connected" }, input: { privateSecrets: [], privateInputs: { read: (name: string) => name === "PAPERCLIP_CHAT_EVAL_NOTION_API" ? "private-api-fixture" : "private-mcp-fixture", configuration: () => ({ schemaVersion: 1, providers: { notion: binding } }) } } } as unknown as ChatCatalogContext; +} +function provider(options: { wrongWorkspace?: boolean; missingBody?: boolean; reverseDates?: boolean } = {}) { + const seen: string[] = []; + vi.stubGlobal("fetch", vi.fn(async (url: URL, input: RequestInit) => { + expect(url.origin).toBe("https://api.notion.com"); expect(input.method ?? "GET").toBe("GET"); expect(input.redirect).toBe("error"); seen.push(url.pathname); + if (url.pathname === "/v1/users/me") return Response.json({ object: "user", type: "bot", bot: { workspace_id: options.wrongWorkspace ? older : workspaceId, workspace_name: "Not retained in oracle projection" } }); + const id = url.pathname.split("/")[3], isRecent = id === recent; + if (url.pathname.endsWith("/children")) return Response.json({ results: [{ id, type: "paragraph", has_children: false, paragraph: { rich_text: [{ plain_text: options.missingBody ? "Unrelated content with a convincing title" : isRecent ? "Current fixture fact" : "Earlier fixture fact" }] } }], has_more: false }); + return Response.json({ object: "page", id, url: `https://www.notion.so/${id}`, last_edited_time: isRecent !== !!options.reverseDates ? "2026-10-25T00:00:00Z" : "2026-10-01T00:00:00Z", properties: { title: { title: [{ plain_text: "Launch notes" }] } } }); + })); + return seen; +} +afterEach(() => vi.unstubAllGlobals()); +describe("named Notion callback calibration, no live provider operations", () => { + it("reads only the fixed official identity and allowlisted page trees and projects no raw account metadata", async () => { + const seen = provider(); const pages = await readNotionFixturePages(context()); expect(pages).toHaveLength(2); expect(pages[0]).toMatchObject({ id: recent, marker: "Current fixture fact" }); + expect(JSON.stringify(pages)).not.toContain("Not retained"); expect(seen).toEqual(["/v1/users/me", `/v1/pages/${recent}`, `/v1/blocks/${recent}/children`, `/v1/pages/${older}`, `/v1/blocks/${older}/children`]); + }); + it("rejects a wrong token workspace before reading any page", async () => { + const seen = provider({ wrongWorkspace: true }); await expect(readNotionFixturePages(context())).rejects.toThrow(/workspace differs/); expect(seen).toEqual(["/v1/users/me"]); + }); + it("title-only content and stale date assumptions cannot satisfy the fixture prerequisite", async () => { + provider({ missingBody: true }); await expect(readNotionFixturePages(context())).rejects.toThrow(/no longer contains/); + provider({ reverseDates: true }); await expect(readNotionFixturePages(context())).rejects.toThrow(/actually be newer/); + }); + it("missing selected private references block before provider traffic", async () => { + const call = vi.fn(); vi.stubGlobal("fetch", call); + await expect(readNotionFixturePages({ scenario: { id: "connection-setup.notion-connected" }, input: {} } as ChatCatalogContext)).rejects.toThrow(/private inputs/); expect(call).not.toHaveBeenCalled(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts new file mode 100644 index 00000000000..ffa94b64e87 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts @@ -0,0 +1,81 @@ +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { selectedExternalBinding, selectedExternalCredential, type ProviderBinding } from "./external-fixtures.js"; +import { root } from "./external-support.js"; +export interface NotionPageOracle { id: string; url: string; editedAt: string; marker: string; title: string } +interface Page { object: string; id: string; url: string; last_edited_time: string; parent?: { type?: string; page_id?: string; workspace?: boolean }; properties?: Record }> } +interface Block { id: string; type: string; has_children: boolean; [key: string]: unknown } +const normalized = (id: string) => id.replaceAll("-", "").toLowerCase(); +/** Only the selected disposable resource allowlist is accepted. This does not + * enumerate an account, search workspaces, or infer permission from a token. */ +export function assertNotionFixture(binding: ProviderBinding) { + const p = binding.prerequisites; + for (const field of ["workspaceId", "recentPageId", "olderPageId"]) { + const id = p[field]; if (!id || !/^[a-f0-9-]{32,36}$/i.test(id) || !binding.resourceIds.some(resource => normalized(resource) === normalized(id))) throw new Error(`Setup blocker: Notion ${field} must be an explicitly allowlisted disposable UUID`); + } + const url = new URL(p.mcpUrl!); + if (url.protocol !== "https:" || url.hostname !== "mcp.notion.com" || url.pathname !== "/mcp" || url.username || url.password || url.search || url.hash) throw new Error("Setup blocker: named Notion proof requires the actual official Notion MCP resource"); + if (!/^\d{4}-\d{2}-\d{2}$/.test(p.apiVersion ?? "")) throw new Error("Setup blocker: Notion API version reference is missing"); + if (!p.recentMarker || !p.olderMarker || p.recentMarker === p.olderMarker) throw new Error("Setup blocker: distinct synthetic recent/background Notion marker references are required"); +} +async function notionJson(token: string, version: string, pathname: string): Promise { + const url = new URL(pathname, "https://api.notion.com"); + if (url.origin !== "https://api.notion.com" || !/^\/v1\/(?:users\/me|pages\/[a-f0-9-]+|blocks\/[a-f0-9-]+\/children)$/.test(url.pathname)) throw new Error("Notion oracle escaped its read-only fixture resource"); + const response = await fetch(url, { redirect: "error", signal: AbortSignal.timeout(20_000), headers: { Authorization: `Bearer ${token}`, "Notion-Version": version, "content-type": "application/json" } }); + if (!response.ok) { await response.body?.cancel(); throw new Error(`Setup blocker: authorized disposable Notion resource read returned HTTP ${response.status}; contents withheld`); } + const bytes = Buffer.from(await response.arrayBuffer()); if (bytes.length > 2 * 1024 * 1024) throw new Error("Setup blocker: Notion oracle response exceeds bounded fixture size"); + try { return JSON.parse(bytes.toString("utf8")) as T; } catch { throw new Error("Notion oracle returned invalid JSON; contents withheld"); } +} +function richText(value: unknown): string { + if (!value || typeof value !== "object") return ""; + if (Array.isArray(value)) return value.map(richText).join(" "); + const row = value as Record; + if (typeof row.plain_text === "string") return row.plain_text; + return Object.values(row).filter(value => value && typeof value === "object").map(richText).join(" "); +} +async function blocksText(token: string, version: string, pageId: string, maxChildren = 100) { + let blocks = 0; const visited = new Set(); + async function children(id: string, depth: number): Promise { + if (depth > 3 || visited.has(id)) throw new Error("Setup blocker: Notion page exceeds bounded fixture tree"); visited.add(id); + const page = await notionJson<{ results: Block[]; has_more: boolean }>(token, version, `/v1/blocks/${id}/children?page_size=100`); + if (!Array.isArray(page.results) || page.has_more || (blocks += page.results.length) > maxChildren) throw new Error("Setup blocker: Notion fixture page exceeds bounded child count"); + const contents: string[] = []; + for (const block of page.results) { contents.push(richText(block[block.type])); if (block.has_children) contents.push(await children(block.id, depth + 1)); } + return contents.join("\n"); + } + return children(pageId, 0); +} +export async function readNotionFixturePages(ctx: ChatCatalogContext, binding = selectedExternalBinding(ctx)) { + assertNotionFixture(binding); const p = binding.prerequisites; + const token = selectedExternalCredential(ctx, binding, "apiTokenEnv"); + // Provider identity is checked privately; names, email and the raw account + // receipt never enter state/evidence. Provider contract: + // https://developers.notion.com/reference/user#bots + const self = await notionJson<{ object: string; type: string; bot?: { workspace_id?: string } }>(token, p.apiVersion!, "/v1/users/me"); + if (self.object !== "user" || self.type !== "bot" || !self.bot?.workspace_id || normalized(self.bot.workspace_id) !== normalized(p.workspaceId!)) throw new Error("Setup blocker: actual Notion token workspace differs from its authorized disposable workspace reference"); + const result: NotionPageOracle[] = []; + for (const [idField, markerField] of [["recentPageId", "recentMarker"], ["olderPageId", "olderMarker"]] as const) { + const id = p[idField]!, marker = p[markerField]!; + const page = await notionJson(token, p.apiVersion!, `/v1/pages/${id}`); + if (page.object !== "page" || normalized(page.id) !== normalized(id) || !Number.isFinite(Date.parse(page.last_edited_time))) throw new Error("Setup blocker: independent Notion page identity/date differs from its allowlisted fixture"); + const url = new URL(page.url); if (url.protocol !== "https:" || !/(^|\.)notion\.(?:so|site)$/.test(url.hostname)) throw new Error("Setup blocker: Notion page returned an unexpected source host"); + const body = await blocksText(token, p.apiVersion!, id); + if (!body.includes(marker)) throw new Error(`Setup blocker: ${idField} no longer contains its authorized synthetic marker`); + result.push({ id, url: page.url, editedAt: page.last_edited_time, marker, title: Object.values(page.properties ?? {}).flatMap(row => row.title ?? []).map(row => row.plain_text ?? "").join("") }); + } + if (Date.parse(result[0]!.editedAt) <= Date.parse(result[1]!.editedAt)) throw new Error("Setup blocker: recent Notion fixture must actually be newer than the background page"); + return result; +} +export async function connectNamedNotion(ctx: ChatCatalogContext, binding: ProviderBinding, agentIds: string[]) { + assertNotionFixture(binding); + const token = selectedExternalCredential(ctx, binding, "mcpTokenEnv"); + const result = await ctx.input.api.postSensitive<{ connectionId: string; actions: { readOnly: Array<{ catalogEntryId: string }>; canMakeChanges: Array<{ catalogEntryId: string }> } }>(`${root(ctx)}/tools/apps/connect`, { + link: binding.prerequisites.mcpUrl, name: "Launch Notion", authMode: "bearer", credentialValues: { "credentials.authorization": token }, grantKind: "organization", + }); + const read = result.actions.readOnly.map(row => row.catalogEntryId); + if (!read.length) throw new Error("Setup blocker: actual Notion MCP connection did not expose any classified read actions"); + try { + await ctx.input.api.post(`${root(ctx)}/tools/apps/${result.connectionId}/finish`, { enabledCatalogEntryIds: read, askFirstCatalogEntryIds: [], access: { agentIds } }); + await ctx.input.api.putSensitive(`/api/tool-connections/${result.connectionId}/installs`, { installs: agentIds.map(targetId => ({ targetType: "agent", targetId })) }); + return { connectionId: result.connectionId, readEntryIds: read }; + } catch (error) { await ctx.input.api.delete(`/api/tool-connections/${result.connectionId}`, { allowNotFound: true }); throw error; } +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-read.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-read.ts new file mode 100644 index 00000000000..f47450b03a7 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-read.ts @@ -0,0 +1,58 @@ +import { expect } from "@playwright/test"; +import type { ToolConnection, ToolRunDecisionLookup, ToolProfileEffectiveSummary } from "../../../packages/shared/src/types/tool-access.js"; +import { externalRequirements, selectedExternalBinding } from "./external-fixtures.js"; +import { connectNamedNotion, readNotionFixturePages, type NotionPageOracle } from "./external-notion-provider.js"; +import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +export function gradeNotionRead(value: unknown) { + const c = checks(), o = externalObservation(value); + c.add("complete-named-notion-evidence", !!o, "Real official Notion account/resource checks, provider-backed catalog admission and source-grounded summary are required."); if (!o) return c.result(); + const pages = o.state.pages as NotionPageOracle[] | undefined; const recent = pages?.[0], older = pages?.[1]; + c.add("actual-current-provider-fixtures", o.state.provider === "notion" && o.state.providerOrigin === "https://mcp.notion.com" && o.state.workspaceIndependentlyMatched === true + && !!recent && !!older && recent.id !== older.id && Date.parse(recent.editedAt) > Date.parse(older.editedAt) && o.state.fixturesUnchanged === true, + "An independent official Notion API checks token workspace identity and actual current and older allowlisted page bodies/dates before and after the query."); + const effective = o.state.effective as ToolProfileEffectiveSummary | undefined; + const readIds = o.state.readEntryIds as string[] | undefined; + c.add("existing-assigned-healthy-connection", o.state.healthyBeforeQuery === true && o.connections.filter(row => row.id === o.state.connectionId).length === 1 + && o.connections.length === o.state.connectionCountBefore && effective?.agentId === o.leadId && Array.isArray(readIds) + && readIds.some(id => effective.allowedTools.some(tool => tool.id === id)), "The same pre-existing official Notion connection remains usable for this CEO without duplicate setup or repaired credentials."); + const decisions = o.state.decisions as ToolRunDecisionLookup[] | undefined; + c.add("actual-model-notion-read", Array.isArray(decisions) && decisions.some(row => row.decisions.some(decision => decision.invocation.runId === row.runId + && decision.invocation.agentId === o.leadId && decision.invocation.connectionId === o.state.connectionId && decision.invocation.issueId === o.state.sourceIssueId + && readIds?.includes(decision.invocation.catalogEntryId ?? "") && decision.invocation.status === "succeeded")), + "The actual canonical-query model run successfully invokes a read action from this named provider connection; independent oracle reads cannot substitute."); + const recentDate = recent?.editedAt.slice(0, 10); + c.add("latest-body-grounded-summary", !!recent && o.reply.includes(recent.marker) && !!recentDate && o.reply.includes(recentDate) + && o.links.some(link => { try { return new URL(link).href.replace(/\/$/, "") === recent.url.replace(/\/$/, ""); } catch { return false; } }) + && (!older || !o.reply.includes(older.marker) || /older|background|previous|earlier/i.test(o.reply)), + "The answer contains the freshly observed body fact, its real source and update date, and distinguishes older background if mentioned."); + c.add("no-needless-reconnect", !o.interactions.some(row => row.kind === "connection_intent" && row.payload?.serviceSlug === "notion") + && !/\b(?:must|need to|please) (?:re)?connect (?:your |the )?notion/i.test(o.reply), "An already usable account is used without a needless setup request."); + return c.result(); +} +export const notionReadExternalCases = [definition("connection-setup.notion-connected", async ctx => { + const binding = selectedExternalBinding(ctx); const pages = await readNotionFixturePages(ctx, binding); + const connected = await connectNamedNotion(ctx, binding, [ctx.input.fixtures.agent.id]); + const connection = await ctx.input.api.get(`/api/tool-connections/${connected.connectionId}`); + if (connection.status !== "active" || !connection.enabled || connection.healthStatus !== "ok") throw new Error("Setup blocker: authorized official Notion fixture did not produce a healthy active connection"); + const connections = await ctx.input.api.get<{ connections: ToolConnection[] }>(`${root(ctx)}/tools/connections`); + return { ...connected, pages, provider: "notion", providerOrigin: "https://mcp.notion.com", workspaceIndependentlyMatched: true, healthyBeforeQuery: true, connectionCountBefore: connections.connections.length }; +}, gradeNotionRead, { + externalInput: externalRequirements["connection-setup.notion-connected"], + sourceFiles: ["chat-catalog-domains/external-notion-read.ts", "chat-catalog-domains/external-notion-provider.ts", "chat-catalog-domains/external-fixtures.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/connection-setup.yaml"], + async observe(ctx, state) { + const current = await readNotionFixturePages(ctx); const before = state.pages as NotionPageOracle[]; + state.fixturesUnchanged = current.length === before.length && current.every((page, index) => page.id === before[index]?.id && page.editedAt === before[index]?.editedAt && page.marker === before[index]?.marker); + state.effective = await ctx.input.api.get(`${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`); + const runs = await ctx.allRuns(); state.decisions = await Promise.all(runs.map(run => ctx.input.api.get(`${root(ctx)}/tools/runs/${run.id}/decisions`))); + state.sourceIssueId = ctx.issue()?.id; return observeExternal(ctx, state); + }, + async verifyResult(ctx, state, value) { + await verifyRenderedReply(ctx, value); const page = (state.pages as NotionPageOracle[])[0]!; + const anchor = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(page.url)}]`).first(); await expect(anchor).toBeVisible(); + // External page browsers can require a second login. The actual source was + // independently retrieved above; verify the rendered exact link identity. + await expect(anchor).toHaveAttribute("href", page.url); + await ctx.input.evidence("chat-catalog-notion-source.json", { provider: "notion", pageId: page.id, url: page.url, editedAt: page.editedAt, workspaceMatched: true }); + }, + async cleanup(ctx, state) { if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true }); }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 716ffe604cb..8351092c2ff 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -5,6 +5,8 @@ import { gatewayClientExternalCases } from "./external-gateway-client.js"; import { pluginInstallExternalCases } from "./external-plugin-install.js"; import { mcpFailureExternalCases } from "./external-mcp-failure.js"; import { assistantConsentExternalCases } from "./external-assistant-consent.js"; +import { notionReadExternalCases } from "./external-notion-read.js"; +import { assertNotionFixture } from "./external-notion-provider.js"; import { assistantDeviceExternalCases } from "./external-assistant-device.js"; import { pluginExternalCases } from "./external-plugins.js"; import { appCatalogExternalCases } from "./external-app-catalog.js"; @@ -16,7 +18,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js"; import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js"; import type { ExternalObservation } from "./external-support.js"; -const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases]; +const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases, ...notionReadExternalCases]; function base(scenarioId: string): ExternalObservation { return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" }, phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], @@ -24,10 +26,17 @@ function base(scenarioId: string): ExternalObservation { } function valid(id: string) { const o = base(id); - if (id === "assistant-connections.consent-read-only") { + if (id === "connection-setup.notion-connected") { + Object.assign(o.state, { provider: "notion", providerOrigin: "https://mcp.notion.com", workspaceIndependentlyMatched: true, + pages: [{ id: "recent", url: "https://www.notion.so/recent", editedAt: "2026-10-25T12:00:00Z", marker: "New observed launch fact" }, { id: "old", url: "https://www.notion.so/old", editedAt: "2026-10-01T12:00:00Z", marker: "Old background" }], fixturesUnchanged: true, + connectionId: "notion", healthyBeforeQuery: true, connectionCountBefore: 1, effective: { agentId: "ceo", allowedTools: [{ id: "read" }] }, readEntryIds: ["read"], sourceIssueId: "source", + decisions: [{ runId: "initial", decisions: [{ invocation: { runId: "initial", agentId: "ceo", connectionId: "notion", issueId: "source", catalogEntryId: "read", status: "succeeded" } }] }] }); + o.connections = [{ id: "notion" }]; o.links = ["https://www.notion.so/recent"]; o.reply = "New observed launch fact (updated 2026-10-25)."; + } else if (id === "assistant-connections.consent-read-only") { Object.assign(o.state, { requestedWrite: true, readOnlyRoleVisible: true, writeCheckboxDisabled: true, writeCheckboxUnchecked: true, companyVisible: true, viewerEscalationStatus: 403, viewerConnections: [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "offline_access"] }], tokenScopesReadOnly: true, viewerGrantId: "grant", humanConsented: true, viewerId: "viewer", - readTaskId: "task", taskId: "task", readTaskCompanyId: "company", readMarkerMatched: true, createDeniedForScope: true, updateDeniedForScope: true, taskPreserved: true, forbiddenTaskCount: 0 }); + readTaskId: "task", taskId: "task", readTaskCompanyId: "company", readMarkerMatched: true, createDeniedForScope: true, updateDeniedForScope: true, taskPreserved: true, forbiddenTaskCount: 0, + phaseBoundary: "operator-chat-to-viewer-browser-consent", chatCallerId: "operator", viewerSessionBefore: "viewer", viewerSessionAtConsent: "viewer", viewerSessionAfter: "viewer" }); o.activity = [{ id: "authorization", action: "mcp.connection_authorized", entityId: "grant", actorType: "user", actorId: "viewer" }]; } else if (id === "connection-setup.unreachable-mcp") { Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true }); @@ -100,6 +109,7 @@ function valid(id: string) { return o; } const wrong: Record void> = { + "connection-setup.notion-connected": o => { o.reply = "The launch remains on schedule, according to the page title."; }, "assistant-connections.consent-read-only": o => { o.state.viewerConnections = [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "paperclip:write"] }]; }, "connection-setup.unreachable-mcp": o => { o.connections = [{ id: "phantom", status: "active" }]; }, "plugins.install-plugin": o => { o.state.pluginAfter = { id: "plugin", pluginKey: "synthetic.preview", status: "error" }; }, @@ -120,6 +130,12 @@ const wrong: Record void> = { "connection-setup.notion-declined": o => { o.interactions[0]!.resolvedByUserId = "someone-else"; }, }; describe("external domain independent oracles", () => { + it("viewer consent cannot borrow the operator's session or silently claim viewer chat coverage", () => { + for (const field of ["viewerSessionBefore", "viewerSessionAtConsent", "viewerSessionAfter", "phaseBoundary"]) { + const o = valid("assistant-connections.consent-read-only"); o.state[field] = "operator"; + expect(assistantConsentExternalCases[0]!.grade(o).passed).toBe(false); + } + }); it("an inactive or expired gateway session cannot prove profile write denial", () => { for (const reason of ["run_inactive", "session_run_inactive", "gateway_token_expired", "tool_not_found", undefined]) { const o = valid("gateways-profiles.connect-client"); o.state.writeProbeReason = reason; @@ -149,7 +165,7 @@ describe("private disposable provider prerequisite contract", () => { it("owning runtime callbacks read only selected private references and never ambient env", () => { const binding = { disposable: true as const, resourceIds: ["workspace"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_AUTH"], prerequisites: { workspaceId: "workspace", recentPageId: "recent", olderPageId: "old", mcpUrl: "https://fixture.invalid" } }; const redactions: string[] = []; - const ctx = { scenario: { id: "connection-setup.notion-connected" }, input: { privateSecrets: redactions, privateInputs: { + const ctx = { scenario: { id: "connection-setup.notion-agent-unassigned" }, input: { privateSecrets: redactions, privateInputs: { configuration: () => ({ schemaVersion: 1, providers: { notion: binding } }), read: (name: string) => name === "PAPERCLIP_CHAT_EVAL_NOTION_AUTH" ? "private-fixture-token" : undefined, } } } as unknown as import("../chat-catalog-contract.js").ChatCatalogContext; expect(selectedExternalBinding(ctx)).toEqual(binding); @@ -166,10 +182,18 @@ describe("private disposable provider prerequisite contract", () => { it("missing input produces field-only receipts, never a configured pass", () => { for (const id of Object.keys(externalRequirements)) { const receipt = externalPrerequisiteReceipt(id, undefined, {}); expect(receipt.configured).toBe(false); expect(receipt.missing.length).toBeGreaterThan(1); } }); - it("local fixtures need no unrelated provider account", () => { for (const definition of definitions) expect(externalPrerequisiteReceipt(definition.scenarioId, undefined, {}).configured).toBe(true); }); + it("local fixtures need no unrelated provider account", () => { for (const definition of definitions.filter(row => !row.externalInput)) expect(externalPrerequisiteReceipt(definition.scenarioId, undefined, {}).configured).toBe(true); }); + it("named Notion fixtures require official transport and every resource in the disposable allowlist", () => { + const workspaceId = "11111111-1111-4111-8111-111111111111", recentPageId = "22222222-2222-4222-8222-222222222222", olderPageId = "33333333-3333-4333-8333-333333333333"; + const binding = { disposable: true as const, resourceIds: [workspaceId, recentPageId, olderPageId], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_API", "PAPERCLIP_CHAT_EVAL_NOTION_MCP"], prerequisites: { workspaceId, recentPageId, olderPageId, mcpUrl: "https://mcp.notion.com/mcp", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_API", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_MCP", recentMarker: "Recent synthetic fact", olderMarker: "Background fact" } }; + expect(() => assertNotionFixture(binding)).not.toThrow(); + expect(() => assertNotionFixture({ ...binding, resourceIds: [workspaceId] })).toThrow(/allowlisted/); + expect(() => assertNotionFixture({ ...binding, prerequisites: { ...binding.prerequisites, mcpUrl: "https://synthetic.invalid/mcp" } })).toThrow(/actual official Notion/); + expect(externalPrerequisiteReceipt("connection-setup.notion-connected", undefined, {}).missing).toContain("apiTokenEnv"); + }); it("requires explicitly disposable allowlisted resources and namespaced env references", () => { const config = { schemaVersion: 1, providers: { notion: { disposable: true, resourceIds: ["workspace"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_AUTH"], - prerequisites: { workspaceId: "workspace", recentPageId: "recent", olderPageId: "old", mcpUrl: "https://fixture.invalid" } } } }; + prerequisites: { workspaceId: "workspace", recentPageId: "recent", olderPageId: "old", mcpUrl: "https://fixture.invalid", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", recentMarker: "Recent", olderMarker: "Older" } } } }; const parsed = validateExternalConfig(config); const env = { PAPERCLIP_CHAT_EVAL_NOTION_AUTH: "secret-value-not-for-receipts" }; expect(externalPrerequisiteReceipt("connection-setup.notion-connected", parsed, env)).toEqual({ scenarioId: "connection-setup.notion-connected", provider: "notion", configured: true, missing: [] });