From c5d20de8f52a45b5a03b3b5c101fc4c952502495 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 03:30:55 -0500 Subject: [PATCH] test(evals): prove personal Notion workspace selection preserves shared access --- .../chat-catalog-domains/external-fixtures.md | 12 ++ .../chat-catalog-domains/external-fixtures.ts | 2 +- .../external-notion-oauth-contract.ts | 6 + .../external-notion-oauth-worker.ts | 1 + .../external-notion-oauth.ts | 3 + .../external-notion-provider.ts | 34 ++++ .../external-notion-setup.test.ts | 42 ++++- .../external-notion-setup.ts | 176 ++++++++++++++++-- 8 files changed, 262 insertions(+), 14 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md index ce71547f4a8..4aafd0f4985 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md @@ -286,3 +286,15 @@ reloads that same draft, then requires its eventual authorization to reuse the same connection ID. No literal Save & exit control or API-created draft is invented. All of this remains authored/unmeasured until a reserved execution has a real authorized disposable Notion session, workspace/pages, and owned origin. + +`notion-wrong-account` additionally requires `sharedWorkspaceId`, +`sharedApiTokenEnv`, and `sharedMcpTokenEnv`, with both workspace IDs and personal +page IDs explicitly allowlisted. The existing shared connection is prepared via +the supported board API, independently matched to its real MCP and REST +workspace, and its healthy REST credential must actually receive 403/404 for the +known-existing personal page. This historical fixture is separate from the +human's evaluated personal OAuth setup. The real supported `new=1` personal +connection route prevents reusing the organization account; no shared audience, +delegation, or install is broadened. Account refusal alone cannot satisfy this +case: its actual personal setup, callback and source-bound launch read must all +complete. diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts index 3129f01023c..5088d95fc3d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts @@ -21,7 +21,7 @@ export const externalRequirements: Record 0 && diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts index dea1da8298c..056b68538e5 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts @@ -93,6 +93,7 @@ async function main(): Promise { const params = new URLSearchParams({ source: "notion", stage: "setup" }); if (input.intentId) params.set("intent", input.intentId); if (input.draftId) params.set("resume", input.draftId); + if (input.newPersonal) params.set("new", "1"); const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`; await page.goto(setupURL, { waitUntil: "domcontentloaded" }); await page diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts index 9f4c3d49d87..b78d7700795 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts @@ -94,6 +94,9 @@ export async function privateNotionOAuth( ); const input: NotionOAuthInput = { ...options, + ...(ctx.scenario.id === "connection-setup.notion-wrong-account" + ? { newPersonal: true as const } + : {}), ...(options.intentId ? { issueId: ctx.issue()?.id } : {}), publicOrigin: origin.publicOrigin, companyId: ctx.input.fixtures.company.id, diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts index aaa31856c68..e9ac256d875 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts @@ -90,3 +90,37 @@ export async function connectNamedNotion(ctx: ChatCatalogContext, binding: Provi return { connectionId: result.connectionId, readEntryIds: read }; } catch (error) { await ctx.input.api.delete(`/api/tool-connections/${result.connectionId}`, { allowNotFound: true }); throw error; } } + +/** A known-existing personal fixture page must really be inaccessible under the + * independently healthy shared workspace credential; never infer mismatch only + * from a display name. No discovery or non-allowlisted page is attempted. */ +export async function verifyNotionPersonalPageDenied( + ctx: ChatCatalogContext, + shared: ProviderBinding, + pageId: string, +) { + assertNotionConnection(shared); + if ( + !/^[a-f0-9-]{32,36}$/i.test(pageId) || + !shared.resourceIds.some((id) => normalized(id) === normalized(pageId)) + ) + throw new Error( + "Setup blocker: personal Notion page is outside explicit disposable resource allowlist", + ); + const token = selectedExternalCredential(ctx, shared, "apiTokenEnv"); + const response = await fetch(`https://api.notion.com/v1/pages/${pageId}`, { + method: "GET", + redirect: "error", + signal: AbortSignal.timeout(20000), + headers: { + Authorization: `Bearer ${token}`, + "Notion-Version": shared.prerequisites.apiVersion!, + }, + }); + await response.body?.cancel(); + if (![403, 404].includes(response.status)) + throw new Error( + "Setup blocker: healthy shared Notion credential did not prove the selected personal page inaccessible", + ); + return true; +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts index d0e56c65b27..4bb0ca82a7b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { gradeNotionSetup, + personalNotionGuidance, notionSetupExternalCases, } from "./external-notion-setup.js"; import { @@ -246,6 +247,45 @@ describe("named Notion human setup controls", () => { mutate(o); expect(gradeNotionSetup(o).passed).toBe(false); }); + it("requires independently distinct real shared/personal authority and preserves the shared audience", () => { + const o = valid("connection-setup.notion-wrong-account"); + o.connections.push({ id: "shared" }); + Object.assign(o.state, { + sharedConnectionId: "shared", + sharedWorkspaceMatched: true, + sharedMcpWorkspaceMatched: true, + sharedCannotReadPersonal: true, + sharedAuthorityPreserved: true, + }); + expect(gradeNotionSetup(o).passed).toBe(true); + for (const key of [ + "sharedWorkspaceMatched", + "sharedMcpWorkspaceMatched", + "sharedCannotReadPersonal", + "sharedAuthorityPreserved", + ]) { + const wrong = structuredClone(o); + wrong.state[key] = false; + expect(gradeNotionSetup(wrong).passed).toBe(false); + } + const wrong = structuredClone(o); + wrong.phaseReplies[0]!.body = + "Connect the shared company Notion account and give everyone access."; + expect(gradeNotionSetup(wrong).passed).toBe(false); + }); + it("gates human personal setup on actual personal-account guidance", () => { + expect( + personalNotionGuidance( + "Please connect your personal Notion workspace using the setup request.", + ), + ).toBe(true); + for (const text of [ + "Connect the shared company account.", + "I've connected your personal workspace.", + "I have already authorized your personal account.", + ]) + expect(personalNotionGuidance(text)).toBe(false); + }); it("rejects a fabricated or replaced resume draft", () => { const o = valid("connection-setup.save-and-resume"); o.state.savedDraft = { ...receipt(), connectionId: "other" }; @@ -253,7 +293,7 @@ describe("named Notion human setup controls", () => { expect(gradeNotionSetup(o).passed).toBe(false); }); it("keeps exact concrete callbacks and phase bounds", () => { - expect(notionSetupExternalCases.map((c) => c.scenarioId)).toHaveLength(3); + expect(notionSetupExternalCases.map((c) => c.scenarioId)).toHaveLength(4); for (const c of notionSetupExternalCases) { expect( [c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every( diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts index b8b8756a6ad..ab0e955a584 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts @@ -5,13 +5,24 @@ import type { ConnectionGrantsResponse, ToolRunDecisionLookup, ToolProfileEffectiveSummary, + ToolConnectionTestCallResult, } from "../../../packages/shared/src/types/tool-access.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import { pollUntil } from "../api.js"; -import { externalRequirements } from "./external-fixtures.js"; +import { + externalRequirements, + selectedExternalBinding, +} from "./external-fixtures.js"; import { privateNotionOAuth } from "./external-notion-oauth.js"; -import type { NotionOAuthReceipt } from "./external-notion-oauth-contract.js"; +import { + notionSelfWorkspaceMatches, + type NotionOAuthReceipt, +} from "./external-notion-oauth-contract.js"; import { readNotionFixturePages, + connectNamedNotion, + verifyNotionWorkspace, + verifyNotionPersonalPageDenied, type NotionPageOracle, } from "./external-notion-provider.js"; import { @@ -28,6 +39,59 @@ import { verifyRenderedReply, } from "./external-support.js"; +export function personalNotionGuidance(text: string) { + return text + .split(/[.!?;\n]+/) + .some( + (clause) => + /\bpersonal\b/i.test(clause) && + /\b(?:connect|authorize|authorization|sign.?in)\b/i.test(clause) && + !/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:connected|authorized)\b/i.test( + clause, + ), + ); +} +async function sharedNotionAuthority(ctx: ChatCatalogContext, id: string) { + const connection = await ctx.input.api.get( + `/api/tool-connections/${id}`, + ); + const { grants } = await ctx.input.api.get( + `/api/tool-connections/${id}/grants`, + ); + const { installs } = await ctx.input.api.get<{ + installs: Array<{ targetType: string; targetId: string }>; + }>(`/api/tool-connections/${id}/installs`); + const sorted = (rows: unknown[]) => + rows.map((row) => JSON.stringify(row)).sort(); + return { + id: connection.id, + companyId: connection.companyId, + enabled: connection.enabled, + credentialPolicy: connection.credentialPolicy, + ownership: connection.ownership, + grants: sorted( + grants.map((g) => ({ + id: g.id, + kind: g.kind, + subjectUserId: g.subjectUserId, + subjectAgentId: g.subjectAgentId, + status: g.status, + members: sorted( + (g.members ?? []).map((m) => ({ + subjectType: m.subjectType, + subjectId: m.subjectId, + })), + ), + delegations: sorted( + (g.delegations ?? []).map((d) => ({ agentId: d.agentId })), + ), + })), + ), + installs: sorted( + installs.map((i) => ({ targetType: i.targetType, targetId: i.targetId })), + ), + }; +} export function gradeNotionSetup(value: unknown) { const c = checks(), o = externalObservation(value); @@ -100,7 +164,8 @@ export function gradeNotionSetup(value: unknown) { card.resolvedByUserId === o.state.humanId && !card.resolvedByAgentId && o.state.connectionId === receipt?.connectionId && - o.connections.length === 1 && + o.connections.length === + (o.scenarioId === "connection-setup.notion-wrong-account" ? 2 : 1) && o.state.samePersonalActive === true && o.state.currentHumanPreserved === true && o.state.originalSourcePreserved === true, @@ -141,6 +206,22 @@ export function gradeNotionSetup(value: unknown) { o.state.pagesUnchanged === true, "The continuation summarizes the actual recent body fact with its real date/source, distinguishing older background.", ); + if (o.scenarioId === "connection-setup.notion-wrong-account") { + c.add( + "actual-shared-personal-mismatch", + o.state.sharedWorkspaceMatched === true && + o.state.sharedMcpWorkspaceMatched === true && + o.state.sharedCannotReadPersonal === true && + o.state.sharedConnectionId !== o.state.connectionId && + personalNotionGuidance(initial), + "The existing healthy shared workspace really lacks the known personal page; actual guidance requests the personal workspace rather than substituting shared notes.", + ); + c.add( + "shared-authority-preserved", + o.state.sharedAuthorityPreserved === true, + "Actual shared account ownership, human audience, agent delegation and installs remain unchanged after creating the explicitly personal account.", + ); + } if (o.scenarioId === "connection-setup.save-and-resume") { const saved = o.state.savedDraft as NotionOAuthReceipt | undefined; c.add( @@ -187,6 +268,7 @@ export const notionSetupExternalCases = [ "connection-setup.notion-missing", "connection-setup.return-from-login", "connection-setup.save-and-resume", + "connection-setup.notion-wrong-account", ].map((id) => definition( id, @@ -212,6 +294,61 @@ export const notionSetupExternalCases = [ baselineConnectionIds: [], savedBeforeCanonical: false, }; + if (id === "connection-setup.notion-wrong-account") { + const binding = selectedExternalBinding(ctx), + p = binding.prerequisites; + if ( + p.sharedWorkspaceId!.replaceAll("-", "").toLowerCase() === + p.workspaceId!.replaceAll("-", "").toLowerCase() + ) + throw new Error( + "Setup blocker: shared and personal Notion workspaces must be genuinely distinct", + ); + const shared = { + ...binding, + prerequisites: { + ...p, + workspaceId: p.sharedWorkspaceId!, + apiTokenEnv: p.sharedApiTokenEnv!, + mcpTokenEnv: p.sharedMcpTokenEnv!, + }, + }; + await verifyNotionWorkspace(ctx, shared); + await verifyNotionPersonalPageDenied(ctx, shared, p.recentPageId!); + const connected = await connectNamedNotion(ctx, shared, [ + ctx.input.fixtures.agent.id, + ]); + const proof = + await ctx.input.api.postSensitive( + `/api/tool-connections/${connected.connectionId}/test-calls`, + { + agentId: ctx.input.fixtures.agent.id, + toolName: "notion-get-self", + parameters: {}, + }, + ); + if ( + proof.decision !== "allowed" || + proof.error || + !notionSelfWorkspaceMatches(proof.result, p.sharedWorkspaceId!) + ) { + await ctx.input.api.delete( + `/api/tool-connections/${connected.connectionId}`, + { allowNotFound: true }, + ); + throw new Error( + "Setup blocker: actual existing shared MCP authorization does not match its designated workspace", + ); + } + state.sharedMcpWorkspaceMatched = true; + state.sharedConnectionId = connected.connectionId; + state.sharedAuthorityBefore = await sharedNotionAuthority( + ctx, + connected.connectionId, + ); + state.sharedWorkspaceMatched = true; + state.sharedCannotReadPersonal = true; + } if (id === "connection-setup.save-and-resume") { try { state.savedDraft = await privateNotionOAuth(ctx, { @@ -299,15 +436,18 @@ export const notionSetupExternalCases = [ ).connections; const saved = state.savedDraft as NotionOAuthReceipt | undefined; state.pendingWithoutCredential = - before.length === (saved ? 1 : 0) && - before.every( - (row) => - row.id === saved?.connectionId && - row.status === "draft" && - row.credentialPolicy === "per_user" && - !row.credentialSecretRefs.length && - !row.credentialRefs?.length, - ); + before.filter((row) => row.id !== state.sharedConnectionId).length === + (saved ? 1 : 0) && + before + .filter((row) => row.id !== state.sharedConnectionId) + .every( + (row) => + row.id === saved?.connectionId && + row.status === "draft" && + row.credentialPolicy === "per_user" && + !row.credentialSecretRefs.length && + !row.credentialRefs?.length, + ); const current = await observeExternal(ctx, state); const initial = current.phaseReplies .filter((row) => row.runId === card.sourceRunId) @@ -322,6 +462,13 @@ export const notionSetupExternalCases = [ throw new Error( "Notion request was prematurely satisfied or already exposed unseen fixture facts", ); + if ( + id === "connection-setup.notion-wrong-account" && + !personalNotionGuidance(initial) + ) + throw new Error( + "Actual Notion guidance does not identify the requested personal workspace before the human acts", + ); // The visible addressed card's supported full-page setup host preserves the // exact same intent. No API connection or completion substitutes for UI. state.oauthReceipt = await privateNotionOAuth(ctx, { @@ -373,6 +520,11 @@ export const notionSetupExternalCases = [ ) .map((e) => e.id); } + if (typeof state.sharedConnectionId === "string") + state.sharedAuthorityPreserved = + JSON.stringify( + await sharedNotionAuthority(ctx, state.sharedConnectionId), + ) === JSON.stringify(state.sharedAuthorityBefore); state.decisions = await Promise.all( (await ctx.allRuns()).map((run) => ctx.input.api.get(