test(evals): prove personal Notion workspace selection preserves shared access

tests/runner-e2e/chat-catalog-domains/external-fixtures.md

@@ -286,3 +286,15 @@ reloads that same draft, then requires its eventual authorization to reuse the
 same connection ID. No literal Save & exit control or API-created draft is
 invented. All of this remains authored/unmeasured until a reserved execution has
 a real authorized disposable Notion session, workspace/pages, and owned origin.
+
+`notion-wrong-account` additionally requires `sharedWorkspaceId`,
+`sharedApiTokenEnv`, and `sharedMcpTokenEnv`, with both workspace IDs and personal
+page IDs explicitly allowlisted. The existing shared connection is prepared via
+the supported board API, independently matched to its real MCP and REST
+workspace, and its healthy REST credential must actually receive 403/404 for the
+known-existing personal page. This historical fixture is separate from the
+human's evaluated personal OAuth setup. The real supported `new=1` personal
+connection route prevents reusing the organization account; no shared audience,
+delegation, or install is broadened. Account refusal alone cannot satisfy this
+case: its actual personal setup, callback and source-bound launch read must all
+complete.

tests/runner-e2e/chat-catalog-domains/external-fixtures.ts

@@ -21,7 +21,7 @@ export const externalRequirements: Record<string, { provider: ExternalProvider;
   "gateways-profiles.define-profile": { provider: "notion", fields: ["workspaceId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv"] },
   "connection-setup.notion-connected": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
   "connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
-  "connection-setup.notion-wrong-account": { provider: "notion", fields: ["sharedWorkspaceId", "personalWorkspaceId", "personalPageId", "oauthBrowserSession"] },
+  "connection-setup.notion-wrong-account": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv", "sharedWorkspaceId", "sharedApiTokenEnv", "sharedMcpTokenEnv"] },
   "connection-setup.notion-agent-unassigned": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
   "connection-setup.notion-expired": { provider: "notion", fields: ["workspaceId", "recentPageId", "expiredAuthorizationEnv", "oauthBrowserSession"] },
   "connection-setup.return-from-login": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },

tests/runner-e2e/chat-catalog-domains/external-notion-oauth-contract.ts

@@ -9,6 +9,7 @@ export interface NotionOAuthInput {
   intentId?: string;
   issueId?: string;
   draftId?: string;
+  newPersonal?: true;
   workspaceId: string;
   workspaceName: string;
   providerEmail: string;
@@ -98,6 +99,11 @@ export function validNotionOAuthInput(input: NotionOAuthInput) {
       !!input.intentId === !!input.issueId &&
       (!input.issueId || id(input.issueId)) &&
       (!input.draftId || id(input.draftId)) &&
+      (!input.newPersonal ||
+        (input.newPersonal === true &&
+          !input.draftId &&
+          !!input.intentId &&
+          input.operation === "authorize")) &&
       /^[a-f0-9-]{32,36}$/i.test(input.workspaceId) &&
       typeof input.workspaceName === "string" &&
       input.workspaceName.length > 0 &&

tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts

@@ -93,6 +93,7 @@ async function main(): Promise<NotionOAuthReceipt> {
     const params = new URLSearchParams({ source: "notion", stage: "setup" });
     if (input.intentId) params.set("intent", input.intentId);
     if (input.draftId) params.set("resume", input.draftId);
+    if (input.newPersonal) params.set("new", "1");
     const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`;
     await page.goto(setupURL, { waitUntil: "domcontentloaded" });
     await page

tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts

@@ -94,6 +94,9 @@ export async function privateNotionOAuth(
   );
   const input: NotionOAuthInput = {
     ...options,
+    ...(ctx.scenario.id === "connection-setup.notion-wrong-account"
+      ? { newPersonal: true as const }
+      : {}),
     ...(options.intentId ? { issueId: ctx.issue()?.id } : {}),
     publicOrigin: origin.publicOrigin,
     companyId: ctx.input.fixtures.company.id,

tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts

@@ -90,3 +90,37 @@ export async function connectNamedNotion(ctx: ChatCatalogContext, binding: Provi
     return { connectionId: result.connectionId, readEntryIds: read };
   } catch (error) { await ctx.input.api.delete(`/api/tool-connections/${result.connectionId}`, { allowNotFound: true }); throw error; }
 }
+
+/** A known-existing personal fixture page must really be inaccessible under the
+ * independently healthy shared workspace credential; never infer mismatch only
+ * from a display name. No discovery or non-allowlisted page is attempted. */
+export async function verifyNotionPersonalPageDenied(
+  ctx: ChatCatalogContext,
+  shared: ProviderBinding,
+  pageId: string,
+) {
+  assertNotionConnection(shared);
+  if (
+    !/^[a-f0-9-]{32,36}$/i.test(pageId) ||
+    !shared.resourceIds.some((id) => normalized(id) === normalized(pageId))
+  )
+    throw new Error(
+      "Setup blocker: personal Notion page is outside explicit disposable resource allowlist",
+    );
+  const token = selectedExternalCredential(ctx, shared, "apiTokenEnv");
+  const response = await fetch(`https://api.notion.com/v1/pages/${pageId}`, {
+    method: "GET",
+    redirect: "error",
+    signal: AbortSignal.timeout(20000),
+    headers: {
+      Authorization: `Bearer ${token}`,
+      "Notion-Version": shared.prerequisites.apiVersion!,
+    },
+  });
+  await response.body?.cancel();
+  if (![403, 404].includes(response.status))
+    throw new Error(
+      "Setup blocker: healthy shared Notion credential did not prove the selected personal page inaccessible",
+    );
+  return true;
+}

tests/runner-e2e/chat-catalog-domains/external-notion-setup.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, it } from "vitest";
 import {
   gradeNotionSetup,
+  personalNotionGuidance,
   notionSetupExternalCases,
 } from "./external-notion-setup.js";
 import {
@@ -246,6 +247,45 @@ describe("named Notion human setup controls", () => {
       mutate(o);
       expect(gradeNotionSetup(o).passed).toBe(false);
     });
+  it("requires independently distinct real shared/personal authority and preserves the shared audience", () => {
+    const o = valid("connection-setup.notion-wrong-account");
+    o.connections.push({ id: "shared" });
+    Object.assign(o.state, {
+      sharedConnectionId: "shared",
+      sharedWorkspaceMatched: true,
+      sharedMcpWorkspaceMatched: true,
+      sharedCannotReadPersonal: true,
+      sharedAuthorityPreserved: true,
+    });
+    expect(gradeNotionSetup(o).passed).toBe(true);
+    for (const key of [
+      "sharedWorkspaceMatched",
+      "sharedMcpWorkspaceMatched",
+      "sharedCannotReadPersonal",
+      "sharedAuthorityPreserved",
+    ]) {
+      const wrong = structuredClone(o);
+      wrong.state[key] = false;
+      expect(gradeNotionSetup(wrong).passed).toBe(false);
+    }
+    const wrong = structuredClone(o);
+    wrong.phaseReplies[0]!.body =
+      "Connect the shared company Notion account and give everyone access.";
+    expect(gradeNotionSetup(wrong).passed).toBe(false);
+  });
+  it("gates human personal setup on actual personal-account guidance", () => {
+    expect(
+      personalNotionGuidance(
+        "Please connect your personal Notion workspace using the setup request.",
+      ),
+    ).toBe(true);
+    for (const text of [
+      "Connect the shared company account.",
+      "I've connected your personal workspace.",
+      "I have already authorized your personal account.",
+    ])
+      expect(personalNotionGuidance(text)).toBe(false);
+  });
   it("rejects a fabricated or replaced resume draft", () => {
     const o = valid("connection-setup.save-and-resume");
     o.state.savedDraft = { ...receipt(), connectionId: "other" };
@@ -253,7 +293,7 @@ describe("named Notion human setup controls", () => {
     expect(gradeNotionSetup(o).passed).toBe(false);
   });
   it("keeps exact concrete callbacks and phase bounds", () => {
-    expect(notionSetupExternalCases.map((c) => c.scenarioId)).toHaveLength(3);
+    expect(notionSetupExternalCases.map((c) => c.scenarioId)).toHaveLength(4);
     for (const c of notionSetupExternalCases) {
       expect(
         [c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every(

tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts

@@ -5,13 +5,24 @@ import type {
   ConnectionGrantsResponse,
   ToolRunDecisionLookup,
   ToolProfileEffectiveSummary,
+  ToolConnectionTestCallResult,
 } from "../../../packages/shared/src/types/tool-access.js";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import { pollUntil } from "../api.js";
-import { externalRequirements } from "./external-fixtures.js";
+import {
+  externalRequirements,
+  selectedExternalBinding,
+} from "./external-fixtures.js";
 import { privateNotionOAuth } from "./external-notion-oauth.js";
-import type { NotionOAuthReceipt } from "./external-notion-oauth-contract.js";
+import {
+  notionSelfWorkspaceMatches,
+  type NotionOAuthReceipt,
+} from "./external-notion-oauth-contract.js";
 import {
   readNotionFixturePages,
+  connectNamedNotion,
+  verifyNotionWorkspace,
+  verifyNotionPersonalPageDenied,
   type NotionPageOracle,
 } from "./external-notion-provider.js";
 import {
@@ -28,6 +39,59 @@ import {
   verifyRenderedReply,
 } from "./external-support.js";
 
+export function personalNotionGuidance(text: string) {
+  return text
+    .split(/[.!?;\n]+/)
+    .some(
+      (clause) =>
+        /\bpersonal\b/i.test(clause) &&
+        /\b(?:connect|authorize|authorization|sign.?in)\b/i.test(clause) &&
+        !/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:connected|authorized)\b/i.test(
+          clause,
+        ),
+    );
+}
+async function sharedNotionAuthority(ctx: ChatCatalogContext, id: string) {
+  const connection = await ctx.input.api.get<ToolConnection>(
+    `/api/tool-connections/${id}`,
+  );
+  const { grants } = await ctx.input.api.get<ConnectionGrantsResponse>(
+    `/api/tool-connections/${id}/grants`,
+  );
+  const { installs } = await ctx.input.api.get<{
+    installs: Array<{ targetType: string; targetId: string }>;
+  }>(`/api/tool-connections/${id}/installs`);
+  const sorted = (rows: unknown[]) =>
+    rows.map((row) => JSON.stringify(row)).sort();
+  return {
+    id: connection.id,
+    companyId: connection.companyId,
+    enabled: connection.enabled,
+    credentialPolicy: connection.credentialPolicy,
+    ownership: connection.ownership,
+    grants: sorted(
+      grants.map((g) => ({
+        id: g.id,
+        kind: g.kind,
+        subjectUserId: g.subjectUserId,
+        subjectAgentId: g.subjectAgentId,
+        status: g.status,
+        members: sorted(
+          (g.members ?? []).map((m) => ({
+            subjectType: m.subjectType,
+            subjectId: m.subjectId,
+          })),
+        ),
+        delegations: sorted(
+          (g.delegations ?? []).map((d) => ({ agentId: d.agentId })),
+        ),
+      })),
+    ),
+    installs: sorted(
+      installs.map((i) => ({ targetType: i.targetType, targetId: i.targetId })),
+    ),
+  };
+}
 export function gradeNotionSetup(value: unknown) {
   const c = checks(),
     o = externalObservation(value);
@@ -100,7 +164,8 @@ export function gradeNotionSetup(value: unknown) {
       card.resolvedByUserId === o.state.humanId &&
       !card.resolvedByAgentId &&
       o.state.connectionId === receipt?.connectionId &&
-      o.connections.length === 1 &&
+      o.connections.length ===
+        (o.scenarioId === "connection-setup.notion-wrong-account" ? 2 : 1) &&
       o.state.samePersonalActive === true &&
       o.state.currentHumanPreserved === true &&
       o.state.originalSourcePreserved === true,
@@ -141,6 +206,22 @@ export function gradeNotionSetup(value: unknown) {
       o.state.pagesUnchanged === true,
     "The continuation summarizes the actual recent body fact with its real date/source, distinguishing older background.",
   );
+  if (o.scenarioId === "connection-setup.notion-wrong-account") {
+    c.add(
+      "actual-shared-personal-mismatch",
+      o.state.sharedWorkspaceMatched === true &&
+        o.state.sharedMcpWorkspaceMatched === true &&
+        o.state.sharedCannotReadPersonal === true &&
+        o.state.sharedConnectionId !== o.state.connectionId &&
+        personalNotionGuidance(initial),
+      "The existing healthy shared workspace really lacks the known personal page; actual guidance requests the personal workspace rather than substituting shared notes.",
+    );
+    c.add(
+      "shared-authority-preserved",
+      o.state.sharedAuthorityPreserved === true,
+      "Actual shared account ownership, human audience, agent delegation and installs remain unchanged after creating the explicitly personal account.",
+    );
+  }
   if (o.scenarioId === "connection-setup.save-and-resume") {
     const saved = o.state.savedDraft as NotionOAuthReceipt | undefined;
     c.add(
@@ -187,6 +268,7 @@ export const notionSetupExternalCases = [
   "connection-setup.notion-missing",
   "connection-setup.return-from-login",
   "connection-setup.save-and-resume",
+  "connection-setup.notion-wrong-account",
 ].map((id) =>
   definition(
     id,
@@ -212,6 +294,61 @@ export const notionSetupExternalCases = [
         baselineConnectionIds: [],
         savedBeforeCanonical: false,
       };
+      if (id === "connection-setup.notion-wrong-account") {
+        const binding = selectedExternalBinding(ctx),
+          p = binding.prerequisites;
+        if (
+          p.sharedWorkspaceId!.replaceAll("-", "").toLowerCase() ===
+          p.workspaceId!.replaceAll("-", "").toLowerCase()
+        )
+          throw new Error(
+            "Setup blocker: shared and personal Notion workspaces must be genuinely distinct",
+          );
+        const shared = {
+          ...binding,
+          prerequisites: {
+            ...p,
+            workspaceId: p.sharedWorkspaceId!,
+            apiTokenEnv: p.sharedApiTokenEnv!,
+            mcpTokenEnv: p.sharedMcpTokenEnv!,
+          },
+        };
+        await verifyNotionWorkspace(ctx, shared);
+        await verifyNotionPersonalPageDenied(ctx, shared, p.recentPageId!);
+        const connected = await connectNamedNotion(ctx, shared, [
+          ctx.input.fixtures.agent.id,
+        ]);
+        const proof =
+          await ctx.input.api.postSensitive<ToolConnectionTestCallResult>(
+            `/api/tool-connections/${connected.connectionId}/test-calls`,
+            {
+              agentId: ctx.input.fixtures.agent.id,
+              toolName: "notion-get-self",
+              parameters: {},
+            },
+          );
+        if (
+          proof.decision !== "allowed" ||
+          proof.error ||
+          !notionSelfWorkspaceMatches(proof.result, p.sharedWorkspaceId!)
+        ) {
+          await ctx.input.api.delete(
+            `/api/tool-connections/${connected.connectionId}`,
+            { allowNotFound: true },
+          );
+          throw new Error(
+            "Setup blocker: actual existing shared MCP authorization does not match its designated workspace",
+          );
+        }
+        state.sharedMcpWorkspaceMatched = true;
+        state.sharedConnectionId = connected.connectionId;
+        state.sharedAuthorityBefore = await sharedNotionAuthority(
+          ctx,
+          connected.connectionId,
+        );
+        state.sharedWorkspaceMatched = true;
+        state.sharedCannotReadPersonal = true;
+      }
       if (id === "connection-setup.save-and-resume") {
         try {
           state.savedDraft = await privateNotionOAuth(ctx, {
@@ -299,15 +436,18 @@ export const notionSetupExternalCases = [
         ).connections;
         const saved = state.savedDraft as NotionOAuthReceipt | undefined;
         state.pendingWithoutCredential =
-          before.length === (saved ? 1 : 0) &&
-          before.every(
-            (row) =>
-              row.id === saved?.connectionId &&
-              row.status === "draft" &&
-              row.credentialPolicy === "per_user" &&
-              !row.credentialSecretRefs.length &&
-              !row.credentialRefs?.length,
-          );
+          before.filter((row) => row.id !== state.sharedConnectionId).length ===
+            (saved ? 1 : 0) &&
+          before
+            .filter((row) => row.id !== state.sharedConnectionId)
+            .every(
+              (row) =>
+                row.id === saved?.connectionId &&
+                row.status === "draft" &&
+                row.credentialPolicy === "per_user" &&
+                !row.credentialSecretRefs.length &&
+                !row.credentialRefs?.length,
+            );
         const current = await observeExternal(ctx, state);
         const initial = current.phaseReplies
           .filter((row) => row.runId === card.sourceRunId)
@@ -322,6 +462,13 @@ export const notionSetupExternalCases = [
           throw new Error(
             "Notion request was prematurely satisfied or already exposed unseen fixture facts",
           );
+        if (
+          id === "connection-setup.notion-wrong-account" &&
+          !personalNotionGuidance(initial)
+        )
+          throw new Error(
+            "Actual Notion guidance does not identify the requested personal workspace before the human acts",
+          );
         // The visible addressed card's supported full-page setup host preserves the
         // exact same intent. No API connection or completion substitutes for UI.
         state.oauthReceipt = await privateNotionOAuth(ctx, {
@@ -373,6 +520,11 @@ export const notionSetupExternalCases = [
             )
             .map((e) => e.id);
         }
+        if (typeof state.sharedConnectionId === "string")
+          state.sharedAuthorityPreserved =
+            JSON.stringify(
+              await sharedNotionAuthority(ctx, state.sharedConnectionId),
+            ) === JSON.stringify(state.sharedAuthorityBefore);
         state.decisions = await Promise.all(
           (await ctx.allRuns()).map((run) =>
             ctx.input.api.get<ToolRunDecisionLookup>(