test(evals): prove secure key setup and flexible Notion semantics

tests/runner-e2e/chat-catalog-domains/external-definitions.ts

@@ -15,3 +15,4 @@ export { pluginInstallExternalCases } from "./external-plugin-install.js";
 export { mcpFailureExternalCases } from "./external-mcp-failure.js";
 export { assistantConsentExternalCases } from "./external-assistant-consent.js";
 export { notionReadExternalCases } from "./external-notion-read.js";
+export { secureKeyExternalCases } from "./external-secure-key.js";

tests/runner-e2e/chat-catalog-domains/external-fixtures.md

@@ -21,8 +21,10 @@ of credential environment variables belong there. The file schema is:
         "apiVersion": "2022-06-28",
         "apiTokenEnv": "PAPERCLIP_CHAT_EVAL_NOTION_API",
         "mcpTokenEnv": "PAPERCLIP_CHAT_EVAL_NOTION_MCP",
-        "recentMarker": "approved synthetic current launch fact",
-        "olderMarker": "approved synthetic older background fact"
+        "recentMarker": "Launch moves to November 8",
+        "olderMarker": "Launch originally targeted October 1",
+        "recentFactJson": "{\"subjectAliases\":[\"launch\",\"release\"],\"valueAliases\":[\"November 8\",\"Nov 8\",\"8 November\"]}",
+        "olderFactJson": "{\"subjectAliases\":[\"launch\",\"release\"],\"valueAliases\":[\"October 1\",\"Oct 1\",\"1 October\"]}"
       }
     }
   }
@@ -83,6 +85,14 @@ the oracle's REST reads cannot satisfy that requirement. No generic or synthetic
 MCP host is accepted as a named Notion proof. Missing bindings remain executable
 setup blockers; source-reference availability is not provider validity evidence.
 
+The two marker fields identify substantive facts present in the controlled page
+bodies, never nonces or mandatory answer wording. Their bounded fact JSON defines
+approved subject and value aliases (at most eight strings each). Each actual body
+must substantiate its subject/value relation; the answer may paraphrase that
+relation within a bounded passage. The grader accepts ordinary month-name dates
+and validates Notion links by approved host and exact page UUID rather than
+requiring a title slug, hyphenation style or query string.
+
 Synthetic notes apps, MCP gateways, local plugins, team packages, local encrypted
 secrets and test endpoints do not require this file. They use real public
 Paperclip APIs and controlled fixture implementations; they cannot count as live

tests/runner-e2e/chat-catalog-domains/external-fixtures.ts

@@ -14,7 +14,7 @@ export interface ExternalFixtureConfig { schemaVersion: 1; providers: Partial<Re
 export const externalRequirements: Record<string, { provider: ExternalProvider; fields: readonly string[] }> = {
   "projects.add-repository": { provider: "github", fields: ["siteRepositoryId", "siteRepositoryFullName", "tokenEnv"] },
   "projects.replace-repository": { provider: "github", fields: ["oldRepositoryId", "oldRepositoryFullName", "newRepositoryId", "newRepositoryFullName", "tokenEnv"] },
-  "connection-setup.notion-connected": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker"] },
+  "connection-setup.notion-connected": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
   "connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "recentPageId", "oauthBrowserSession"] },
   "connection-setup.notion-wrong-account": { provider: "notion", fields: ["sharedWorkspaceId", "personalWorkspaceId", "personalPageId", "oauthBrowserSession"] },
   "connection-setup.notion-agent-unassigned": { provider: "notion", fields: ["workspaceId", "recentPageId", "mcpUrl"] },

tests/runner-e2e/chat-catalog-domains/external-notion-provider.test.ts

@@ -4,7 +4,7 @@ import { readNotionFixturePages } from "./external-notion-provider.js";
 const workspaceId = "11111111-1111-4111-8111-111111111111", recent = "22222222-2222-4222-8222-222222222222", older = "33333333-3333-4333-8333-333333333333";
 function context() {
   const binding = { disposable: true, resourceIds: [workspaceId, recent, older], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_API", "PAPERCLIP_CHAT_EVAL_NOTION_MCP"], prerequisites: {
-    workspaceId, recentPageId: recent, olderPageId: older, mcpUrl: "https://mcp.notion.com/mcp", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_API", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_MCP", recentMarker: "Current fixture fact", olderMarker: "Earlier fixture fact",
+    workspaceId, recentPageId: recent, olderPageId: older, mcpUrl: "https://mcp.notion.com/mcp", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_API", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_MCP", recentMarker: "Current fixture fact", olderMarker: "Earlier fixture fact", recentFactJson: JSON.stringify({subjectAliases:["current"],valueAliases:["fixture fact"]}), olderFactJson: JSON.stringify({subjectAliases:["earlier"],valueAliases:["fixture fact"]}),
   } };
   return { scenario: { id: "connection-setup.notion-connected" }, input: { privateSecrets: [], privateInputs: { read: (name: string) => name === "PAPERCLIP_CHAT_EVAL_NOTION_API" ? "private-api-fixture" : "private-mcp-fixture", configuration: () => ({ schemaVersion: 1, providers: { notion: binding } }) } } } as unknown as ChatCatalogContext;
 }

tests/runner-e2e/chat-catalog-domains/external-notion-provider.ts

@@ -1,7 +1,8 @@
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import { selectedExternalBinding, selectedExternalCredential, type ProviderBinding } from "./external-fixtures.js";
 import { root } from "./external-support.js";
-export interface NotionPageOracle { id: string; url: string; editedAt: string; marker: string; title: string }
+import { notionFactSpan, parseNotionFact, type NotionFact } from "./external-notion-semantics.js";
+export interface NotionPageOracle { id: string; url: string; editedAt: string; marker: string; title: string; fact: NotionFact }
 interface Page { object: string; id: string; url: string; last_edited_time: string; parent?: { type?: string; page_id?: string; workspace?: boolean }; properties?: Record<string, { type?: string; title?: Array<{ plain_text?: string }> }> }
 interface Block { id: string; type: string; has_children: boolean; [key: string]: unknown }
 const normalized = (id: string) => id.replaceAll("-", "").toLowerCase();
@@ -16,6 +17,7 @@ export function assertNotionFixture(binding: ProviderBinding) {
   if (url.protocol !== "https:" || url.hostname !== "mcp.notion.com" || url.pathname !== "/mcp" || url.username || url.password || url.search || url.hash) throw new Error("Setup blocker: named Notion proof requires the actual official Notion MCP resource");
   if (!/^\d{4}-\d{2}-\d{2}$/.test(p.apiVersion ?? "")) throw new Error("Setup blocker: Notion API version reference is missing");
   if (!p.recentMarker || !p.olderMarker || p.recentMarker === p.olderMarker) throw new Error("Setup blocker: distinct synthetic recent/background Notion marker references are required");
+  parseNotionFact(p.recentFactJson); parseNotionFact(p.olderFactJson);
 }
 async function notionJson<T>(token: string, version: string, pathname: string): Promise<T> {
   const url = new URL(pathname, "https://api.notion.com");
@@ -60,7 +62,9 @@ export async function readNotionFixturePages(ctx: ChatCatalogContext, binding =
     const url = new URL(page.url); if (url.protocol !== "https:" || !/(^|\.)notion\.(?:so|site)$/.test(url.hostname)) throw new Error("Setup blocker: Notion page returned an unexpected source host");
     const body = await blocksText(token, p.apiVersion!, id);
     if (!body.includes(marker)) throw new Error(`Setup blocker: ${idField} no longer contains its authorized synthetic marker`);
-    result.push({ id, url: page.url, editedAt: page.last_edited_time, marker, title: Object.values(page.properties ?? {}).flatMap(row => row.title ?? []).map(row => row.plain_text ?? "").join("") });
+    const fact = parseNotionFact(p[idField === "recentPageId" ? "recentFactJson" : "olderFactJson"]);
+    if (!notionFactSpan(body, fact)) throw new Error(`Setup blocker: ${idField} body does not substantiate its bounded approved fact`);
+    result.push({ id, url: page.url, editedAt: page.last_edited_time, marker, fact, title: Object.values(page.properties ?? {}).flatMap(row => row.title ?? []).map(row => row.plain_text ?? "").join("") });
   }
   if (Date.parse(result[0]!.editedAt) <= Date.parse(result[1]!.editedAt)) throw new Error("Setup blocker: recent Notion fixture must actually be newer than the background page");
   return result;

tests/runner-e2e/chat-catalog-domains/external-notion-read.ts

@@ -2,6 +2,7 @@ import { expect } from "@playwright/test";
 import type { ToolConnection, ToolRunDecisionLookup, ToolProfileEffectiveSummary } from "../../../packages/shared/src/types/tool-access.js";
 import { externalRequirements, selectedExternalBinding } from "./external-fixtures.js";
 import { connectNamedNotion, readNotionFixturePages, type NotionPageOracle } from "./external-notion-provider.js";
+import { notionDateMentioned, notionFactSpan, notionPageIdentityMatches } from "./external-notion-semantics.js";
 import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
 export function gradeNotionRead(value: unknown) {
   const c = checks(), o = externalObservation(value);
@@ -20,10 +21,9 @@ export function gradeNotionRead(value: unknown) {
     && decision.invocation.agentId === o.leadId && decision.invocation.connectionId === o.state.connectionId && decision.invocation.issueId === o.state.sourceIssueId
     && readIds?.includes(decision.invocation.catalogEntryId ?? "") && decision.invocation.status === "succeeded")),
     "The actual canonical-query model run successfully invokes a read action from this named provider connection; independent oracle reads cannot substitute.");
-  const recentDate = recent?.editedAt.slice(0, 10);
-  c.add("latest-body-grounded-summary", !!recent && o.reply.includes(recent.marker) && !!recentDate && o.reply.includes(recentDate)
-    && o.links.some(link => { try { return new URL(link).href.replace(/\/$/, "") === recent.url.replace(/\/$/, ""); } catch { return false; } })
-    && (!older || !o.reply.includes(older.marker) || /older|background|previous|earlier/i.test(o.reply)),
+  c.add("latest-body-grounded-summary", !!recent?.fact && !!notionFactSpan(o.reply, recent.fact) && notionDateMentioned(o.reply, recent.editedAt)
+    && o.links.some(link => notionPageIdentityMatches(link, recent.id))
+    && (!older?.fact || !notionFactSpan(o.reply, older.fact) || /older|background|previous|earlier/i.test(o.reply)),
     "The answer contains the freshly observed body fact, its real source and update date, and distinguishes older background if mentioned.");
   c.add("no-needless-reconnect", !o.interactions.some(row => row.kind === "connection_intent" && row.payload?.serviceSlug === "notion")
     && !/\b(?:must|need to|please) (?:re)?connect (?:your |the )?notion/i.test(o.reply), "An already usable account is used without a needless setup request.");
@@ -38,7 +38,7 @@ export const notionReadExternalCases = [definition("connection-setup.notion-conn
   return { ...connected, pages, provider: "notion", providerOrigin: "https://mcp.notion.com", workspaceIndependentlyMatched: true, healthyBeforeQuery: true, connectionCountBefore: connections.connections.length };
 }, gradeNotionRead, {
   externalInput: externalRequirements["connection-setup.notion-connected"],
-  sourceFiles: ["chat-catalog-domains/external-notion-read.ts", "chat-catalog-domains/external-notion-provider.ts", "chat-catalog-domains/external-fixtures.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/connection-setup.yaml"],
+  sourceFiles: ["chat-catalog-domains/external-notion-read.ts", "chat-catalog-domains/external-notion-provider.ts", "chat-catalog-domains/external-notion-semantics.ts", "chat-catalog-domains/external-fixtures.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/connection-setup.yaml"],
   async observe(ctx, state) {
     const current = await readNotionFixturePages(ctx); const before = state.pages as NotionPageOracle[];
     state.fixturesUnchanged = current.length === before.length && current.every((page, index) => page.id === before[index]?.id && page.editedAt === before[index]?.editedAt && page.marker === before[index]?.marker);
@@ -48,10 +48,11 @@ export const notionReadExternalCases = [definition("connection-setup.notion-conn
   },
   async verifyResult(ctx, state, value) {
     await verifyRenderedReply(ctx, value); const page = (state.pages as NotionPageOracle[])[0]!;
-    const anchor = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(page.url)}]`).first(); await expect(anchor).toBeVisible();
+    const o = externalObservation(value)!; const href = o.links.find(link => notionPageIdentityMatches(link, page.id)); if (!href) throw new Error("No current Notion page identity in the rendered response");
+    const anchor = ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first(); await expect(anchor).toBeVisible();
     // External page browsers can require a second login. The actual source was
     // independently retrieved above; verify the rendered exact link identity.
-    await expect(anchor).toHaveAttribute("href", page.url);
+    await expect(anchor).toHaveAttribute("href", href);
     await ctx.input.evidence("chat-catalog-notion-source.json", { provider: "notion", pageId: page.id, url: page.url, editedAt: page.editedAt, workspaceMatched: true });
   },
   async cleanup(ctx, state) { if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true }); },

tests/runner-e2e/chat-catalog-domains/external-notion-semantics.ts

@@ -0,0 +1,41 @@
+export interface NotionFact { subjectAliases: string[]; valueAliases: string[] }
+export function parseNotionFact(value: string | undefined): NotionFact {
+  let fact: NotionFact;
+  try { fact = JSON.parse(value ?? "") as NotionFact; } catch { throw new Error("Setup blocker: bounded Notion fact reference is missing or invalid"); }
+  if (!fact || Object.keys(fact).some(key => !["subjectAliases", "valueAliases"].includes(key))
+    || ![fact.subjectAliases, fact.valueAliases].every(aliases => Array.isArray(aliases) && aliases.length > 0 && aliases.length <= 8
+      && aliases.every(alias => typeof alias === "string" && alias.trim().length >= 2 && alias.length <= 100 && !/[\r\n]/.test(alias)))) throw new Error("Setup blocker: bounded Notion fact aliases are invalid");
+  return fact;
+}
+const normalize = (text: string) => text.toLocaleLowerCase("en-US").normalize("NFKC").replace(/[^\p{L}\p{N}]+/gu, " ").trim();
+/** A fact is the same approved subject/value relation in a bounded passage;
+ * aliases represent fixture semantics, never an arbitrary prose verdict. */
+export function notionFactSpan(text: string, fact: NotionFact): { start: number; end: number } | undefined {
+  const subjectWords = fact.subjectAliases.map(normalize), valueWords = fact.valueAliases.map(normalize);
+  for (let start = 0; start < text.length; start += 1) {
+    if (start && /[\p{L}\p{N}]/u.test(text[start - 1]!)) continue;
+    const passage = text.slice(start, start + 240).split(/\n\s*\n/)[0]!;
+    const normalized = ` ${normalize(passage)} `;
+    if (subjectWords.some(word => normalized.includes(` ${word} `)) && valueWords.some(word => normalized.includes(` ${word} `))) return { start, end: start + passage.length };
+  }
+}
+export function notionPageIdentityMatches(href: string, pageId: string): boolean {
+  try {
+    const url = new URL(href); if (url.protocol !== "https:" || !/(^|\.)notion\.(?:so|site)$/.test(url.hostname)) return false;
+    const wanted = pageId.replaceAll("-", "").toLowerCase(); if (!/^[a-f0-9]{32}$/.test(wanted)) return false;
+    return decodeURIComponent(url.pathname).split("/").some(segment => {
+      const match = segment.match(/(?:^|-)([a-f0-9]{32}|[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$/i);
+      return match?.[1]?.replaceAll("-", "").toLowerCase() === wanted;
+    });
+  } catch { return false; }
+}
+const months = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"];
+export function notionDateMentioned(text: string, iso: string): boolean {
+  const date = new Date(iso); if (!Number.isFinite(date.getTime())) return false;
+  const year = date.getUTCFullYear(), month = date.getUTCMonth(), day = date.getUTCDate();
+  if (new RegExp(`\\b${year}-${String(month + 1).padStart(2, "0")}-${String(day).padStart(2, "0")}\\b`).test(text)) return true;
+  const name = months[month]!, monthPattern = `${name}|${name.slice(0, 3)}\\.?`;
+  const patterns = [new RegExp(`\\b(?:${monthPattern})\\s*${day}(?:st|nd|rd|th)?(?:\\s*,?\\s*(\\d{4}))?(?!\\d)`, "gi"),
+    new RegExp(`\\b${day}(?:st|nd|rd|th)?\\s+(?:${monthPattern})(?:\\s*,?\\s*(\\d{4}))?(?!\\d)`, "gi")];
+  return patterns.some(pattern => [...text.matchAll(pattern)].some(match => !match[1] || Number(match[1]) === year));
+}

tests/runner-e2e/chat-catalog-domains/external-secure-key.ts

@@ -0,0 +1,83 @@
+import { expect } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { ToolConnection } from "../../../packages/shared/src/types/tool-access.js";
+import { collectRunEvents } from "../run-observations.js";
+import { privateScanJson, bytesContainSecret, scanReachableSecretOutputs } from "./external-secret-scan.js";
+import { startNotesProvider, type NotesProvider } from "./external-provider.js";
+import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
+import { waitForReply } from "./identity-support.js";
+const fixtures = new WeakMap<ChatCatalogContext, NotesProvider>();
+export function gradeSecureKey(value: unknown) {
+  const c = checks(), o = externalObservation(value);
+  c.add("complete-secure-key-evidence", !!o, "Actual secure form submission, saved secret references, provider health and private output/draft scans are required."); if (!o) return c.result();
+  c.add("actual-secure-credential-control", o.state.challengeVisible === true && o.state.passwordInput === true && o.state.autocompleteOff === true && o.state.humanSubmittedSecurely === true
+    && o.state.secureBrowserUntraced === true && o.state.visibleReady === true, "A separate untraced human browser enters the credential only into the product's password control and sees actual completed setup.");
+  const connection = o.state.connectionAfter as { id: string; status: string; enabled: boolean; healthStatus: string; secretIds: string[]; serializedValueLeaked: boolean } | undefined;
+  c.add("stored-reference-and-verified-connectivity", !!connection && connection.id === o.state.connectionId && connection.status === "active" && connection.enabled === true
+    && connection.healthStatus === "ok" && connection.secretIds.length >= 1 && connection.serializedValueLeaked === false && Number(o.state.providerAuthorizedCatalogReads) > 0
+    && o.connections.length === 1 && o.state.secureSecretsExist === true,
+    "Exactly one active connection stores real credential references and a successful authenticated provider catalog check; no value is copied into public connection configuration.");
+  c.add("no-key-in-history-or-drafts", o.state.storageClean === true && o.state.secretScanComplete === true && o.state.secretLeaked === false,
+    "Private scans check actual browser storage, persisted comments, documents/revisions, attachments/artifacts, run events and run results for the actual credential bytes.");
+  c.add("supported-secure-entry-handoff", o.links.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === o.state.setupPath; } catch { return false; } }),
+    "The model actually links the supported secure setup entry rather than merely mentioning that credentials exist.");
+  c.add("no-ordinary-chat-key-request", !/(?:paste|send|share|provide|enter).{0,70}(?:key|credential|token).{0,40}(?:chat|message|conversation|here)|(?:paste|send|share).{0,20}(?:here|in chat).{0,40}(?:key|credential|token)/is.test(o.reply)
+    && !/\bI (?:have )?connected\b|\b(?:is|already|now) connected\b/i.test(o.reply), "The initial reply directs the secure entry step without asking for plaintext in chat or preclaiming the later human completion.");
+  return c.result();
+}
+export const secureKeyExternalCases = [definition("connection-setup.secure-api-key", async ctx => {
+  if (!ctx.input.privateSecrets) throw new Error("Secure setup requires private credential redaction ownership");
+  const provider = await startNotesProvider("Synthetic key-authenticated fixture"); fixtures.set(ctx, provider); ctx.input.privateSecrets.push(provider.token);
+  try {
+    await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
+    await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The selected account is a synthetic generic MCP provider at ${provider.url}. It authenticates using an API key entered in its existing secure setup field. No credential has been supplied in this conversation.` });
+    return { serverUrl: provider.url, setupPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/connect` };
+  } catch (error) { fixtures.delete(ctx); await provider.close(); throw error; }
+}, gradeSecureKey, {
+  sourceFiles: ["chat-catalog-domains/external-secure-key.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-secret-scan.ts", "run-observations.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); const provider = fixtures.get(ctx), browser = ctx.input.page.context().browser(); if (!provider || !browser) throw new Error("Secure provider/browser fixture missing");
+    // This context has no tracing, video or screenshots. It receives the same
+    // human session, while the source chat never receives credential bytes.
+    const secure = await browser.newContext({ baseURL: ctx.input.api.baseURL }); state.secureBrowserUntraced = true;
+    try {
+      await secure.addCookies(await ctx.input.page.context().cookies(ctx.input.api.baseURL)); const page = await secure.newPage();
+      const links = await ctx.input.page.locator(`[id^="comment-"] a`).evaluateAll(anchors => anchors.map(anchor => (anchor as HTMLAnchorElement).href));
+      const target = links.find(href => { try { const url = new URL(href); return url.origin === ctx.input.api.baseURL && url.pathname === state.setupPath; } catch { return false; } });
+      if (!target) throw new Error("The model did not link the intended secure setup entry");
+      await page.goto(target, { waitUntil: "domcontentloaded" });
+      await page.getByRole("textbox", { name: "MCP server URL", exact: true }).fill(provider.url); await page.getByRole("button", { name: "Continue", exact: true }).click();
+      await page.getByRole("button", { name: "Check link", exact: true }).click();
+      const key = page.getByLabel("App key", { exact: true }); await expect(key).toBeVisible(); state.challengeVisible = true;
+      state.passwordInput = await key.getAttribute("type") === "password"; state.autocompleteOff = await key.getAttribute("autocomplete") === "off";
+      await key.fill(provider.token);
+      const created = page.waitForResponse(response => new URL(response.url()).pathname === `${root(ctx)}/tools/apps/connect` && response.request().method() === "POST" && response.status() === 201);
+      await page.getByRole("button", { name: "Check link", exact: true }).click();
+      state.connectionId = (await (await created).json() as { connectionId: string }).connectionId; state.humanSubmittedSecurely = true;
+      await expect(page.getByRole("heading", { name: /is ready\./i })).toBeVisible(); state.visibleReady = true;
+      state.storageClean = await page.evaluate(secret => !JSON.stringify([Object.entries(localStorage), Object.entries(sessionStorage), location.href]).includes(secret), provider.token);
+      await page.getByRole("button", { name: "View connection", exact: true }).click(); await expect(key).toHaveCount(0);
+    } finally { await secure.close(); }
+  },
+  async observe(ctx, state) {
+    const provider = fixtures.get(ctx); if (!provider) throw new Error("Secure fixture missing");
+    const connection = await ctx.input.api.get<ToolConnection>(`/api/tool-connections/${state.connectionId}`);
+    const secretIds = [...new Set(connection.credentialSecretRefs.map(ref => ref.secretId))];
+    state.connectionAfter = { id: connection.id, status: connection.status, enabled: connection.enabled, healthStatus: connection.healthStatus, secretIds, serializedValueLeaked: JSON.stringify(connection).includes(provider.token) };
+    state.secretIds = secretIds; const o = await observeExternal(ctx, state); state.secureSecretsExist = secretIds.length > 0 && secretIds.every(id => o.secrets.some(secret => secret.id === id));
+    state.providerAuthorizedCatalogReads = provider.calls.filter(call => call.authorized && call.method === "tools/list").length;
+    const events: unknown[] = []; for (const run of o.runs) events.push(...await collectRunEvents(afterSeq => privateScanJson(ctx, `/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=1000`)));
+    const source = ctx.issue(); if (!source) throw new Error("Secure output scan needs the actual source task"); const reachable = new Set([source.id]);
+    for (;;) { const count = reachable.size; for (const task of o.tasks) if (task.parentId && reachable.has(task.parentId)) reachable.add(task.id); if (count === reachable.size) break; }
+    const scan = await scanReachableSecretOutputs(ctx, [...reachable], provider.token);
+    state.secretScanComplete = scan.complete; state.secretLeaked = scan.leaked || bytesContainSecret(Buffer.from(JSON.stringify([events, o.runs])), provider.token);
+    return { ...o, state };
+  },
+  async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
+  async cleanup(ctx, state) {
+    const provider = fixtures.get(ctx); fixtures.delete(ctx);
+    try { if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true });
+      for (const id of Array.isArray(state.secretIds) ? state.secretIds : []) if (typeof id === "string") await ctx.input.api.delete(`/api/secrets/${id}`, { allowNotFound: true });
+    } finally { await provider?.close(); }
+  },
+})];

tests/runner-e2e/chat-catalog-domains/external.test.ts

@@ -7,6 +7,7 @@ import { mcpFailureExternalCases } from "./external-mcp-failure.js";
 import { assistantConsentExternalCases } from "./external-assistant-consent.js";
 import { notionReadExternalCases } from "./external-notion-read.js";
 import { assertNotionFixture } from "./external-notion-provider.js";
+import { secureKeyExternalCases } from "./external-secure-key.js";
 import { assistantDeviceExternalCases } from "./external-assistant-device.js";
 import { pluginExternalCases } from "./external-plugins.js";
 import { appCatalogExternalCases } from "./external-app-catalog.js";
@@ -18,7 +19,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js";
 import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js";
 import type { ExternalObservation } from "./external-support.js";
 
-const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases, ...notionReadExternalCases];
+const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases, ...notionReadExternalCases, ...secureKeyExternalCases];
 function base(scenarioId: string): ExternalObservation {
   return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" },
     phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [],
@@ -26,12 +27,15 @@ function base(scenarioId: string): ExternalObservation {
 }
 function valid(id: string) {
   const o = base(id);
-  if (id === "connection-setup.notion-connected") {
+  if (id === "connection-setup.secure-api-key") {
+    Object.assign(o.state, { challengeVisible: true, passwordInput: true, autocompleteOff: true, humanSubmittedSecurely: true, secureBrowserUntraced: true, visibleReady: true, connectionId: "connection", connectionAfter: { id: "connection", status: "active", enabled: true, healthStatus: "ok", secretIds: ["secret"], serializedValueLeaked: false }, providerAuthorizedCatalogReads: 1, secureSecretsExist: true, storageClean: true, secretScanComplete: true, secretLeaked: false, setupPath: "/ACORN/apps/connect" });
+    o.connections = [{ id: "connection" }]; o.links = ["/ACORN/apps/connect"]; o.reply = "Use the secure setup field to enter the account API key.";
+  } else if (id === "connection-setup.notion-connected") {
     Object.assign(o.state, { provider: "notion", providerOrigin: "https://mcp.notion.com", workspaceIndependentlyMatched: true,
-      pages: [{ id: "recent", url: "https://www.notion.so/recent", editedAt: "2026-10-25T12:00:00Z", marker: "New observed launch fact" }, { id: "old", url: "https://www.notion.so/old", editedAt: "2026-10-01T12:00:00Z", marker: "Old background" }], fixturesUnchanged: true,
+      pages: [{ id: "22222222-2222-4222-8222-222222222222", url: "https://www.notion.so/22222222222242228222222222222222", editedAt: "2026-10-25T12:00:00Z", marker: "Launch moves to November 8", fact: { subjectAliases: ["launch", "release"], valueAliases: ["November 8", "Nov 8", "8 November"] } }, { id: "33333333-3333-4333-8333-333333333333", url: "https://www.notion.so/33333333333343338333333333333333", editedAt: "2026-10-01T12:00:00Z", marker: "Launch originally October 1", fact: { subjectAliases: ["launch", "release"], valueAliases: ["October 1", "Oct 1"] } }], fixturesUnchanged: true,
       connectionId: "notion", healthyBeforeQuery: true, connectionCountBefore: 1, effective: { agentId: "ceo", allowedTools: [{ id: "read" }] }, readEntryIds: ["read"], sourceIssueId: "source",
       decisions: [{ runId: "initial", decisions: [{ invocation: { runId: "initial", agentId: "ceo", connectionId: "notion", issueId: "source", catalogEntryId: "read", status: "succeeded" } }] }] });
-    o.connections = [{ id: "notion" }]; o.links = ["https://www.notion.so/recent"]; o.reply = "New observed launch fact (updated 2026-10-25).";
+    o.connections = [{ id: "notion" }]; o.links = ["https://www.notion.so/22222222222242228222222222222222"]; o.reply = "Launch moves to November 8 (updated 2026-10-25).";
   } else if (id === "assistant-connections.consent-read-only") {
     Object.assign(o.state, { requestedWrite: true, readOnlyRoleVisible: true, writeCheckboxDisabled: true, writeCheckboxUnchecked: true, companyVisible: true, viewerEscalationStatus: 403,
       viewerConnections: [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "offline_access"] }], tokenScopesReadOnly: true, viewerGrantId: "grant", humanConsented: true, viewerId: "viewer",
@@ -109,6 +113,7 @@ function valid(id: string) {
   return o;
 }
 const wrong: Record<string, (o: ExternalObservation) => void> = {
+  "connection-setup.secure-api-key": o => { o.state.storageClean = false; },
   "connection-setup.notion-connected": o => { o.reply = "The launch remains on schedule, according to the page title."; },
   "assistant-connections.consent-read-only": o => { o.state.viewerConnections = [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "paperclip:write"] }]; },
   "connection-setup.unreachable-mcp": o => { o.connections = [{ id: "phantom", status: "active" }]; },
@@ -130,6 +135,16 @@ const wrong: Record<string, (o: ExternalObservation) => void> = {
   "connection-setup.notion-declined": o => { o.interactions[0]!.resolvedByUserId = "someone-else"; },
 };
 describe("external domain independent oracles", () => {
+  it("Notion accepts a natural paraphrase/date and same-page URL variants but rejects stale or fabricated research", () => {
+    const definition = notionReadExternalCases[0]!;
+    const natural = valid("connection-setup.notion-connected"); natural.reply = "As of Oct 25, 2026, the release is now set for 8 November.";
+    natural.links = ["https://www.notion.so/Launch-update-22222222-2222-4222-8222-222222222222?source=chat"];
+    expect(definition.grade(natural).passed).toBe(true);
+    for (const reply of ["Launch update (Oct 25, 2026), according to its title.", "The launch targets October 1 (updated Oct 1, 2026).", "The release is set for 8 November (updated Oct 24, 2026).", "The release is set for 8 November (updated Oct 25, 2025)."])
+      expect(definition.grade({ ...natural, reply }).passed).toBe(false);
+    for (const url of ["https://www.notion.so/Launch-33333333333343338333333333333333", "https://notion.so.attacker.invalid/22222222222242228222222222222222"])
+      expect(definition.grade({ ...natural, links: [url] }).passed).toBe(false);
+  });
   it("viewer consent cannot borrow the operator's session or silently claim viewer chat coverage", () => {
     for (const field of ["viewerSessionBefore", "viewerSessionAtConsent", "viewerSessionAfter", "phaseBoundary"]) {
       const o = valid("assistant-connections.consent-read-only"); o.state[field] = "operator";
@@ -185,7 +200,7 @@ describe("private disposable provider prerequisite contract", () => {
   it("local fixtures need no unrelated provider account", () => { for (const definition of definitions.filter(row => !row.externalInput)) expect(externalPrerequisiteReceipt(definition.scenarioId, undefined, {}).configured).toBe(true); });
   it("named Notion fixtures require official transport and every resource in the disposable allowlist", () => {
     const workspaceId = "11111111-1111-4111-8111-111111111111", recentPageId = "22222222-2222-4222-8222-222222222222", olderPageId = "33333333-3333-4333-8333-333333333333";
-    const binding = { disposable: true as const, resourceIds: [workspaceId, recentPageId, olderPageId], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_API", "PAPERCLIP_CHAT_EVAL_NOTION_MCP"], prerequisites: { workspaceId, recentPageId, olderPageId, mcpUrl: "https://mcp.notion.com/mcp", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_API", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_MCP", recentMarker: "Recent synthetic fact", olderMarker: "Background fact" } };
+    const binding = { disposable: true as const, resourceIds: [workspaceId, recentPageId, olderPageId], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_API", "PAPERCLIP_CHAT_EVAL_NOTION_MCP"], prerequisites: { workspaceId, recentPageId, olderPageId, mcpUrl: "https://mcp.notion.com/mcp", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_API", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_MCP", recentMarker: "Recent synthetic fact", olderMarker: "Background fact", recentFactJson: JSON.stringify({subjectAliases:["launch"],valueAliases:["November 8"]}), olderFactJson: JSON.stringify({subjectAliases:["launch"],valueAliases:["October 1"]}) } };
     expect(() => assertNotionFixture(binding)).not.toThrow();
     expect(() => assertNotionFixture({ ...binding, resourceIds: [workspaceId] })).toThrow(/allowlisted/);
     expect(() => assertNotionFixture({ ...binding, prerequisites: { ...binding.prerequisites, mcpUrl: "https://synthetic.invalid/mcp" } })).toThrow(/actual official Notion/);
@@ -193,7 +208,7 @@ describe("private disposable provider prerequisite contract", () => {
   });
   it("requires explicitly disposable allowlisted resources and namespaced env references", () => {
     const config = { schemaVersion: 1, providers: { notion: { disposable: true, resourceIds: ["workspace"], credentialEnv: ["PAPERCLIP_CHAT_EVAL_NOTION_AUTH"],
-      prerequisites: { workspaceId: "workspace", recentPageId: "recent", olderPageId: "old", mcpUrl: "https://fixture.invalid", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", recentMarker: "Recent", olderMarker: "Older" } } } };
+      prerequisites: { workspaceId: "workspace", recentPageId: "recent", olderPageId: "old", mcpUrl: "https://fixture.invalid", apiVersion: "2026-03-11", apiTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", mcpTokenEnv: "PAPERCLIP_CHAT_EVAL_NOTION_AUTH", recentMarker: "Recent", olderMarker: "Older", recentFactJson: "{}", olderFactJson: "{}" } } } };
     const parsed = validateExternalConfig(config);
     const env = { PAPERCLIP_CHAT_EVAL_NOTION_AUTH: "secret-value-not-for-receipts" };
     expect(externalPrerequisiteReceipt("connection-setup.notion-connected", parsed, env)).toEqual({ scenarioId: "connection-setup.notion-connected", provider: "notion", configured: true, missing: [] });