feat(smoke): automate Notion email verification

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/connections/GENERIC-REMOTE-MCP.md

@@ -200,6 +200,12 @@ provided by the execution environment:
   under its normalized key `generic-flow-test-account`, for the existing Notion
   test account.
 
+If that account requires an emailed one-time code, also set `AGENTMAIL_API_KEY`
+and `NOTION_AGENTMAIL_INBOX_ID` for the already-configured forwarding inbox. The
+smoke loads the AgentMail SDK only after Notion presents the code challenge,
+accepts only a fresh authenticated Notion message, fills the code once in
+memory, and never records the message, address, or code.
+
 Check the URL, health endpoint, and binding metadata without retrieving the
 credential value or opening a browser:
 

package.json

@@ -72,6 +72,7 @@
   },
   "devDependencies": {
     "@playwright/test": "^1.62.1",
+    "agentmail": "^0.5.14",
     "cross-env": "^10.1.0",
     "esbuild": "^0.28.1",
     "typescript": "^5.7.3",

scripts/smoke/notion-generic-live-lib.mjs

@@ -12,6 +12,8 @@ const REQUIRED_ENVIRONMENT = [
 // normalized delivery key; the live harness never reads any other binding.
 export const NOTION_SECRET_BINDING_KEY = "generic-flow-test-account";
 
+const NOTION_LOGIN_MAIL_DOMAINS = ["notion.so", "makenotion.com"];
+
 export class NotionGenericLivePreflightError extends Error {
   constructor(code, details = {}) {
     super(code);
@@ -21,6 +23,73 @@ export class NotionGenericLivePreflightError extends Error {
   }
 }
 
+function emailDomain(value) {
+  const match = String(value ?? "").match(/@([A-Za-z0-9.-]+)/);
+  return match?.[1]?.toLowerCase().replace(/\.$/, "") ?? "";
+}
+
+function notionMailDomain(value) {
+  const domain = emailDomain(value);
+  return NOTION_LOGIN_MAIL_DOMAINS.some((allowed) => domain === allowed || domain.endsWith(`.${allowed}`));
+}
+
+export function isFreshNotionVerificationMessage(message, { notBefore }) {
+  const timestamp = new Date(message?.timestamp ?? message?.createdAt ?? Number.NaN).getTime();
+  const lowerSubject = String(message?.subject ?? "").toLowerCase();
+  const challengeSubject = /(?:login|verification|one[-\s]?time|temporary|security).*(?:code|pin)|(?:code|pin).*(?:login|verification|one[-\s]?time|temporary|security)/i.test(lowerSubject);
+  return notionMailDomain(message?.from)
+    && challengeSubject
+    && Number.isFinite(timestamp)
+    && timestamp >= new Date(notBefore).getTime();
+}
+
+export function notionVerificationAuthenticationPassed(message) {
+  const headers = message?.headers && typeof message.headers === "object" && !Array.isArray(message.headers)
+    ? message.headers
+    : {};
+  const authentication = Object.entries(headers)
+    .filter(([key]) => key.toLowerCase().includes("authentication-results"))
+    .map(([, value]) => String(value).toLowerCase())
+    .join(" ");
+  return authentication.includes("dkim=pass")
+    && (authentication.includes("dmarc=pass")
+      || authentication.includes("spf=pass")
+      || authentication.includes("arc=pass"));
+}
+
+export function extractNotionVerificationCode(message) {
+  const content = [
+    message?.subject,
+    message?.extractedText,
+    message?.text,
+    message?.extractedHtml,
+    message?.html,
+  ]
+    .filter((value) => typeof value === "string")
+    .join("\n")
+    .replace(/<[^>]+>/g, " ");
+  const digitSequence = "([0-9](?:[\\s-]?[0-9]){5})";
+  const patterns = [
+    new RegExp(`(?:login|verification|one[-\\s]?time|temporary|security)[^\\n]{0,40}(?:code|pin)[^0-9]{0,40}${digitSequence}`, "gi"),
+    new RegExp(`(?:code|pin)[^0-9]{0,40}${digitSequence}`, "gi"),
+    new RegExp(`${digitSequence}[^\\n]{0,40}(?:login|verification|one[-\\s]?time|temporary|security)?[^\\n]{0,20}(?:code|pin)`, "gi"),
+  ];
+  const codes = new Set();
+  for (const pattern of patterns) {
+    for (const match of content.matchAll(pattern)) {
+      const code = match[1].replace(/[^0-9]/g, "");
+      if (code.length === 6) codes.add(code);
+    }
+  }
+  for (const label of content.matchAll(/(?:code|pin)/gi)) {
+    const window = content.slice(label.index, label.index + 120);
+    for (const match of window.matchAll(/(^|[^0-9])([0-9](?:[\s-]?[0-9]){5})(?![0-9])/g)) {
+      codes.add(match[2].replace(/[^0-9]/g, ""));
+    }
+  }
+  return codes.size === 1 ? [...codes][0] : null;
+}
+
 function requiredValue(environment, key) {
   const value = environment[key];
   return typeof value === "string" ? value.trim() : "";

scripts/smoke/notion-generic-live.mjs

@@ -7,9 +7,12 @@ import {
   assertSanitizedEvidence,
   connectionRemovalFacts,
   extractNotionIdentity,
+  extractNotionVerificationCode,
   fetchNotionTestCredentials,
   inspectAuthorizationUrl,
+  isFreshNotionVerificationMessage,
   NotionGenericLivePreflightError,
+  notionVerificationAuthenticationPassed,
   parseRuntimeAbsenceProof,
   parseSanitizedAgentProof,
   persistedOAuthStartResult,
@@ -24,6 +27,7 @@ const NOTION_RESOURCE = "https://mcp.notion.com/mcp";
 const NOTION_GET_SELF = "notion-get-self";
 const NOTION_CREATE_PAGES = "notion-create-pages";
 const DEFAULT_AGENT_TIMEOUT_MS = 15 * 60_000;
+const DEFAULT_VERIFICATION_TIMEOUT_MS = 2 * 60_000;
 
 class SmokeFailure extends Error {
   constructor(checkpoint, code) {
@@ -198,11 +202,73 @@ async function clickVisibleButton(page, names) {
   return false;
 }
 
+function verificationTimeoutMs() {
+  const configured = Number(process.env.NOTION_VERIFICATION_TIMEOUT_MS || DEFAULT_VERIFICATION_TIMEOUT_MS);
+  return Number.isFinite(configured)
+    ? Math.min(Math.max(configured, 30_000), 5 * 60_000)
+    : DEFAULT_VERIFICATION_TIMEOUT_MS;
+}
+
+async function fetchNotionVerificationCodeFromAgentMail({ notBefore }) {
+  const apiKey = process.env.AGENTMAIL_API_KEY?.trim();
+  const inboxId = process.env.NOTION_AGENTMAIL_INBOX_ID?.trim();
+  if (!apiKey || !inboxId) fail("C.notion-login", "verification_inbox_unavailable");
+
+  let AgentMailClient;
+  try {
+    ({ AgentMailClient } = await import("agentmail"));
+  } catch {
+    fail("C.notion-login", "agentmail_sdk_unavailable");
+  }
+  const client = new AgentMailClient({ apiKey });
+  try {
+    const inbox = await client.inboxes.get(inboxId);
+    if (inbox.inboxId !== inboxId && inbox.email !== inboxId) {
+      fail("C.notion-login", "verification_inbox_mismatch");
+    }
+  } catch (error) {
+    if (error instanceof SmokeFailure) throw error;
+    fail("C.notion-login", "verification_inbox_request_failed");
+  }
+
+  const deadline = Date.now() + verificationTimeoutMs();
+  const after = new Date(new Date(notBefore).getTime() - 5_000);
+  while (Date.now() < deadline) {
+    try {
+      let pageToken;
+      let inspected = 0;
+      do {
+        const response = await client.inboxes.messages.list(inboxId, {
+          limit: 50,
+          after,
+          ...(pageToken ? { pageToken } : {}),
+        });
+        for (const item of response.messages) {
+          inspected += 1;
+          if (!isFreshNotionVerificationMessage(item, { notBefore: after })) continue;
+          const message = await client.inboxes.messages.get(inboxId, item.messageId);
+          if (!notionVerificationAuthenticationPassed(message)) continue;
+          const code = extractNotionVerificationCode(message);
+          if (code) return code;
+        }
+        pageToken = response.nextPageToken;
+      } while (pageToken && inspected < 100);
+    } catch (error) {
+      if (error instanceof SmokeFailure) throw error;
+      fail("C.notion-login", "verification_inbox_request_failed");
+    }
+    await new Promise((resolve) => setTimeout(resolve, 2_000));
+  }
+  fail("C.notion-login", "verification_code_timed_out");
+}
+
 async function completeNotionAuthorization(page, config, credential, connectionId) {
   const paperclipOrigin = new URL(config.baseUrl).origin;
   const setupPath = `/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/setup`;
   const deadline = Date.now() + 6 * 60_000;
+  const verificationNotBefore = new Date();
   let providerSeen = false;
+  let verificationCodeSubmitted = false;
   while (Date.now() < deadline) {
     let current;
     try {
@@ -219,7 +285,25 @@ async function completeNotionAuthorization(page, config, credential, connectionI
 
     const bodyText = await page.locator("body").innerText().catch(() => "");
     if (/check your (?:email|inbox)|verification code|one-time code/i.test(bodyText)) {
-      fail("C.notion-login", "interactive_code_required");
+      if (verificationCodeSubmitted) fail("C.notion-login", "verification_code_rejected");
+      const codeInput = page.locator([
+        'input[autocomplete="one-time-code"]',
+        'input[name*="code" i]',
+        'input[inputmode="numeric"]',
+      ].join(",")).filter({ visible: true }).first();
+      if (!await codeInput.count()) {
+        await page.waitForTimeout(300);
+        continue;
+      }
+      let code = await fetchNotionVerificationCodeFromAgentMail({ notBefore: verificationNotBefore });
+      await codeInput.fill(code);
+      code = "";
+      if (!await clickVisibleButton(page, [/^verify$/i, /^continue$/i, /^submit$/i, /^sign in$/i])) {
+        await codeInput.press("Enter");
+      }
+      verificationCodeSubmitted = true;
+      await page.waitForTimeout(600);
+      continue;
     }
 
     const usernameInput = page.locator([

scripts/smoke/notion-generic-live.test.mjs

@@ -5,8 +5,11 @@ import {
   assertSanitizedEvidence,
   connectionRemovalFacts,
   extractNotionIdentity,
+  extractNotionVerificationCode,
   inspectAuthorizationUrl,
+  isFreshNotionVerificationMessage,
   NotionGenericLivePreflightError,
+  notionVerificationAuthenticationPassed,
   parseRuntimeAbsenceProof,
   parseSanitizedAgentProof,
   persistedOAuthStartResult,
@@ -108,6 +111,35 @@ test("an unavailable secret binding fails before browser or credential entry", a
   assert.equal(browserLoaded, false);
 });
 
+test("selects only fresh authenticated Notion verification mail and extracts one code", () => {
+  const notBefore = new Date("2026-08-18T12:00:00.000Z");
+  const message = {
+    timestamp: new Date("2026-08-18T12:00:05.000Z"),
+    from: "Notion <login@mail.notion.so>",
+    subject: "Your Notion login code",
+    extractedText: "Your temporary login code is 123 456.",
+    headers: {
+      "authentication-results": "dkim=pass; spf=pass; dmarc=pass",
+    },
+  };
+  assert.equal(isFreshNotionVerificationMessage(message, { notBefore }), true);
+  assert.equal(notionVerificationAuthenticationPassed(message), true);
+  assert.equal(extractNotionVerificationCode(message), "123456");
+  assert.equal(isFreshNotionVerificationMessage({
+    ...message,
+    timestamp: new Date("2026-08-18T11:59:59.000Z"),
+  }, { notBefore }), false);
+  assert.equal(isFreshNotionVerificationMessage({
+    ...message,
+    from: "Notion <login@example.test>",
+  }, { notBefore }), false);
+  assert.equal(notionVerificationAuthenticationPassed({
+    ...message,
+    headers: { "authentication-results": "dkim=fail; spf=pass" },
+  }), false);
+  assert.equal(extractNotionVerificationCode({ ...message, extractedText: "Codes 123456 and 654321" }), null);
+});
+
 test("authorization proof requires automatic registration, PKCE, callback, resource, and safe endpoints", () => {
   assert.equal(assertAutomaticRegistrationSource("cimd"), "cimd");
   assert.equal(assertAutomaticRegistrationSource("dcr"), "dcr");