From 8183b6d7a3be69f49aba53dd3604e3a682117014 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 03:43:30 -0500 Subject: [PATCH] test(evals): author natural Notion expiry cancellation and real reconnect --- .../chat-catalog-domains/external-fixtures.md | 23 + .../chat-catalog-domains/external-fixtures.ts | 2 +- .../external-notion-expired.test.ts | 350 +++++++++ .../external-notion-expired.ts | 695 ++++++++++++++++++ .../external-notion-expiry.ts | 83 +++ .../external-notion-oauth-contract.ts | 16 +- .../external-notion-oauth-worker.ts | 27 +- .../external-notion-oauth.ts | 5 +- .../external-notion-setup.ts | 9 +- 9 files changed, 1194 insertions(+), 16 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-notion-expired.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/external-notion-expired.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/external-notion-expiry.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md index 4aafd0f4985..9c11496d55d 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md @@ -298,3 +298,26 @@ connection route prevents reusing the organization account; no shared audience, delegation, or install is broadened. Account refusal alone cannot satisfy this case: its actual personal setup, callback and source-bound launch read must all complete. + +`notion-expired` additionally requires `expiredAuthorizationEnv`, +`authorizationExpiresAt`, and `authorizationFailureKind=natural-expiry`. The +selected real official MCP OAuth access credential must initially work for the +designated workspace and expire in two to eight minutes. The historical fixture +uses supported public encrypted-secret and typed connection APIs, with the fixed +Notion/mcp-oauth descriptor and real credential reference. Actual health, +catalog discovery, assignment and get-self must succeed before expiry. It does +not invent an OAuth response, refresh token, healthy receipt or failed row. +After the declared time, at most three fixed read-only MCP probes must observe +actual HTTP401; failure to qualify this condition is a setup blocker. No token +revocation, rotation, provider write or arbitrary metadata discovery is used. + +The unchanged canonical request must then produce its own failed native read and +exact existing-account reconnect handoff. A real addressed Not now decision and +truthful cancellation continuation are saved before an explicit second unchanged +human query. Only actual normal Notion OAuth reconnection of that same shared +connection, preserved audience/agent authority, and a new source-bound native +page read can pass. The private reconnect receipt explicitly records that it +operated on an existing credential-bearing shared account, rather than claiming +a new credential-free personal draft. The finite case declares four native runs, +two actual human queries and a 25-minute attempt ceiling. Missing qualified +credential/session/origin stays separate from authored callback completeness. diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts index 5088d95fc3d..e904b12097e 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.ts @@ -23,7 +23,7 @@ export const externalRequirements: Record { + it("accepts independent natural expiry, retained cancellation and a fresh same-account read", () => + expect(gradeNotionExpired(valid()).passed).toBe(true)); + const wrong: Array<[string, (o: ExternalObservation) => void]> = [ + [ + "no real expiry", + (o) => { + o.state.actualProvider401 = false; + }, + ], + [ + "credential was invalid from the start", + (o) => { + o.state.validBeforeExpiry = false; + }, + ], + [ + "no original native failed read", + (o) => { + o.state.firstAuthFailure = false; + }, + ], + [ + "false initial completion", + (o) => { + o.state.firstReply = + "I've already reconnected Notion and read the launch."; + }, + ], + [ + "wrong repair route", + (o) => { + o.state.exactReconnectPath = false; + }, + ], + [ + "cancel was used as consent", + (o) => { + o.state.cancelNoSuccessfulRead = false; + }, + ], + [ + "cancellation fabricated", + (o) => { + o.state.cancelledSnapshot = false; + }, + ], + [ + "unrelated second request", + (o) => { + o.humanQueries[1] = "Summarize a different workspace."; + }, + ], + [ + "same request resolved twice", + (o) => { + o.state.finalIntentId = "first"; + }, + ], + [ + "wrong account repaired", + (o) => { + (o.state.repairReceipt as NotionOAuthReceipt).connectionId = "other"; + }, + ], + [ + "provider unavailable without positive outcome", + (o) => { + o.state.repairReceipt = undefined; + o.state.decisions = []; + }, + ], + [ + "shared audience expanded", + (o) => { + o.state.authorityPreserved = false; + }, + ], + [ + "no fresh native provider read", + (o) => { + o.state.decisions = []; + }, + ], + [ + "cached facts claimed before repair", + (o) => { + o.state.firstReply = + "Notion authorization expired; reconnect it. The launch is November 8."; + }, + ], + [ + "title-only final summary", + (o) => { + o.state.repairedReply = "The launch page title is promising."; + }, + ], + ]; + for (const [name, mutate] of wrong) + it(`rejects ${name}`, () => { + const o = valid(); + mutate(o); + expect(gradeNotionExpired(o).passed).toBe(false); + }); + it("qualifies only a finite explicit natural-expiry window", () => { + const now = Date.parse("2026-10-27T10:00:00Z"), + p = { + authorizationExpiresAt: new Date(now + 180000).toISOString(), + authorizationFailureKind: "natural-expiry", + }; + expect(qualifiedNotionExpiry(p, now)).toBe(true); + for (const change of [ + { authorizationFailureKind: "revoked" }, + { authorizationExpiresAt: new Date(now - 1).toISOString() }, + { authorizationExpiresAt: new Date(now + 3600000).toISOString() }, + { authorizationExpiresAt: "invalid" }, + ]) + expect(qualifiedNotionExpiry({ ...p, ...change }, now)).toBe(false); + }); + it("binds actual authentication failure to exact native run, source and account", () => { + const rows = [ + { + runId: "run", + decisions: [ + { + invocation: { + runId: "run", + issueId: "source", + connectionId: "connection", + status: "failed", + errorCode: "upstream_error", + errorMessage: "HTTP 401 Unauthorized", + }, + }, + ], + }, + ] as unknown as ToolRunDecisionLookup[]; + expect(hasNotionAuthFailure(rows, "run", "connection", "source")).toBe( + true, + ); + for (const args of [ + ["other", "connection", "source"], + ["run", "other", "source"], + ["run", "connection", "other"], + ]) + expect(hasNotionAuthFailure(rows, args[0]!, args[1]!, args[2]!)).toBe( + false, + ); + const wrong = structuredClone(rows); + wrong[0]!.decisions[0]!.invocation.status = "succeeded"; + expect(hasNotionAuthFailure(wrong, "run", "connection", "source")).toBe( + false, + ); + }); + it("accepts truthful repair direction but rejects affirmative completed claims", () => { + for (const text of [ + "I have not reconnected Notion; please reconnect the expired account.", + "Notion authorization expired. Use its reconnect path.", + ]) + expect(notionRepairGuidance(text)).toBe(true); + for (const text of [ + "I've already repaired Notion; reconnect again if needed.", + "I have already read Notion after reconnecting.", + ]) + expect(notionRepairGuidance(text)).toBe(false); + }); + it("keeps public reconnect distinct from new personal setup in private receipts", () => { + const storage = (domain: string) => ({ + cookies: [ + { + name: "session", + value: "synthetic-private-value", + domain, + path: "/", + secure: true, + }, + ], + origins: [], + }); + const input: NotionOAuthInput = { + operation: "authorize", + publicOrigin: "https://owned.example.test", + companyId: "company", + prefix: "ACORN", + humanId: "person", + agentId: "ceo", + intentId: "second", + issueId: "source", + reconnectId: "connection", + workspaceId: "11111111-1111-4111-8111-111111111111", + workspaceName: "Synthetic Launch", + providerEmail: "synthetic@example.test", + publicSession: storage("owned.example.test"), + notionSession: storage(".notion.so"), + }; + const receipt = valid().state.repairReceipt as NotionOAuthReceipt; + expect(validNotionOAuthInput(input)).toBe(true); + expect(validNotionOAuthReceipt(receipt, input)).toBe(true); + expect( + validNotionOAuthReceipt({ ...receipt, personalDraft: true }, input), + ).toBe(false); + expect( + validNotionOAuthReceipt({ ...receipt, connectionId: "other" }, input), + ).toBe(false); + expect(validNotionOAuthInput({ ...input, draftId: "another" })).toBe(false); + }); + it("provides all callbacks and explicitly bounded actual phases", () => { + const c = notionExpiredExternalCases[0]!; + expect( + [c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every( + (f) => typeof f === "function", + ), + ).toBe(true); + expect(c.expectedRunCount).toBe(4); + expect(c.expectedHumanMessageCount).toBe(2); + expect(c.coverageGaps).toBeUndefined(); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-expired.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-expired.ts new file mode 100644 index 00000000000..a904ecb0109 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-expired.ts @@ -0,0 +1,695 @@ +import { expect } from "@playwright/test"; +import { isDeepStrictEqual } from "node:util"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js"; +import type { + ToolConnection, + ToolConnectionHealthCheckResult, + ToolCatalogRefreshResult, + ToolConnectionTestCallResult, + ToolRunDecisionLookup, +} from "../../../packages/shared/src/types/tool-access.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { + externalRequirements, + selectedExternalBinding, + selectedExternalCredential, +} from "./external-fixtures.js"; +import { + readNotionFixturePages, + type NotionPageOracle, +} from "./external-notion-provider.js"; +import { + probeNotionAuthorization, + qualifiedNotionExpiry, +} from "./external-notion-expiry.js"; +import { privateNotionOAuth } from "./external-notion-oauth.js"; +import { + notionSelfWorkspaceMatches, + type NotionOAuthReceipt, +} from "./external-notion-oauth-contract.js"; +import { + notionSetupSourceFiles, + sharedNotionAuthority, +} from "./external-notion-setup.js"; +import { + notionDateMentioned, + notionFactSpan, + notionPageIdentityMatches, +} from "./external-notion-semantics.js"; +import { + definition, + checks, + externalObservation, + observeExternal, + root, + verifyRenderedReply, +} from "./external-support.js"; +async function decisions(ctx: ChatCatalogContext) { + return Promise.all( + (await ctx.allRuns()).map((run) => + ctx.input.api.get( + `${root(ctx)}/tools/runs/${run.id}/decisions`, + ), + ), + ); +} +export function hasNotionAuthFailure( + rows: ToolRunDecisionLookup[], + runId: string, + connectionId: string, + issueId: string, +) { + return rows.some( + (row) => + row.runId === runId && + row.decisions.some( + (d) => + d.invocation.runId === runId && + d.invocation.issueId === issueId && + d.invocation.connectionId === connectionId && + d.invocation.status === "failed" && + /401|unauthori[sz]ed|authentication|reauthorization|oauth.*(?:expired|required)|invalid.grant/i.test( + `${d.invocation.errorCode ?? ""} ${d.invocation.errorMessage ?? ""}`, + ), + ), + ); +} +export function notionRepairGuidance(reply: string) { + const falseCompletion = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => { + const claim = + /\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:reconnected|repaired|read|reviewed|summarized)\b|\b(?:Notion|(?:this|the) account)\s+(?:is|has been)\s+(?:now |already )?(?:reconnected|repaired|connected)\b/i.exec( + clause, + ); + if (!claim) return false; + return !/\b(?:once|after|when|if|until|before)\b[^,]{0,80}$/i.test( + clause.slice(0, claim.index), + ); + }); + return ( + /notion/i.test(reply) && + /reconnect|reauthori[sz]|sign.?in.*again|authorization.*expir/i.test( + reply, + ) && + !falseCompletion + ); +} +async function pending(ctx: ChatCatalogContext, excluded: string[] = []) { + const rows = await pollUntil({ + label: "new original-account Notion repair request", + deadlineAt: Date.now() + 150000, + load: async () => { + await ctx.refreshIssue(); + return ctx.issue() + ? ctx.input.api.get( + `/api/issues/${ctx.issue()!.id}/interactions`, + ) + : []; + }, + accept: (rows) => + rows.some( + (r) => + r.kind === "connection_intent" && + r.payload.serviceSlug === "notion" && + r.status === "pending" && + !excluded.includes(r.id), + ), + }); + const selected = rows.filter( + (r) => + r.kind === "connection_intent" && + r.payload.serviceSlug === "notion" && + r.status === "pending" && + !excluded.includes(r.id), + ); + if (selected.length !== 1) + throw new Error("Repair requires exactly one new actual Notion request"); + if (typeof selected[0]!.sourceRunId !== "string") + throw new Error( + "Notion repair request lacks actual source native run attribution", + ); + return { ...selected[0]!, sourceRunId: selected[0]!.sourceRunId }; +} +async function waitForPhase(ctx: ChatCatalogContext, runId: string) { + await pollUntil({ + label: "actual source-specific Notion request phase", + deadlineAt: Date.now() + 150000, + load: async () => { + await ctx.refreshIssue(); + return ctx.allRuns(); + }, + accept: (rows) => + rows.some((r) => r.id === runId && r.status === "succeeded") && + ctx.issue()?.conversationState === "waiting", + reject: (rows) => + rows.some( + (r) => + r.id === runId && + ["failed", "cancelled", "timed_out"].includes(r.status), + ) + ? "Actual Notion source phase failed" + : undefined, + }); +} +export function gradeNotionExpired(value: unknown) { + const c = checks(), + o = externalObservation(value); + c.add( + "actual-expired-notion-phases", + !!o, + "Actual expired provider authorization, cancellation and later successful same-account repair are required.", + ); + if (!o) return c.result(); + const first = o.interactions.find( + (r) => r.id === o.state.firstIntentId, + ) as unknown as ConnectionIntentInteraction | undefined; + const final = o.interactions.find( + (r) => r.id === o.state.finalIntentId, + ) as unknown as ConnectionIntentInteraction | undefined; + const receipt = o.state.repairReceipt as NotionOAuthReceipt | undefined; + c.add( + "genuine-natural-expiry-context", + o.state.validBeforeExpiry === true && + o.state.sameWorkspaceBeforeExpiry === true && + o.state.actualProvider401 === true && + o.state.expiryQualified === true, + "A real currently valid official OAuth credential first works, then actually returns provider HTTP401 after qualified natural expiry; no injected failure or token mutation.", + ); + c.add( + "original-native-authentication-error", + o.state.firstAuthFailure === true && + notionRepairGuidance(String(o.state.firstReply ?? "")) && + o.state.exactReconnectPath === true, + "The original canonical native read fails against the existing connection and truthful chat identifies its actual supported reconnect route before any human action.", + ); + c.add( + "actual-cancel-preserves-incomplete-research", + !!first && + first.result?.outcome === "declined" && + first.addresseeUserId === o.state.humanId && + first.resolvedByUserId === o.state.humanId && + !first.resolvedByAgentId && + o.state.cancelledSnapshot === true && + o.state.cancelConnectionUnchanged === true && + o.state.cancelNoSuccessfulRead === true && + o.state.truthfulCancelReply === true, + "The addressed person's actual Not now decision is retained; its own continuation reports no research, without repaired credentials, successful reads or a repeated request before a new human request.", + ); + c.add( + "explicit-repeat-request-lineage", + !!final && + final.id !== first?.id && + o.humanQueries.length === 2 && + o.humanQueries.every((q) => q === o.humanQueries[0]) && + o.state.finalAuthFailure === true && + o.state.finalGuidanceCorrect === true && + final.addresseeUserId === o.state.humanId && + final.sourceRunId === o.state.finalInitialRunId, + "A second unchanged human request creates its own distinct actual failed-read/repair phase; cancellation never acts as consent.", + ); + c.add( + "actual-same-account-human-reconnect", + !!receipt && + receipt.connectionId === o.state.connectionId && + receipt.intentId === final?.id && + receipt.humanId === o.state.humanId && + receipt.personalDraft === false && + receipt.credentialAbsentAtHandoff === false && + receipt.providerHandoff === true && + receipt.providerHumanMatched === true && + receipt.callbackReturned === true && + receipt.workspaceMatched === true && + receipt.consentCompleted === true && + receipt.processIsolation === true && + receipt.testRunnerAttached === false && + receipt.tracingEnabled === false && + receipt.videoEnabled === false && + final?.result?.outcome === "connected" && + final.result.connectionId === o.state.connectionId && + final.resolvedByUserId === o.state.humanId && + o.state.authorityPreserved === true && + o.connections.length === 1, + "Actual normal Notion OAuth reconnect repairs the same existing connection and original scope/person/request, with no duplicate account or API credential rescue.", + ); + const rows = o.state.decisions as ToolRunDecisionLookup[] | undefined; + c.add( + "fresh-post-repair-native-read", + !!rows?.some( + (row) => + row.runId !== o.state.firstInitialRunId && + row.runId !== o.state.finalInitialRunId && + (o.state.repairRunIds as string[] | undefined)?.includes(row.runId) && + row.decisions.some( + (d) => + d.invocation.runId === row.runId && + d.invocation.agentId === o.leadId && + d.invocation.issueId === o.state.sourceIssueId && + d.invocation.connectionId === o.state.connectionId && + d.invocation.status === "succeeded" && + (o.state.readEntryIds as string[] | undefined)?.includes( + d.invocation.catalogEntryId ?? "", + ), + ), + ), + "A new success continuation actually reads the real provider after human repair; independent setup probes cannot substitute.", + ); + const recent = (o.state.pages as NotionPageOracle[] | undefined)?.[0], + reply = String(o.state.repairedReply ?? ""); + c.add( + "no-pre-repair-cached-summary", + !!recent && + [ + o.state.firstReply, + o.state.finalInitialReply, + o.state.cancelReply, + ].every( + (reply) => + typeof reply === "string" && !notionFactSpan(reply, recent.fact), + ), + "Initial failure, cancellation and explicit re-request cannot report cached launch facts as fresh provider research before repair.", + ); + c.add( + "fresh-grounded-summary-after-repair", + !!recent && + !!notionFactSpan(reply, recent.fact) && + notionDateMentioned(reply, recent.editedAt) && + o.links.some((link) => notionPageIdentityMatches(link, recent.id)) && + o.state.pagesUnchanged === true, + "Only the post-repair native reply summarizes current source body/date/link, never cached material as fresh research.", + ); + return c.result(); +} +export const notionExpiredExternalCases = [ + definition( + "connection-setup.notion-expired", + async (ctx) => { + const binding = selectedExternalBinding(ctx), + p = binding.prerequisites; + if (!qualifiedNotionExpiry(p, Date.now())) + throw new Error( + "Setup blocker: designated natural-expiry Notion OAuth access token must expire in two to eight minutes", + ); + if ( + ( + await ctx.input.api.get<{ connections: ToolConnection[] }>( + `${root(ctx)}/tools/connections`, + ) + ).connections.length + ) + throw new Error( + "Expired Notion fixture requires a fresh disposable company", + ); + const pages = await readNotionFixturePages(ctx), + old = await probeNotionAuthorization(ctx, binding); + if (!old.validWorkspace || old.authenticationDenied) + throw new Error( + "Setup blocker: actual old Notion OAuth authorization is not initially valid for the selected workspace", + ); + const credential = selectedExternalCredential( + ctx, + binding, + "expiredAuthorizationEnv", + ); + const secret = await ctx.input.api.postSensitive<{ id: string }>( + `${root(ctx)}/secrets`, + { + name: "Historical Notion OAuth authorization", + key: "chat-eval-notion-old-authorization", + provider: "local_encrypted", + value: credential, + }, + ); + let connection: ToolConnection | undefined; + try { + connection = await ctx.input.api.post( + `${root(ctx)}/tools/connections`, + { + name: "Launch Notion", + applicationName: "Notion", + transport: "mcp_remote", + authKind: "oauth", + credentialPolicy: "shared", + ownership: "customer", + config: { + url: p.mcpUrl, + sourceTemplateKey: "notion", + connectionMethodKey: "mcp-oauth", + }, + transportConfig: { + url: p.mcpUrl, + sourceTemplateKey: "notion", + connectionMethodKey: "mcp-oauth", + }, + credentialSecretRefs: [ + { + configPath: "oauth.access_token", + secretId: secret.id, + versionSelector: "latest", + required: true, + }, + ], + credentialRefs: [ + { + name: "oauth.access_token", + secretId: secret.id, + version: "latest", + placement: "header", + key: "Authorization", + prefix: "Bearer ", + }, + ], + }, + ); + const health = + await ctx.input.api.post( + `/api/tool-connections/${connection.id}/health-check`, + {}, + ); + if (health.connection.healthStatus !== "ok") + throw new Error( + "Setup blocker: historical named Notion credential failed actual health qualification", + ); + const refresh = await ctx.input.api.post( + `/api/tool-connections/${connection.id}/catalog/refresh`, + {}, + ), + read = refresh.catalog + .filter((e) => e.isReadOnly && e.status === "active") + .map((e) => e.id); + if (!read.length) + throw new Error( + "Setup blocker: actual Notion read catalog is unavailable before natural expiry", + ); + await ctx.input.api.post( + `${root(ctx)}/tools/apps/${connection.id}/finish`, + { + enabledCatalogEntryIds: read, + askFirstCatalogEntryIds: [], + access: { agentIds: [ctx.input.fixtures.agent.id] }, + }, + ); + const proof = + await ctx.input.api.postSensitive( + `/api/tool-connections/${connection.id}/test-calls`, + { + agentId: ctx.input.fixtures.agent.id, + toolName: "notion-get-self", + parameters: {}, + }, + ); + if ( + proof.decision !== "allowed" || + proof.error || + !notionSelfWorkspaceMatches(proof.result, p.workspaceId!) + ) + throw new Error( + "Setup blocker: actual stored existing Notion OAuth authorization did not prove intended workspace before expiry", + ); + const authorityBefore = await sharedNotionAuthority(ctx, connection.id), + expires = Date.parse(p.authorizationExpiresAt!); + while (Date.now() < expires) + await new Promise((r) => + setTimeout(r, Math.min(10000, expires - Date.now())), + ); + let actualProvider401 = false; + for (let n = 0; n < 3; n++) { + const probe = await probeNotionAuthorization(ctx, binding); + if (probe.authenticationDenied) { + actualProvider401 = true; + break; + } + if (n < 2) await new Promise((r) => setTimeout(r, 5000)); + } + if (!actualProvider401) + throw new Error( + "Setup blocker: designated Notion authorization did not actually fail with provider HTTP401 at its qualified natural expiry", + ); + return { + connectionId: connection.id, + secretId: secret.id, + pages, + readEntryIds: read, + authorityBefore, + validBeforeExpiry: true, + sameWorkspaceBeforeExpiry: true, + expiryQualified: true, + actualProvider401, + humanId: ( + await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ) + ).user.id, + }; + } catch (error) { + if (connection) + await ctx.input.api.delete(`/api/tool-connections/${connection.id}`, { + allowNotFound: true, + }); + await ctx.input.api.delete(`${root(ctx)}/secrets/${secret.id}`, { + allowNotFound: true, + }); + throw error; + } + }, + gradeNotionExpired, + { + authentication: "human-session", + externalInput: externalRequirements["connection-setup.notion-expired"], + expectedRunCount: 4, + minimumRunCount: 4, + expectedHumanMessageCount: 2, + turnTimeoutMs: 120000, + attemptTimeoutMs: 1500000, + sourceFiles: [ + ...notionSetupSourceFiles, + "chat-catalog-domains/external-notion-expired.ts", + "chat-catalog-domains/external-notion-expiry.ts", + "chat-catalog-domains/identity-support.ts", + "chat-flow.ts", + "../../server/src/services/tool-access.ts", + ], + async act(ctx, state) { + const first = await pending(ctx); + state.firstIntentId = first.id; + state.firstInitialRunId = first.sourceRunId; + state.sourceIssueId = ctx.issue()!.id; + await waitForPhase(ctx, first.sourceRunId); + const before = await observeExternal(ctx, state); + state.firstReply = before.phaseReplies + .filter((r) => r.runId === first.sourceRunId) + .map((r) => r.body) + .join("\n"); + state.firstAuthFailure = hasNotionAuthFailure( + await decisions(ctx), + first.sourceRunId, + String(state.connectionId), + ctx.issue()!.id, + ); + const path = `/${ctx.input.fixtures.company.issuePrefix}/apps/connect`, + correct = (href: string) => { + try { + const u = new URL(href, ctx.input.api.baseURL); + return ( + u.origin === ctx.input.api.baseURL && + u.pathname === path && + u.searchParams.get("source") === "notion" && + u.searchParams.get("reconnect") === state.connectionId + ); + } catch { + return false; + } + }; + state.exactReconnectPath = before.links.some(correct); + if ( + first.addresseeUserId !== state.humanId || + !state.firstAuthFailure || + !state.exactReconnectPath || + !notionRepairGuidance(String(state.firstReply)) + ) + throw new Error( + "Actual initial Notion error/guidance does not identify the real existing account's reconnect flow", + ); + await expect( + ctx.input.page.getByRole("button", { name: "Not now", exact: true }), + ).toBeVisible(); + await ctx.input.page + .getByRole("button", { name: "Not now", exact: true }) + .click(); + const cancellation = await pollUntil({ + label: "truthful actual cancellation continuation", + deadlineAt: Date.now() + 150000, + load: () => observeExternal(ctx, state), + accept: (o) => + o.phaseReplies.some( + (r) => + r.runId !== first.sourceRunId && + o.runs.some( + (run) => run.id === r.runId && run.status === "succeeded", + ), + ), + }); + const cancelRows = await decisions(ctx), + cancelReply = cancellation.phaseReplies + .filter((r) => r.runId !== first.sourceRunId) + .map((r) => r.body) + .join("\n"), + recent = (state.pages as NotionPageOracle[])[0]!; + state.cancelledSnapshot = + cancellation.interactions.filter( + (r) => r.kind === "connection_intent", + ).length === 1 && + cancellation.interactions.some( + (r) => + r.id === first.id && + (r as unknown as ConnectionIntentInteraction).result?.outcome === + "declined", + ); + state.cancelConnectionUnchanged = isDeepStrictEqual( + await sharedNotionAuthority(ctx, String(state.connectionId)), + state.authorityBefore, + ); + state.cancelNoSuccessfulRead = !cancelRows.some((r) => + r.decisions.some( + (d) => + d.invocation.connectionId === state.connectionId && + d.invocation.status === "succeeded", + ), + ); + state.cancelReply = cancelReply; + state.truthfulCancelReply = + /not|couldn.t|cannot|can.t|declin|cancel|incomplete|unable/i.test( + cancelReply, + ) && + /read|summary|research|notion/i.test(cancelReply) && + !notionFactSpan(cancelReply, recent.fact) && + !/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:read|reviewed|summarized|reconnected)\b/i.test( + cancelReply, + ); + await ctx.input.evidence("chat-catalog-notion-cancel-phase.json", { + intentId: first.id, + sourceIssueId: state.sourceIssueId, + firstRunId: first.sourceRunId, + cancelRunIds: cancellation.runs + .filter((r) => r.id !== first.sourceRunId) + .map((r) => r.id), + cancelled: state.cancelledSnapshot, + noSuccessfulRead: state.cancelNoSuccessfulRead, + authorityUnchanged: state.cancelConnectionUnchanged, + truthfulReply: state.truthfulCancelReply, + }); + if ( + !state.cancelledSnapshot || + !state.cancelNoSuccessfulRead || + !state.cancelConnectionUnchanged || + !state.truthfulCancelReply + ) + throw new Error( + "Actual cancellation phase did not retain truthful incomplete research before re-request", + ); + const beforeRepairRunIds = (await ctx.allRuns()).map((r) => r.id); + await sendChatMessage(ctx.input.page, ctx.scenario.query); + const final = await pending(ctx, [first.id]); + state.finalIntentId = final.id; + state.finalInitialRunId = final.sourceRunId; + await waitForPhase(ctx, final.sourceRunId); + const again = await observeExternal(ctx, state); + const reply = again.phaseReplies + .filter((r) => r.runId === final.sourceRunId) + .map((r) => r.body) + .join("\n"); + state.finalInitialReply = reply; + state.finalAuthFailure = hasNotionAuthFailure( + await decisions(ctx), + final.sourceRunId, + String(state.connectionId), + String(state.sourceIssueId), + ); + state.finalGuidanceCorrect = + notionRepairGuidance(reply) && again.links.some(correct); + if ( + final.addresseeUserId !== state.humanId || + !state.finalAuthFailure || + !state.finalGuidanceCorrect + ) + throw new Error( + "Actual repeated request does not truthfully direct repair of the same failed Notion account", + ); + state.repairReceipt = await privateNotionOAuth(ctx, { + operation: "authorize", + intentId: final.id, + reconnectId: String(state.connectionId), + }); + state.beforeRepairRunIds = [...beforeRepairRunIds, final.sourceRunId]; + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + const o = await observeExternal(ctx, state); + state.decisions = await decisions(ctx); + state.authorityPreserved = isDeepStrictEqual( + await sharedNotionAuthority(ctx, String(state.connectionId)), + state.authorityBefore, + ); + state.repairRunIds = o.runs + .filter( + (r) => + !((state.beforeRepairRunIds as string[]) ?? []).includes(r.id), + ) + .map((r) => r.id); + state.repairedReply = o.phaseReplies + .filter((r) => (state.repairRunIds as string[]).includes(r.runId)) + .map((r) => r.body) + .join("\n"); + const pages = await readNotionFixturePages(ctx), + before = state.pages as NotionPageOracle[]; + state.pagesUnchanged = + pages.length === before.length && + pages.every( + (p, n) => + p.id === before[n]?.id && + p.editedAt === before[n]?.editedAt && + p.marker === before[n]?.marker, + ); + return { ...o, state }; + }, + async verifyResult(ctx, state, value) { + await verifyRenderedReply(ctx, value, String(state.finalInitialRunId)); + const o = externalObservation(value)!, + recent = (state.pages as NotionPageOracle[])[0]!, + href = o.links.find((link) => + notionPageIdentityMatches(link, recent.id), + ); + if (!href) + throw new Error( + "Current Notion source is missing after real reconnect", + ); + await expect( + ctx.input.page + .locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`) + .first(), + ).toBeVisible(); + await ctx.input.evidence("chat-catalog-notion-repair.json", { + connectionId: state.connectionId, + firstIntentId: state.firstIntentId, + finalIntentId: state.finalIntentId, + initialRunId: state.firstInitialRunId, + repeatedRunId: state.finalInitialRunId, + repairRunIds: state.repairRunIds, + receipt: state.repairReceipt, + authorityPreserved: state.authorityPreserved, + }); + }, + async cleanup(ctx, state) { + if (typeof state.connectionId === "string") + await ctx.input.api.delete( + `/api/tool-connections/${state.connectionId}`, + { allowNotFound: true }, + ); + if (typeof state.secretId === "string") + await ctx.input.api.delete(`${root(ctx)}/secrets/${state.secretId}`, { + allowNotFound: true, + }); + }, + }, + ), +]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-expiry.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-expiry.ts new file mode 100644 index 00000000000..02c91cbfe32 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-expiry.ts @@ -0,0 +1,83 @@ +import { Client } from "../../../server/node_modules/@modelcontextprotocol/sdk/dist/esm/client/index.js"; +import { StreamableHTTPClientTransport } from "../../../server/node_modules/@modelcontextprotocol/sdk/dist/esm/client/streamableHttp.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { ProviderBinding } from "./external-fixtures.js"; +import { selectedExternalCredential } from "./external-fixtures.js"; +import { assertNotionConnection } from "./external-notion-provider.js"; +import { notionSelfWorkspaceMatches } from "./external-notion-oauth-contract.js"; +/** Exact selected official read-only MCP probe. Never stores raw account/tool + * output, and never supplies a refresh client or performs provider discovery. */ +export async function probeNotionAuthorization( + ctx: ChatCatalogContext, + binding: ProviderBinding, +) { + assertNotionConnection(binding); + const token = selectedExternalCredential( + ctx, + binding, + "expiredAuthorizationEnv", + ); + let denied = false, + calls = 0; + const client = new Client({ + name: "paperclip-notion-expiry-control", + version: "1.0.0", + }); + client.onerror = () => undefined; + const transport = new StreamableHTTPClientTransport( + new URL(binding.prerequisites.mcpUrl!), + { + requestInit: { headers: { Authorization: `Bearer ${token}` } }, + fetch: async (url, init) => { + const target = new URL(String(url)); + if (target.href !== "https://mcp.notion.com/mcp" || ++calls > 5) + throw new Error( + "Notion expiry probe exceeded fixed resource/request bounds", + ); + const response = await fetch(target, { + ...init, + redirect: "error", + signal: AbortSignal.timeout(15000), + }); + if (response.status === 401) denied = true; + return response; + }, + reconnectionOptions: { + maxRetries: 0, + initialReconnectionDelay: 100, + maxReconnectionDelay: 100, + reconnectionDelayGrowFactor: 1, + }, + }, + ); + try { + await client.connect(transport); + const result = await client.callTool( + { name: "notion-get-self", arguments: {} }, + undefined, + { timeout: 15000 }, + ); + return { + validWorkspace: + result.isError !== true && + notionSelfWorkspaceMatches(result, binding.prerequisites.workspaceId!), + authenticationDenied: false, + }; + } catch { + return { validWorkspace: false, authenticationDenied: denied }; + } finally { + await client.close().catch(() => undefined); + } +} +export function qualifiedNotionExpiry( + prerequisites: Record, + now: number, +) { + const expiresAt = Date.parse(prerequisites.authorizationExpiresAt ?? ""); + return ( + prerequisites.authorizationFailureKind === "natural-expiry" && + Number.isFinite(expiresAt) && + expiresAt >= now + 120000 && + expiresAt <= now + 480000 + ); +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-contract.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-contract.ts index e6e914d0fdb..d3b505cb10a 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-contract.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-contract.ts @@ -9,6 +9,7 @@ export interface NotionOAuthInput { intentId?: string; issueId?: string; draftId?: string; + reconnectId?: string; newPersonal?: true; workspaceId: string; workspaceName: string; @@ -20,9 +21,9 @@ export interface NotionOAuthReceipt { connectionId: string; humanId: string; intentId: string | null; - personalDraft: true; + personalDraft: boolean; providerHandoff: true; - credentialAbsentAtHandoff: true; + credentialAbsentAtHandoff: boolean; resumedSameDraft: boolean; draftSurvivedReload: boolean; consentCompleted: boolean; @@ -99,6 +100,12 @@ export function validNotionOAuthInput(input: NotionOAuthInput) { !!input.intentId === !!input.issueId && (!input.issueId || id(input.issueId)) && (!input.draftId || id(input.draftId)) && + (!input.reconnectId || + (id(input.reconnectId) && + !input.draftId && + !input.newPersonal && + !!input.intentId && + input.operation === "authorize")) && (!input.newPersonal || (input.newPersonal === true && !input.draftId && @@ -172,11 +179,12 @@ export function validNotionOAuthReceipt( Object.keys(value).sort().join(",") === keys && id(value.connectionId) && (!input.draftId || input.draftId === value.connectionId) && + (!input.reconnectId || input.reconnectId === value.connectionId) && value.humanId === input.humanId && value.intentId === (input.intentId ?? null) && - value.personalDraft === true && + value.personalDraft === !input.reconnectId && value.providerHandoff === true && - value.credentialAbsentAtHandoff === true && + value.credentialAbsentAtHandoff === !input.reconnectId && value.resumedSameDraft === !!input.draftId && value.processIsolation === true && value.testRunnerAttached === false && diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts index 056b68538e5..68aec684c3b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth-worker.ts @@ -93,6 +93,7 @@ async function main(): Promise { const params = new URLSearchParams({ source: "notion", stage: "setup" }); if (input.intentId) params.set("intent", input.intentId); if (input.draftId) params.set("resume", input.draftId); + if (input.reconnectId) params.set("reconnect", input.reconnectId); if (input.newPersonal) params.set("new", "1"); const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`; await page.goto(setupURL, { waitUntil: "domcontentloaded" }); @@ -104,7 +105,7 @@ async function main(): Promise { exact: true, }) .waitFor(); - if (!input.draftId) { + if (!input.draftId && !input.reconnectId) { await page.getByRole("button", { name: "Change", exact: true }).click(); await page.getByRole("radio", { name: "Just me", exact: true }).click(); } @@ -117,14 +118,14 @@ async function main(): Promise { timeout: 90000, }); // Source UI starts OAuth directly from a durable resume without connectApp. - if (input.draftId) void started.catch(() => undefined); + if (input.draftId || input.reconnectId) void started.catch(() => undefined); await page .getByRole("button", { name: input.draftId ? "Finish with Notion" : "Continue to Notion", exact: true, }) .click(); - let connectionId = input.draftId; + let connectionId = input.draftId ?? input.reconnectId; if (!connectionId) { const response = await started; if (!response.ok()) @@ -149,7 +150,19 @@ async function main(): Promise { const personal = grantState.grants.filter( (g) => g.kind === "user" && g.subjectUserId === input.humanId, ); - if ( + if (input.reconnectId) { + if ( + draft.id !== input.reconnectId || + draft.companyId !== input.companyId || + draft.authKind !== "oauth" || + draft.credentialPolicy !== "shared" || + !before.some((c) => c.id === draft.id) || + !grantState.grants.some((g) => g.kind === "organization") + ) + throw new Error( + "Actual Notion reconnect changed the existing shared OAuth identity", + ); + } else if ( draft.companyId !== input.companyId || draft.credentialPolicy !== "per_user" || draft.authKind !== "oauth" || @@ -166,9 +179,9 @@ async function main(): Promise { connectionId, humanId: input.humanId, intentId: input.intentId ?? null, - personalDraft: true, + personalDraft: !input.reconnectId, providerHandoff: true, - credentialAbsentAtHandoff: true, + credentialAbsentAtHandoff: !input.reconnectId, resumedSameDraft: !!input.draftId, draftSurvivedReload: false, consentCompleted: false, @@ -300,7 +313,7 @@ async function main(): Promise { connected.companyId !== input.companyId || connected.status !== "active" || connected.healthStatus !== "ok" || - connected.credentialPolicy !== "per_user" + connected.credentialPolicy !== (input.reconnectId ? "shared" : "per_user") ) throw new Error( "Real Notion callback did not activate the same personal draft", diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts index b78d7700795..f7203e5aaff 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-oauth.ts @@ -25,7 +25,10 @@ import { * credential injection into the instrumented Test browser. */ export async function privateNotionOAuth( ctx: ChatCatalogContext, - options: Pick, + options: Pick< + NotionOAuthInput, + "operation" | "intentId" | "draftId" | "reconnectId" + >, ) { const binding = selectedExternalBinding(ctx), p = binding.prerequisites; diff --git a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts index ab0e955a584..da44eb5ce00 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-notion-setup.ts @@ -51,7 +51,10 @@ export function personalNotionGuidance(text: string) { ), ); } -async function sharedNotionAuthority(ctx: ChatCatalogContext, id: string) { +export async function sharedNotionAuthority( + ctx: ChatCatalogContext, + id: string, +) { const connection = await ctx.input.api.get( `/api/tool-connections/${id}`, ); @@ -242,7 +245,7 @@ export function gradeNotionSetup(value: unknown) { } return c.result(); } -const sourceFiles = [ +export const notionSetupSourceFiles = [ "chat-catalog-domains/external-notion-setup.ts", "chat-catalog-domains/external-notion-oauth.ts", "chat-catalog-domains/external-notion-oauth-worker.ts", @@ -380,7 +383,7 @@ export const notionSetupExternalCases = [ expectedHumanMessageCount: 1, turnTimeoutMs: 120000, attemptTimeoutMs: 900000, - sourceFiles, + sourceFiles: notionSetupSourceFiles, async act(ctx, state) { const rows = await pollUntil({ label: "actual addressed pending Notion request",