test(evals): author natural Notion expiry cancellation and real reconnect

This commit is contained in:
Dotta committed 2026-10-27 03:43:30 -05:00
1 parent 21a19b06f3
commit 8183b6d7a3
9 files changed
+1194 -16

No files matched your search

@@ -298,3 +298,26 @@ connection route prevents reusing the organization account; no shared audience,
delegation, or install is broadened. Account refusal alone cannot satisfy this
case: its actual personal setup, callback and source-bound launch read must all
complete.
`notion-expired` additionally requires `expiredAuthorizationEnv`,
`authorizationExpiresAt`, and `authorizationFailureKind=natural-expiry`. The
selected real official MCP OAuth access credential must initially work for the
designated workspace and expire in two to eight minutes. The historical fixture
uses supported public encrypted-secret and typed connection APIs, with the fixed
Notion/mcp-oauth descriptor and real credential reference. Actual health,
catalog discovery, assignment and get-self must succeed before expiry. It does
not invent an OAuth response, refresh token, healthy receipt or failed row.
After the declared time, at most three fixed read-only MCP probes must observe
actual HTTP401; failure to qualify this condition is a setup blocker. No token
revocation, rotation, provider write or arbitrary metadata discovery is used.
The unchanged canonical request must then produce its own failed native read and
exact existing-account reconnect handoff. A real addressed Not now decision and
truthful cancellation continuation are saved before an explicit second unchanged
human query. Only actual normal Notion OAuth reconnection of that same shared
connection, preserved audience/agent authority, and a new source-bound native
page read can pass. The private reconnect receipt explicitly records that it
operated on an existing credential-bearing shared account, rather than claiming
a new credential-free personal draft. The finite case declares four native runs,
two actual human queries and a 25-minute attempt ceiling. Missing qualified
credential/session/origin stays separate from authored callback completeness.
@@ -23,7 +23,7 @@ export const externalRequirements: Record<string, { provider: ExternalProvider;
"connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
"connection-setup.notion-wrong-account": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv", "sharedWorkspaceId", "sharedApiTokenEnv", "sharedMcpTokenEnv"] },
"connection-setup.notion-agent-unassigned": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
"connection-setup.notion-expired": { provider: "notion", fields: ["workspaceId", "recentPageId", "expiredAuthorizationEnv", "oauthBrowserSession"] },
"connection-setup.notion-expired": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv", "expiredAuthorizationEnv", "authorizationExpiresAt", "authorizationFailureKind"] },
"connection-setup.return-from-login": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
"connection-setup.save-and-resume": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
"connection-setup.channel-prerequisites": { provider: "slack", fields: ["workspaceId", "channelId", "senderUserId", "botUserId", "appId", "registeredWebhookOrigin", "webhookRelayPort", "botTokenEnv", "senderTokenEnv", "signingSecretEnv", "configurationTokenEnv", "slackBrowserSessionEnv"] },
@@ -0,0 +1,350 @@
import { describe, it, expect } from "vitest";
import {
gradeNotionExpired,
hasNotionAuthFailure,
notionRepairGuidance,
notionExpiredExternalCases,
} from "./external-notion-expired.js";
import { qualifiedNotionExpiry } from "./external-notion-expiry.js";
import {
validNotionOAuthInput,
validNotionOAuthReceipt,
type NotionOAuthInput,
type NotionOAuthReceipt,
} from "./external-notion-oauth-contract.js";
import type { ExternalObservation } from "./external-support.js";
import type { ToolRunDecisionLookup } from "../../../packages/shared/src/types/tool-access.js";
const pageId = "22222222-2222-4222-8222-222222222222";
function valid(): ExternalObservation {
return {
scenarioId: "connection-setup.notion-expired",
companyId: "company",
leadId: "ceo",
origin: "http://127.0.0.1:3100",
reply: "As of Oct 26, 2026, the launch is now November 8.",
state: {
connectionId: "connection",
humanId: "person",
sourceIssueId: "source",
validBeforeExpiry: true,
sameWorkspaceBeforeExpiry: true,
actualProvider401: true,
expiryQualified: true,
firstAuthFailure: true,
firstReply:
"Notion authorization expired. Please reconnect the existing account.",
exactReconnectPath: true,
firstIntentId: "first",
finalIntentId: "second",
firstInitialRunId: "initial",
finalInitialRunId: "repeat",
cancelledSnapshot: true,
cancelConnectionUnchanged: true,
cancelNoSuccessfulRead: true,
truthfulCancelReply: true,
cancelReply:
"You declined, so I could not read Notion; the research is incomplete.",
finalInitialReply:
"Please reconnect the existing Notion account because its authorization expired.",
finalAuthFailure: true,
finalGuidanceCorrect: true,
repairReceipt: {
connectionId: "connection",
humanId: "person",
intentId: "second",
personalDraft: false,
credentialAbsentAtHandoff: false,
providerHandoff: true,
resumedSameDraft: false,
draftSurvivedReload: false,
consentCompleted: true,
callbackReturned: true,
workspaceMatched: true,
providerHumanMatched: true,
processIsolation: true,
testRunnerAttached: false,
tracingEnabled: false,
videoEnabled: false,
},
authorityPreserved: true,
repairRunIds: ["resumed"],
readEntryIds: ["read"],
decisions: [
{
runId: "resumed",
decisions: [
{
invocation: {
runId: "resumed",
agentId: "ceo",
issueId: "source",
connectionId: "connection",
catalogEntryId: "read",
status: "succeeded",
},
},
],
},
],
pages: [
{
id: pageId,
editedAt: "2026-10-26T08:00:00.000Z",
fact: {
subjectAliases: ["launch"],
valueAliases: ["November 8", "8 November"],
},
},
],
repairedReply: "As of Oct 26, 2026, the launch is now November 8.",
pagesUnchanged: true,
},
phaseReplies: [],
connections: [{ id: "connection" }],
interactions: [
{
id: "first",
kind: "connection_intent",
addresseeUserId: "person",
resolvedByUserId: "person",
result: { outcome: "declined" },
},
{
id: "second",
kind: "connection_intent",
addresseeUserId: "person",
resolvedByUserId: "person",
sourceRunId: "repeat",
result: { outcome: "connected", connectionId: "connection" },
},
],
profiles: [],
secrets: [],
endpoints: [],
activity: [],
links: [`https://www.notion.so/Launch-${pageId}`],
runs: [],
humanQueries: [
"Read my Notion and tell me the latest on the launch.",
"Read my Notion and tell me the latest on the launch.",
],
} as unknown as ExternalObservation;
}
describe("expired Notion cancellation and actual repair controls", () => {
it("accepts independent natural expiry, retained cancellation and a fresh same-account read", () =>
expect(gradeNotionExpired(valid()).passed).toBe(true));
const wrong: Array<[string, (o: ExternalObservation) => void]> = [
[
"no real expiry",
(o) => {
o.state.actualProvider401 = false;
},
],
[
"credential was invalid from the start",
(o) => {
o.state.validBeforeExpiry = false;
},
],
[
"no original native failed read",
(o) => {
o.state.firstAuthFailure = false;
},
],
[
"false initial completion",
(o) => {
o.state.firstReply =
"I've already reconnected Notion and read the launch.";
},
],
[
"wrong repair route",
(o) => {
o.state.exactReconnectPath = false;
},
],
[
"cancel was used as consent",
(o) => {
o.state.cancelNoSuccessfulRead = false;
},
],
[
"cancellation fabricated",
(o) => {
o.state.cancelledSnapshot = false;
},
],
[
"unrelated second request",
(o) => {
o.humanQueries[1] = "Summarize a different workspace.";
},
],
[
"same request resolved twice",
(o) => {
o.state.finalIntentId = "first";
},
],
[
"wrong account repaired",
(o) => {
(o.state.repairReceipt as NotionOAuthReceipt).connectionId = "other";
},
],
[
"provider unavailable without positive outcome",
(o) => {
o.state.repairReceipt = undefined;
o.state.decisions = [];
},
],
[
"shared audience expanded",
(o) => {
o.state.authorityPreserved = false;
},
],
[
"no fresh native provider read",
(o) => {
o.state.decisions = [];
},
],
[
"cached facts claimed before repair",
(o) => {
o.state.firstReply =
"Notion authorization expired; reconnect it. The launch is November 8.";
},
],
[
"title-only final summary",
(o) => {
o.state.repairedReply = "The launch page title is promising.";
},
],
];
for (const [name, mutate] of wrong)
it(`rejects ${name}`, () => {
const o = valid();
mutate(o);
expect(gradeNotionExpired(o).passed).toBe(false);
});
it("qualifies only a finite explicit natural-expiry window", () => {
const now = Date.parse("2026-10-27T10:00:00Z"),
p = {
authorizationExpiresAt: new Date(now + 180000).toISOString(),
authorizationFailureKind: "natural-expiry",
};
expect(qualifiedNotionExpiry(p, now)).toBe(true);
for (const change of [
{ authorizationFailureKind: "revoked" },
{ authorizationExpiresAt: new Date(now - 1).toISOString() },
{ authorizationExpiresAt: new Date(now + 3600000).toISOString() },
{ authorizationExpiresAt: "invalid" },
])
expect(qualifiedNotionExpiry({ ...p, ...change }, now)).toBe(false);
});
it("binds actual authentication failure to exact native run, source and account", () => {
const rows = [
{
runId: "run",
decisions: [
{
invocation: {
runId: "run",
issueId: "source",
connectionId: "connection",
status: "failed",
errorCode: "upstream_error",
errorMessage: "HTTP 401 Unauthorized",
},
},
],
},
] as unknown as ToolRunDecisionLookup[];
expect(hasNotionAuthFailure(rows, "run", "connection", "source")).toBe(
true,
);
for (const args of [
["other", "connection", "source"],
["run", "other", "source"],
["run", "connection", "other"],
])
expect(hasNotionAuthFailure(rows, args[0]!, args[1]!, args[2]!)).toBe(
false,
);
const wrong = structuredClone(rows);
wrong[0]!.decisions[0]!.invocation.status = "succeeded";
expect(hasNotionAuthFailure(wrong, "run", "connection", "source")).toBe(
false,
);
});
it("accepts truthful repair direction but rejects affirmative completed claims", () => {
for (const text of [
"I have not reconnected Notion; please reconnect the expired account.",
"Notion authorization expired. Use its reconnect path.",
])
expect(notionRepairGuidance(text)).toBe(true);
for (const text of [
"I've already repaired Notion; reconnect again if needed.",
"I have already read Notion after reconnecting.",
])
expect(notionRepairGuidance(text)).toBe(false);
});
it("keeps public reconnect distinct from new personal setup in private receipts", () => {
const storage = (domain: string) => ({
cookies: [
{
name: "session",
value: "synthetic-private-value",
domain,
path: "/",
secure: true,
},
],
origins: [],
});
const input: NotionOAuthInput = {
operation: "authorize",
publicOrigin: "https://owned.example.test",
companyId: "company",
prefix: "ACORN",
humanId: "person",
agentId: "ceo",
intentId: "second",
issueId: "source",
reconnectId: "connection",
workspaceId: "11111111-1111-4111-8111-111111111111",
workspaceName: "Synthetic Launch",
providerEmail: "synthetic@example.test",
publicSession: storage("owned.example.test"),
notionSession: storage(".notion.so"),
};
const receipt = valid().state.repairReceipt as NotionOAuthReceipt;
expect(validNotionOAuthInput(input)).toBe(true);
expect(validNotionOAuthReceipt(receipt, input)).toBe(true);
expect(
validNotionOAuthReceipt({ ...receipt, personalDraft: true }, input),
).toBe(false);
expect(
validNotionOAuthReceipt({ ...receipt, connectionId: "other" }, input),
).toBe(false);
expect(validNotionOAuthInput({ ...input, draftId: "another" })).toBe(false);
});
it("provides all callbacks and explicitly bounded actual phases", () => {
const c = notionExpiredExternalCases[0]!;
expect(
[c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every(
(f) => typeof f === "function",
),
).toBe(true);
expect(c.expectedRunCount).toBe(4);
expect(c.expectedHumanMessageCount).toBe(2);
expect(c.coverageGaps).toBeUndefined();
});
});
@@ -0,0 +1,695 @@
import { expect } from "@playwright/test";
import { isDeepStrictEqual } from "node:util";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js";
import type {
ToolConnection,
ToolConnectionHealthCheckResult,
ToolCatalogRefreshResult,
ToolConnectionTestCallResult,
ToolRunDecisionLookup,
} from "../../../packages/shared/src/types/tool-access.js";
import { pollUntil } from "../api.js";
import { sendChatMessage } from "../chat-flow.js";
import {
externalRequirements,
selectedExternalBinding,
selectedExternalCredential,
} from "./external-fixtures.js";
import {
readNotionFixturePages,
type NotionPageOracle,
} from "./external-notion-provider.js";
import {
probeNotionAuthorization,
qualifiedNotionExpiry,
} from "./external-notion-expiry.js";
import { privateNotionOAuth } from "./external-notion-oauth.js";
import {
notionSelfWorkspaceMatches,
type NotionOAuthReceipt,
} from "./external-notion-oauth-contract.js";
import {
notionSetupSourceFiles,
sharedNotionAuthority,
} from "./external-notion-setup.js";
import {
notionDateMentioned,
notionFactSpan,
notionPageIdentityMatches,
} from "./external-notion-semantics.js";
import {
definition,
checks,
externalObservation,
observeExternal,
root,
verifyRenderedReply,
} from "./external-support.js";
async function decisions(ctx: ChatCatalogContext) {
return Promise.all(
(await ctx.allRuns()).map((run) =>
ctx.input.api.get<ToolRunDecisionLookup>(
`${root(ctx)}/tools/runs/${run.id}/decisions`,
),
),
);
}
export function hasNotionAuthFailure(
rows: ToolRunDecisionLookup[],
runId: string,
connectionId: string,
issueId: string,
) {
return rows.some(
(row) =>
row.runId === runId &&
row.decisions.some(
(d) =>
d.invocation.runId === runId &&
d.invocation.issueId === issueId &&
d.invocation.connectionId === connectionId &&
d.invocation.status === "failed" &&
/401|unauthori[sz]ed|authentication|reauthorization|oauth.*(?:expired|required)|invalid.grant/i.test(
`${d.invocation.errorCode ?? ""} ${d.invocation.errorMessage ?? ""}`,
),
),
);
}
export function notionRepairGuidance(reply: string) {
const falseCompletion = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => {
const claim =
/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:reconnected|repaired|read|reviewed|summarized)\b|\b(?:Notion|(?:this|the) account)\s+(?:is|has been)\s+(?:now |already )?(?:reconnected|repaired|connected)\b/i.exec(
clause,
);
if (!claim) return false;
return !/\b(?:once|after|when|if|until|before)\b[^,]{0,80}$/i.test(
clause.slice(0, claim.index),
);
});
return (
/notion/i.test(reply) &&
/reconnect|reauthori[sz]|sign.?in.*again|authorization.*expir/i.test(
reply,
) &&
!falseCompletion
);
}
async function pending(ctx: ChatCatalogContext, excluded: string[] = []) {
const rows = await pollUntil({
label: "new original-account Notion repair request",
deadlineAt: Date.now() + 150000,
load: async () => {
await ctx.refreshIssue();
return ctx.issue()
? ctx.input.api.get<ConnectionIntentInteraction[]>(
`/api/issues/${ctx.issue()!.id}/interactions`,
)
: [];
},
accept: (rows) =>
rows.some(
(r) =>
r.kind === "connection_intent" &&
r.payload.serviceSlug === "notion" &&
r.status === "pending" &&
!excluded.includes(r.id),
),
});
const selected = rows.filter(
(r) =>
r.kind === "connection_intent" &&
r.payload.serviceSlug === "notion" &&
r.status === "pending" &&
!excluded.includes(r.id),
);
if (selected.length !== 1)
throw new Error("Repair requires exactly one new actual Notion request");
if (typeof selected[0]!.sourceRunId !== "string")
throw new Error(
"Notion repair request lacks actual source native run attribution",
);
return { ...selected[0]!, sourceRunId: selected[0]!.sourceRunId };
}
async function waitForPhase(ctx: ChatCatalogContext, runId: string) {
await pollUntil({
label: "actual source-specific Notion request phase",
deadlineAt: Date.now() + 150000,
load: async () => {
await ctx.refreshIssue();
return ctx.allRuns();
},
accept: (rows) =>
rows.some((r) => r.id === runId && r.status === "succeeded") &&
ctx.issue()?.conversationState === "waiting",
reject: (rows) =>
rows.some(
(r) =>
r.id === runId &&
["failed", "cancelled", "timed_out"].includes(r.status),
)
? "Actual Notion source phase failed"
: undefined,
});
}
export function gradeNotionExpired(value: unknown) {
const c = checks(),
o = externalObservation(value);
c.add(
"actual-expired-notion-phases",
!!o,
"Actual expired provider authorization, cancellation and later successful same-account repair are required.",
);
if (!o) return c.result();
const first = o.interactions.find(
(r) => r.id === o.state.firstIntentId,
) as unknown as ConnectionIntentInteraction | undefined;
const final = o.interactions.find(
(r) => r.id === o.state.finalIntentId,
) as unknown as ConnectionIntentInteraction | undefined;
const receipt = o.state.repairReceipt as NotionOAuthReceipt | undefined;
c.add(
"genuine-natural-expiry-context",
o.state.validBeforeExpiry === true &&
o.state.sameWorkspaceBeforeExpiry === true &&
o.state.actualProvider401 === true &&
o.state.expiryQualified === true,
"A real currently valid official OAuth credential first works, then actually returns provider HTTP401 after qualified natural expiry; no injected failure or token mutation.",
);
c.add(
"original-native-authentication-error",
o.state.firstAuthFailure === true &&
notionRepairGuidance(String(o.state.firstReply ?? "")) &&
o.state.exactReconnectPath === true,
"The original canonical native read fails against the existing connection and truthful chat identifies its actual supported reconnect route before any human action.",
);
c.add(
"actual-cancel-preserves-incomplete-research",
!!first &&
first.result?.outcome === "declined" &&
first.addresseeUserId === o.state.humanId &&
first.resolvedByUserId === o.state.humanId &&
!first.resolvedByAgentId &&
o.state.cancelledSnapshot === true &&
o.state.cancelConnectionUnchanged === true &&
o.state.cancelNoSuccessfulRead === true &&
o.state.truthfulCancelReply === true,
"The addressed person's actual Not now decision is retained; its own continuation reports no research, without repaired credentials, successful reads or a repeated request before a new human request.",
);
c.add(
"explicit-repeat-request-lineage",
!!final &&
final.id !== first?.id &&
o.humanQueries.length === 2 &&
o.humanQueries.every((q) => q === o.humanQueries[0]) &&
o.state.finalAuthFailure === true &&
o.state.finalGuidanceCorrect === true &&
final.addresseeUserId === o.state.humanId &&
final.sourceRunId === o.state.finalInitialRunId,
"A second unchanged human request creates its own distinct actual failed-read/repair phase; cancellation never acts as consent.",
);
c.add(
"actual-same-account-human-reconnect",
!!receipt &&
receipt.connectionId === o.state.connectionId &&
receipt.intentId === final?.id &&
receipt.humanId === o.state.humanId &&
receipt.personalDraft === false &&
receipt.credentialAbsentAtHandoff === false &&
receipt.providerHandoff === true &&
receipt.providerHumanMatched === true &&
receipt.callbackReturned === true &&
receipt.workspaceMatched === true &&
receipt.consentCompleted === true &&
receipt.processIsolation === true &&
receipt.testRunnerAttached === false &&
receipt.tracingEnabled === false &&
receipt.videoEnabled === false &&
final?.result?.outcome === "connected" &&
final.result.connectionId === o.state.connectionId &&
final.resolvedByUserId === o.state.humanId &&
o.state.authorityPreserved === true &&
o.connections.length === 1,
"Actual normal Notion OAuth reconnect repairs the same existing connection and original scope/person/request, with no duplicate account or API credential rescue.",
);
const rows = o.state.decisions as ToolRunDecisionLookup[] | undefined;
c.add(
"fresh-post-repair-native-read",
!!rows?.some(
(row) =>
row.runId !== o.state.firstInitialRunId &&
row.runId !== o.state.finalInitialRunId &&
(o.state.repairRunIds as string[] | undefined)?.includes(row.runId) &&
row.decisions.some(
(d) =>
d.invocation.runId === row.runId &&
d.invocation.agentId === o.leadId &&
d.invocation.issueId === o.state.sourceIssueId &&
d.invocation.connectionId === o.state.connectionId &&
d.invocation.status === "succeeded" &&
(o.state.readEntryIds as string[] | undefined)?.includes(
d.invocation.catalogEntryId ?? "",
),
),
),
"A new success continuation actually reads the real provider after human repair; independent setup probes cannot substitute.",
);
const recent = (o.state.pages as NotionPageOracle[] | undefined)?.[0],
reply = String(o.state.repairedReply ?? "");
c.add(
"no-pre-repair-cached-summary",
!!recent &&
[
o.state.firstReply,
o.state.finalInitialReply,
o.state.cancelReply,
].every(
(reply) =>
typeof reply === "string" && !notionFactSpan(reply, recent.fact),
),
"Initial failure, cancellation and explicit re-request cannot report cached launch facts as fresh provider research before repair.",
);
c.add(
"fresh-grounded-summary-after-repair",
!!recent &&
!!notionFactSpan(reply, recent.fact) &&
notionDateMentioned(reply, recent.editedAt) &&
o.links.some((link) => notionPageIdentityMatches(link, recent.id)) &&
o.state.pagesUnchanged === true,
"Only the post-repair native reply summarizes current source body/date/link, never cached material as fresh research.",
);
return c.result();
}
export const notionExpiredExternalCases = [
definition(
"connection-setup.notion-expired",
async (ctx) => {
const binding = selectedExternalBinding(ctx),
p = binding.prerequisites;
if (!qualifiedNotionExpiry(p, Date.now()))
throw new Error(
"Setup blocker: designated natural-expiry Notion OAuth access token must expire in two to eight minutes",
);
if (
(
await ctx.input.api.get<{ connections: ToolConnection[] }>(
`${root(ctx)}/tools/connections`,
)
).connections.length
)
throw new Error(
"Expired Notion fixture requires a fresh disposable company",
);
const pages = await readNotionFixturePages(ctx),
old = await probeNotionAuthorization(ctx, binding);
if (!old.validWorkspace || old.authenticationDenied)
throw new Error(
"Setup blocker: actual old Notion OAuth authorization is not initially valid for the selected workspace",
);
const credential = selectedExternalCredential(
ctx,
binding,
"expiredAuthorizationEnv",
);
const secret = await ctx.input.api.postSensitive<{ id: string }>(
`${root(ctx)}/secrets`,
{
name: "Historical Notion OAuth authorization",
key: "chat-eval-notion-old-authorization",
provider: "local_encrypted",
value: credential,
},
);
let connection: ToolConnection | undefined;
try {
connection = await ctx.input.api.post<ToolConnection>(
`${root(ctx)}/tools/connections`,
{
name: "Launch Notion",
applicationName: "Notion",
transport: "mcp_remote",
authKind: "oauth",
credentialPolicy: "shared",
ownership: "customer",
config: {
url: p.mcpUrl,
sourceTemplateKey: "notion",
connectionMethodKey: "mcp-oauth",
},
transportConfig: {
url: p.mcpUrl,
sourceTemplateKey: "notion",
connectionMethodKey: "mcp-oauth",
},
credentialSecretRefs: [
{
configPath: "oauth.access_token",
secretId: secret.id,
versionSelector: "latest",
required: true,
},
],
credentialRefs: [
{
name: "oauth.access_token",
secretId: secret.id,
version: "latest",
placement: "header",
key: "Authorization",
prefix: "Bearer ",
},
],
},
);
const health =
await ctx.input.api.post<ToolConnectionHealthCheckResult>(
`/api/tool-connections/${connection.id}/health-check`,
{},
);
if (health.connection.healthStatus !== "ok")
throw new Error(
"Setup blocker: historical named Notion credential failed actual health qualification",
);
const refresh = await ctx.input.api.post<ToolCatalogRefreshResult>(
`/api/tool-connections/${connection.id}/catalog/refresh`,
{},
),
read = refresh.catalog
.filter((e) => e.isReadOnly && e.status === "active")
.map((e) => e.id);
if (!read.length)
throw new Error(
"Setup blocker: actual Notion read catalog is unavailable before natural expiry",
);
await ctx.input.api.post(
`${root(ctx)}/tools/apps/${connection.id}/finish`,
{
enabledCatalogEntryIds: read,
askFirstCatalogEntryIds: [],
access: { agentIds: [ctx.input.fixtures.agent.id] },
},
);
const proof =
await ctx.input.api.postSensitive<ToolConnectionTestCallResult>(
`/api/tool-connections/${connection.id}/test-calls`,
{
agentId: ctx.input.fixtures.agent.id,
toolName: "notion-get-self",
parameters: {},
},
);
if (
proof.decision !== "allowed" ||
proof.error ||
!notionSelfWorkspaceMatches(proof.result, p.workspaceId!)
)
throw new Error(
"Setup blocker: actual stored existing Notion OAuth authorization did not prove intended workspace before expiry",
);
const authorityBefore = await sharedNotionAuthority(ctx, connection.id),
expires = Date.parse(p.authorizationExpiresAt!);
while (Date.now() < expires)
await new Promise((r) =>
setTimeout(r, Math.min(10000, expires - Date.now())),
);
let actualProvider401 = false;
for (let n = 0; n < 3; n++) {
const probe = await probeNotionAuthorization(ctx, binding);
if (probe.authenticationDenied) {
actualProvider401 = true;
break;
}
if (n < 2) await new Promise((r) => setTimeout(r, 5000));
}
if (!actualProvider401)
throw new Error(
"Setup blocker: designated Notion authorization did not actually fail with provider HTTP401 at its qualified natural expiry",
);
return {
connectionId: connection.id,
secretId: secret.id,
pages,
readEntryIds: read,
authorityBefore,
validBeforeExpiry: true,
sameWorkspaceBeforeExpiry: true,
expiryQualified: true,
actualProvider401,
humanId: (
await ctx.input.api.get<{ user: { id: string } }>(
"/api/auth/get-session",
)
).user.id,
};
} catch (error) {
if (connection)
await ctx.input.api.delete(`/api/tool-connections/${connection.id}`, {
allowNotFound: true,
});
await ctx.input.api.delete(`${root(ctx)}/secrets/${secret.id}`, {
allowNotFound: true,
});
throw error;
}
},
gradeNotionExpired,
{
authentication: "human-session",
externalInput: externalRequirements["connection-setup.notion-expired"],
expectedRunCount: 4,
minimumRunCount: 4,
expectedHumanMessageCount: 2,
turnTimeoutMs: 120000,
attemptTimeoutMs: 1500000,
sourceFiles: [
...notionSetupSourceFiles,
"chat-catalog-domains/external-notion-expired.ts",
"chat-catalog-domains/external-notion-expiry.ts",
"chat-catalog-domains/identity-support.ts",
"chat-flow.ts",
"../../server/src/services/tool-access.ts",
],
async act(ctx, state) {
const first = await pending(ctx);
state.firstIntentId = first.id;
state.firstInitialRunId = first.sourceRunId;
state.sourceIssueId = ctx.issue()!.id;
await waitForPhase(ctx, first.sourceRunId);
const before = await observeExternal(ctx, state);
state.firstReply = before.phaseReplies
.filter((r) => r.runId === first.sourceRunId)
.map((r) => r.body)
.join("\n");
state.firstAuthFailure = hasNotionAuthFailure(
await decisions(ctx),
first.sourceRunId,
String(state.connectionId),
ctx.issue()!.id,
);
const path = `/${ctx.input.fixtures.company.issuePrefix}/apps/connect`,
correct = (href: string) => {
try {
const u = new URL(href, ctx.input.api.baseURL);
return (
u.origin === ctx.input.api.baseURL &&
u.pathname === path &&
u.searchParams.get("source") === "notion" &&
u.searchParams.get("reconnect") === state.connectionId
);
} catch {
return false;
}
};
state.exactReconnectPath = before.links.some(correct);
if (
first.addresseeUserId !== state.humanId ||
!state.firstAuthFailure ||
!state.exactReconnectPath ||
!notionRepairGuidance(String(state.firstReply))
)
throw new Error(
"Actual initial Notion error/guidance does not identify the real existing account's reconnect flow",
);
await expect(
ctx.input.page.getByRole("button", { name: "Not now", exact: true }),
).toBeVisible();
await ctx.input.page
.getByRole("button", { name: "Not now", exact: true })
.click();
const cancellation = await pollUntil({
label: "truthful actual cancellation continuation",
deadlineAt: Date.now() + 150000,
load: () => observeExternal(ctx, state),
accept: (o) =>
o.phaseReplies.some(
(r) =>
r.runId !== first.sourceRunId &&
o.runs.some(
(run) => run.id === r.runId && run.status === "succeeded",
),
),
});
const cancelRows = await decisions(ctx),
cancelReply = cancellation.phaseReplies
.filter((r) => r.runId !== first.sourceRunId)
.map((r) => r.body)
.join("\n"),
recent = (state.pages as NotionPageOracle[])[0]!;
state.cancelledSnapshot =
cancellation.interactions.filter(
(r) => r.kind === "connection_intent",
).length === 1 &&
cancellation.interactions.some(
(r) =>
r.id === first.id &&
(r as unknown as ConnectionIntentInteraction).result?.outcome ===
"declined",
);
state.cancelConnectionUnchanged = isDeepStrictEqual(
await sharedNotionAuthority(ctx, String(state.connectionId)),
state.authorityBefore,
);
state.cancelNoSuccessfulRead = !cancelRows.some((r) =>
r.decisions.some(
(d) =>
d.invocation.connectionId === state.connectionId &&
d.invocation.status === "succeeded",
),
);
state.cancelReply = cancelReply;
state.truthfulCancelReply =
/not|couldn.t|cannot|can.t|declin|cancel|incomplete|unable/i.test(
cancelReply,
) &&
/read|summary|research|notion/i.test(cancelReply) &&
!notionFactSpan(cancelReply, recent.fact) &&
!/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:read|reviewed|summarized|reconnected)\b/i.test(
cancelReply,
);
await ctx.input.evidence("chat-catalog-notion-cancel-phase.json", {
intentId: first.id,
sourceIssueId: state.sourceIssueId,
firstRunId: first.sourceRunId,
cancelRunIds: cancellation.runs
.filter((r) => r.id !== first.sourceRunId)
.map((r) => r.id),
cancelled: state.cancelledSnapshot,
noSuccessfulRead: state.cancelNoSuccessfulRead,
authorityUnchanged: state.cancelConnectionUnchanged,
truthfulReply: state.truthfulCancelReply,
});
if (
!state.cancelledSnapshot ||
!state.cancelNoSuccessfulRead ||
!state.cancelConnectionUnchanged ||
!state.truthfulCancelReply
)
throw new Error(
"Actual cancellation phase did not retain truthful incomplete research before re-request",
);
const beforeRepairRunIds = (await ctx.allRuns()).map((r) => r.id);
await sendChatMessage(ctx.input.page, ctx.scenario.query);
const final = await pending(ctx, [first.id]);
state.finalIntentId = final.id;
state.finalInitialRunId = final.sourceRunId;
await waitForPhase(ctx, final.sourceRunId);
const again = await observeExternal(ctx, state);
const reply = again.phaseReplies
.filter((r) => r.runId === final.sourceRunId)
.map((r) => r.body)
.join("\n");
state.finalInitialReply = reply;
state.finalAuthFailure = hasNotionAuthFailure(
await decisions(ctx),
final.sourceRunId,
String(state.connectionId),
String(state.sourceIssueId),
);
state.finalGuidanceCorrect =
notionRepairGuidance(reply) && again.links.some(correct);
if (
final.addresseeUserId !== state.humanId ||
!state.finalAuthFailure ||
!state.finalGuidanceCorrect
)
throw new Error(
"Actual repeated request does not truthfully direct repair of the same failed Notion account",
);
state.repairReceipt = await privateNotionOAuth(ctx, {
operation: "authorize",
intentId: final.id,
reconnectId: String(state.connectionId),
});
state.beforeRepairRunIds = [...beforeRepairRunIds, final.sourceRunId];
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
},
async observe(ctx, state) {
const o = await observeExternal(ctx, state);
state.decisions = await decisions(ctx);
state.authorityPreserved = isDeepStrictEqual(
await sharedNotionAuthority(ctx, String(state.connectionId)),
state.authorityBefore,
);
state.repairRunIds = o.runs
.filter(
(r) =>
!((state.beforeRepairRunIds as string[]) ?? []).includes(r.id),
)
.map((r) => r.id);
state.repairedReply = o.phaseReplies
.filter((r) => (state.repairRunIds as string[]).includes(r.runId))
.map((r) => r.body)
.join("\n");
const pages = await readNotionFixturePages(ctx),
before = state.pages as NotionPageOracle[];
state.pagesUnchanged =
pages.length === before.length &&
pages.every(
(p, n) =>
p.id === before[n]?.id &&
p.editedAt === before[n]?.editedAt &&
p.marker === before[n]?.marker,
);
return { ...o, state };
},
async verifyResult(ctx, state, value) {
await verifyRenderedReply(ctx, value, String(state.finalInitialRunId));
const o = externalObservation(value)!,
recent = (state.pages as NotionPageOracle[])[0]!,
href = o.links.find((link) =>
notionPageIdentityMatches(link, recent.id),
);
if (!href)
throw new Error(
"Current Notion source is missing after real reconnect",
);
await expect(
ctx.input.page
.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`)
.first(),
).toBeVisible();
await ctx.input.evidence("chat-catalog-notion-repair.json", {
connectionId: state.connectionId,
firstIntentId: state.firstIntentId,
finalIntentId: state.finalIntentId,
initialRunId: state.firstInitialRunId,
repeatedRunId: state.finalInitialRunId,
repairRunIds: state.repairRunIds,
receipt: state.repairReceipt,
authorityPreserved: state.authorityPreserved,
});
},
async cleanup(ctx, state) {
if (typeof state.connectionId === "string")
await ctx.input.api.delete(
`/api/tool-connections/${state.connectionId}`,
{ allowNotFound: true },
);
if (typeof state.secretId === "string")
await ctx.input.api.delete(`${root(ctx)}/secrets/${state.secretId}`, {
allowNotFound: true,
});
},
},
),
];
@@ -0,0 +1,83 @@
import { Client } from "../../../server/node_modules/@modelcontextprotocol/sdk/dist/esm/client/index.js";
import { StreamableHTTPClientTransport } from "../../../server/node_modules/@modelcontextprotocol/sdk/dist/esm/client/streamableHttp.js";
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
import type { ProviderBinding } from "./external-fixtures.js";
import { selectedExternalCredential } from "./external-fixtures.js";
import { assertNotionConnection } from "./external-notion-provider.js";
import { notionSelfWorkspaceMatches } from "./external-notion-oauth-contract.js";
/** Exact selected official read-only MCP probe. Never stores raw account/tool
* output, and never supplies a refresh client or performs provider discovery. */
export async function probeNotionAuthorization(
ctx: ChatCatalogContext,
binding: ProviderBinding,
) {
assertNotionConnection(binding);
const token = selectedExternalCredential(
ctx,
binding,
"expiredAuthorizationEnv",
);
let denied = false,
calls = 0;
const client = new Client({
name: "paperclip-notion-expiry-control",
version: "1.0.0",
});
client.onerror = () => undefined;
const transport = new StreamableHTTPClientTransport(
new URL(binding.prerequisites.mcpUrl!),
{
requestInit: { headers: { Authorization: `Bearer ${token}` } },
fetch: async (url, init) => {
const target = new URL(String(url));
if (target.href !== "https://mcp.notion.com/mcp" || ++calls > 5)
throw new Error(
"Notion expiry probe exceeded fixed resource/request bounds",
);
const response = await fetch(target, {
...init,
redirect: "error",
signal: AbortSignal.timeout(15000),
});
if (response.status === 401) denied = true;
return response;
},
reconnectionOptions: {
maxRetries: 0,
initialReconnectionDelay: 100,
maxReconnectionDelay: 100,
reconnectionDelayGrowFactor: 1,
},
},
);
try {
await client.connect(transport);
const result = await client.callTool(
{ name: "notion-get-self", arguments: {} },
undefined,
{ timeout: 15000 },
);
return {
validWorkspace:
result.isError !== true &&
notionSelfWorkspaceMatches(result, binding.prerequisites.workspaceId!),
authenticationDenied: false,
};
} catch {
return { validWorkspace: false, authenticationDenied: denied };
} finally {
await client.close().catch(() => undefined);
}
}
export function qualifiedNotionExpiry(
prerequisites: Record<string, string>,
now: number,
) {
const expiresAt = Date.parse(prerequisites.authorizationExpiresAt ?? "");
return (
prerequisites.authorizationFailureKind === "natural-expiry" &&
Number.isFinite(expiresAt) &&
expiresAt >= now + 120000 &&
expiresAt <= now + 480000
);
}
@@ -9,6 +9,7 @@ export interface NotionOAuthInput {
intentId?: string;
issueId?: string;
draftId?: string;
reconnectId?: string;
newPersonal?: true;
workspaceId: string;
workspaceName: string;
@@ -20,9 +21,9 @@ export interface NotionOAuthReceipt {
connectionId: string;
humanId: string;
intentId: string | null;
personalDraft: true;
personalDraft: boolean;
providerHandoff: true;
credentialAbsentAtHandoff: true;
credentialAbsentAtHandoff: boolean;
resumedSameDraft: boolean;
draftSurvivedReload: boolean;
consentCompleted: boolean;
@@ -99,6 +100,12 @@ export function validNotionOAuthInput(input: NotionOAuthInput) {
!!input.intentId === !!input.issueId &&
(!input.issueId || id(input.issueId)) &&
(!input.draftId || id(input.draftId)) &&
(!input.reconnectId ||
(id(input.reconnectId) &&
!input.draftId &&
!input.newPersonal &&
!!input.intentId &&
input.operation === "authorize")) &&
(!input.newPersonal ||
(input.newPersonal === true &&
!input.draftId &&
@@ -172,11 +179,12 @@ export function validNotionOAuthReceipt(
Object.keys(value).sort().join(",") === keys &&
id(value.connectionId) &&
(!input.draftId || input.draftId === value.connectionId) &&
(!input.reconnectId || input.reconnectId === value.connectionId) &&
value.humanId === input.humanId &&
value.intentId === (input.intentId ?? null) &&
value.personalDraft === true &&
value.personalDraft === !input.reconnectId &&
value.providerHandoff === true &&
value.credentialAbsentAtHandoff === true &&
value.credentialAbsentAtHandoff === !input.reconnectId &&
value.resumedSameDraft === !!input.draftId &&
value.processIsolation === true &&
value.testRunnerAttached === false &&
@@ -93,6 +93,7 @@ async function main(): Promise<NotionOAuthReceipt> {
const params = new URLSearchParams({ source: "notion", stage: "setup" });
if (input.intentId) params.set("intent", input.intentId);
if (input.draftId) params.set("resume", input.draftId);
if (input.reconnectId) params.set("reconnect", input.reconnectId);
if (input.newPersonal) params.set("new", "1");
const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`;
await page.goto(setupURL, { waitUntil: "domcontentloaded" });
@@ -104,7 +105,7 @@ async function main(): Promise<NotionOAuthReceipt> {
exact: true,
})
.waitFor();
if (!input.draftId) {
if (!input.draftId && !input.reconnectId) {
await page.getByRole("button", { name: "Change", exact: true }).click();
await page.getByRole("radio", { name: "Just me", exact: true }).click();
}
@@ -117,14 +118,14 @@ async function main(): Promise<NotionOAuthReceipt> {
timeout: 90000,
});
// Source UI starts OAuth directly from a durable resume without connectApp.
if (input.draftId) void started.catch(() => undefined);
if (input.draftId || input.reconnectId) void started.catch(() => undefined);
await page
.getByRole("button", {
name: input.draftId ? "Finish with Notion" : "Continue to Notion",
exact: true,
})
.click();
let connectionId = input.draftId;
let connectionId = input.draftId ?? input.reconnectId;
if (!connectionId) {
const response = await started;
if (!response.ok())
@@ -149,7 +150,19 @@ async function main(): Promise<NotionOAuthReceipt> {
const personal = grantState.grants.filter(
(g) => g.kind === "user" && g.subjectUserId === input.humanId,
);
if (
if (input.reconnectId) {
if (
draft.id !== input.reconnectId ||
draft.companyId !== input.companyId ||
draft.authKind !== "oauth" ||
draft.credentialPolicy !== "shared" ||
!before.some((c) => c.id === draft.id) ||
!grantState.grants.some((g) => g.kind === "organization")
)
throw new Error(
"Actual Notion reconnect changed the existing shared OAuth identity",
);
} else if (
draft.companyId !== input.companyId ||
draft.credentialPolicy !== "per_user" ||
draft.authKind !== "oauth" ||
@@ -166,9 +179,9 @@ async function main(): Promise<NotionOAuthReceipt> {
connectionId,
humanId: input.humanId,
intentId: input.intentId ?? null,
personalDraft: true,
personalDraft: !input.reconnectId,
providerHandoff: true,
credentialAbsentAtHandoff: true,
credentialAbsentAtHandoff: !input.reconnectId,
resumedSameDraft: !!input.draftId,
draftSurvivedReload: false,
consentCompleted: false,
@@ -300,7 +313,7 @@ async function main(): Promise<NotionOAuthReceipt> {
connected.companyId !== input.companyId ||
connected.status !== "active" ||
connected.healthStatus !== "ok" ||
connected.credentialPolicy !== "per_user"
connected.credentialPolicy !== (input.reconnectId ? "shared" : "per_user")
)
throw new Error(
"Real Notion callback did not activate the same personal draft",
@@ -25,7 +25,10 @@ import {
* credential injection into the instrumented Test browser. */
export async function privateNotionOAuth(
ctx: ChatCatalogContext,
options: Pick<NotionOAuthInput, "operation" | "intentId" | "draftId">,
options: Pick<
NotionOAuthInput,
"operation" | "intentId" | "draftId" | "reconnectId"
>,
) {
const binding = selectedExternalBinding(ctx),
p = binding.prerequisites;
@@ -51,7 +51,10 @@ export function personalNotionGuidance(text: string) {
),
);
}
async function sharedNotionAuthority(ctx: ChatCatalogContext, id: string) {
export async function sharedNotionAuthority(
ctx: ChatCatalogContext,
id: string,
) {
const connection = await ctx.input.api.get<ToolConnection>(
`/api/tool-connections/${id}`,
);
@@ -242,7 +245,7 @@ export function gradeNotionSetup(value: unknown) {
}
return c.result();
}
const sourceFiles = [
export const notionSetupSourceFiles = [
"chat-catalog-domains/external-notion-setup.ts",
"chat-catalog-domains/external-notion-oauth.ts",
"chat-catalog-domains/external-notion-oauth-worker.ts",
@@ -380,7 +383,7 @@ export const notionSetupExternalCases = [
expectedHumanMessageCount: 1,
turnTimeoutMs: 120000,
attemptTimeoutMs: 900000,
sourceFiles,
sourceFiles: notionSetupSourceFiles,
async act(ctx, state) {
const rows = await pollUntil({
label: "actual addressed pending Notion request",