mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-05 03:51:54 +01:00
test(evals): author natural Notion expiry cancellation and real reconnect
This commit is contained in:
1 parent
21a19b06f3
commit
8183b6d7a3
9 files changed
+1194
-16
No files matched your search
@@ -298,3 +298,26 @@ connection route prevents reusing the organization account; no shared audience,
|
||||
delegation, or install is broadened. Account refusal alone cannot satisfy this
|
||||
case: its actual personal setup, callback and source-bound launch read must all
|
||||
complete.
|
||||
|
||||
`notion-expired` additionally requires `expiredAuthorizationEnv`,
|
||||
`authorizationExpiresAt`, and `authorizationFailureKind=natural-expiry`. The
|
||||
selected real official MCP OAuth access credential must initially work for the
|
||||
designated workspace and expire in two to eight minutes. The historical fixture
|
||||
uses supported public encrypted-secret and typed connection APIs, with the fixed
|
||||
Notion/mcp-oauth descriptor and real credential reference. Actual health,
|
||||
catalog discovery, assignment and get-self must succeed before expiry. It does
|
||||
not invent an OAuth response, refresh token, healthy receipt or failed row.
|
||||
After the declared time, at most three fixed read-only MCP probes must observe
|
||||
actual HTTP401; failure to qualify this condition is a setup blocker. No token
|
||||
revocation, rotation, provider write or arbitrary metadata discovery is used.
|
||||
|
||||
The unchanged canonical request must then produce its own failed native read and
|
||||
exact existing-account reconnect handoff. A real addressed Not now decision and
|
||||
truthful cancellation continuation are saved before an explicit second unchanged
|
||||
human query. Only actual normal Notion OAuth reconnection of that same shared
|
||||
connection, preserved audience/agent authority, and a new source-bound native
|
||||
page read can pass. The private reconnect receipt explicitly records that it
|
||||
operated on an existing credential-bearing shared account, rather than claiming
|
||||
a new credential-free personal draft. The finite case declares four native runs,
|
||||
two actual human queries and a 25-minute attempt ceiling. Missing qualified
|
||||
credential/session/origin stays separate from authored callback completeness.
|
||||
@@ -23,7 +23,7 @@ export const externalRequirements: Record<string, { provider: ExternalProvider;
|
||||
"connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
|
||||
"connection-setup.notion-wrong-account": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv", "sharedWorkspaceId", "sharedApiTokenEnv", "sharedMcpTokenEnv"] },
|
||||
"connection-setup.notion-agent-unassigned": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
|
||||
"connection-setup.notion-expired": { provider: "notion", fields: ["workspaceId", "recentPageId", "expiredAuthorizationEnv", "oauthBrowserSession"] },
|
||||
"connection-setup.notion-expired": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv", "expiredAuthorizationEnv", "authorizationExpiresAt", "authorizationFailureKind"] },
|
||||
"connection-setup.return-from-login": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
|
||||
"connection-setup.save-and-resume": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
|
||||
"connection-setup.channel-prerequisites": { provider: "slack", fields: ["workspaceId", "channelId", "senderUserId", "botUserId", "appId", "registeredWebhookOrigin", "webhookRelayPort", "botTokenEnv", "senderTokenEnv", "signingSecretEnv", "configurationTokenEnv", "slackBrowserSessionEnv"] },
|
||||
|
||||
@@ -0,0 +1,350 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
gradeNotionExpired,
|
||||
hasNotionAuthFailure,
|
||||
notionRepairGuidance,
|
||||
notionExpiredExternalCases,
|
||||
} from "./external-notion-expired.js";
|
||||
import { qualifiedNotionExpiry } from "./external-notion-expiry.js";
|
||||
import {
|
||||
validNotionOAuthInput,
|
||||
validNotionOAuthReceipt,
|
||||
type NotionOAuthInput,
|
||||
type NotionOAuthReceipt,
|
||||
} from "./external-notion-oauth-contract.js";
|
||||
import type { ExternalObservation } from "./external-support.js";
|
||||
import type { ToolRunDecisionLookup } from "../../../packages/shared/src/types/tool-access.js";
|
||||
const pageId = "22222222-2222-4222-8222-222222222222";
|
||||
function valid(): ExternalObservation {
|
||||
return {
|
||||
scenarioId: "connection-setup.notion-expired",
|
||||
companyId: "company",
|
||||
leadId: "ceo",
|
||||
origin: "http://127.0.0.1:3100",
|
||||
reply: "As of Oct 26, 2026, the launch is now November 8.",
|
||||
state: {
|
||||
connectionId: "connection",
|
||||
humanId: "person",
|
||||
sourceIssueId: "source",
|
||||
validBeforeExpiry: true,
|
||||
sameWorkspaceBeforeExpiry: true,
|
||||
actualProvider401: true,
|
||||
expiryQualified: true,
|
||||
firstAuthFailure: true,
|
||||
firstReply:
|
||||
"Notion authorization expired. Please reconnect the existing account.",
|
||||
exactReconnectPath: true,
|
||||
firstIntentId: "first",
|
||||
finalIntentId: "second",
|
||||
firstInitialRunId: "initial",
|
||||
finalInitialRunId: "repeat",
|
||||
cancelledSnapshot: true,
|
||||
cancelConnectionUnchanged: true,
|
||||
cancelNoSuccessfulRead: true,
|
||||
truthfulCancelReply: true,
|
||||
cancelReply:
|
||||
"You declined, so I could not read Notion; the research is incomplete.",
|
||||
finalInitialReply:
|
||||
"Please reconnect the existing Notion account because its authorization expired.",
|
||||
finalAuthFailure: true,
|
||||
finalGuidanceCorrect: true,
|
||||
repairReceipt: {
|
||||
connectionId: "connection",
|
||||
humanId: "person",
|
||||
intentId: "second",
|
||||
personalDraft: false,
|
||||
credentialAbsentAtHandoff: false,
|
||||
providerHandoff: true,
|
||||
resumedSameDraft: false,
|
||||
draftSurvivedReload: false,
|
||||
consentCompleted: true,
|
||||
callbackReturned: true,
|
||||
workspaceMatched: true,
|
||||
providerHumanMatched: true,
|
||||
processIsolation: true,
|
||||
testRunnerAttached: false,
|
||||
tracingEnabled: false,
|
||||
videoEnabled: false,
|
||||
},
|
||||
authorityPreserved: true,
|
||||
repairRunIds: ["resumed"],
|
||||
readEntryIds: ["read"],
|
||||
decisions: [
|
||||
{
|
||||
runId: "resumed",
|
||||
decisions: [
|
||||
{
|
||||
invocation: {
|
||||
runId: "resumed",
|
||||
agentId: "ceo",
|
||||
issueId: "source",
|
||||
connectionId: "connection",
|
||||
catalogEntryId: "read",
|
||||
status: "succeeded",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
pages: [
|
||||
{
|
||||
id: pageId,
|
||||
editedAt: "2026-10-26T08:00:00.000Z",
|
||||
fact: {
|
||||
subjectAliases: ["launch"],
|
||||
valueAliases: ["November 8", "8 November"],
|
||||
},
|
||||
},
|
||||
],
|
||||
repairedReply: "As of Oct 26, 2026, the launch is now November 8.",
|
||||
pagesUnchanged: true,
|
||||
},
|
||||
phaseReplies: [],
|
||||
connections: [{ id: "connection" }],
|
||||
interactions: [
|
||||
{
|
||||
id: "first",
|
||||
kind: "connection_intent",
|
||||
addresseeUserId: "person",
|
||||
resolvedByUserId: "person",
|
||||
result: { outcome: "declined" },
|
||||
},
|
||||
{
|
||||
id: "second",
|
||||
kind: "connection_intent",
|
||||
addresseeUserId: "person",
|
||||
resolvedByUserId: "person",
|
||||
sourceRunId: "repeat",
|
||||
result: { outcome: "connected", connectionId: "connection" },
|
||||
},
|
||||
],
|
||||
profiles: [],
|
||||
secrets: [],
|
||||
endpoints: [],
|
||||
activity: [],
|
||||
links: [`https://www.notion.so/Launch-${pageId}`],
|
||||
runs: [],
|
||||
humanQueries: [
|
||||
"Read my Notion and tell me the latest on the launch.",
|
||||
"Read my Notion and tell me the latest on the launch.",
|
||||
],
|
||||
} as unknown as ExternalObservation;
|
||||
}
|
||||
describe("expired Notion cancellation and actual repair controls", () => {
|
||||
it("accepts independent natural expiry, retained cancellation and a fresh same-account read", () =>
|
||||
expect(gradeNotionExpired(valid()).passed).toBe(true));
|
||||
const wrong: Array<[string, (o: ExternalObservation) => void]> = [
|
||||
[
|
||||
"no real expiry",
|
||||
(o) => {
|
||||
o.state.actualProvider401 = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"credential was invalid from the start",
|
||||
(o) => {
|
||||
o.state.validBeforeExpiry = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"no original native failed read",
|
||||
(o) => {
|
||||
o.state.firstAuthFailure = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"false initial completion",
|
||||
(o) => {
|
||||
o.state.firstReply =
|
||||
"I've already reconnected Notion and read the launch.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"wrong repair route",
|
||||
(o) => {
|
||||
o.state.exactReconnectPath = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"cancel was used as consent",
|
||||
(o) => {
|
||||
o.state.cancelNoSuccessfulRead = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"cancellation fabricated",
|
||||
(o) => {
|
||||
o.state.cancelledSnapshot = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"unrelated second request",
|
||||
(o) => {
|
||||
o.humanQueries[1] = "Summarize a different workspace.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"same request resolved twice",
|
||||
(o) => {
|
||||
o.state.finalIntentId = "first";
|
||||
},
|
||||
],
|
||||
[
|
||||
"wrong account repaired",
|
||||
(o) => {
|
||||
(o.state.repairReceipt as NotionOAuthReceipt).connectionId = "other";
|
||||
},
|
||||
],
|
||||
[
|
||||
"provider unavailable without positive outcome",
|
||||
(o) => {
|
||||
o.state.repairReceipt = undefined;
|
||||
o.state.decisions = [];
|
||||
},
|
||||
],
|
||||
[
|
||||
"shared audience expanded",
|
||||
(o) => {
|
||||
o.state.authorityPreserved = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"no fresh native provider read",
|
||||
(o) => {
|
||||
o.state.decisions = [];
|
||||
},
|
||||
],
|
||||
[
|
||||
"cached facts claimed before repair",
|
||||
(o) => {
|
||||
o.state.firstReply =
|
||||
"Notion authorization expired; reconnect it. The launch is November 8.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"title-only final summary",
|
||||
(o) => {
|
||||
o.state.repairedReply = "The launch page title is promising.";
|
||||
},
|
||||
],
|
||||
];
|
||||
for (const [name, mutate] of wrong)
|
||||
it(`rejects ${name}`, () => {
|
||||
const o = valid();
|
||||
mutate(o);
|
||||
expect(gradeNotionExpired(o).passed).toBe(false);
|
||||
});
|
||||
it("qualifies only a finite explicit natural-expiry window", () => {
|
||||
const now = Date.parse("2026-10-27T10:00:00Z"),
|
||||
p = {
|
||||
authorizationExpiresAt: new Date(now + 180000).toISOString(),
|
||||
authorizationFailureKind: "natural-expiry",
|
||||
};
|
||||
expect(qualifiedNotionExpiry(p, now)).toBe(true);
|
||||
for (const change of [
|
||||
{ authorizationFailureKind: "revoked" },
|
||||
{ authorizationExpiresAt: new Date(now - 1).toISOString() },
|
||||
{ authorizationExpiresAt: new Date(now + 3600000).toISOString() },
|
||||
{ authorizationExpiresAt: "invalid" },
|
||||
])
|
||||
expect(qualifiedNotionExpiry({ ...p, ...change }, now)).toBe(false);
|
||||
});
|
||||
it("binds actual authentication failure to exact native run, source and account", () => {
|
||||
const rows = [
|
||||
{
|
||||
runId: "run",
|
||||
decisions: [
|
||||
{
|
||||
invocation: {
|
||||
runId: "run",
|
||||
issueId: "source",
|
||||
connectionId: "connection",
|
||||
status: "failed",
|
||||
errorCode: "upstream_error",
|
||||
errorMessage: "HTTP 401 Unauthorized",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
] as unknown as ToolRunDecisionLookup[];
|
||||
expect(hasNotionAuthFailure(rows, "run", "connection", "source")).toBe(
|
||||
true,
|
||||
);
|
||||
for (const args of [
|
||||
["other", "connection", "source"],
|
||||
["run", "other", "source"],
|
||||
["run", "connection", "other"],
|
||||
])
|
||||
expect(hasNotionAuthFailure(rows, args[0]!, args[1]!, args[2]!)).toBe(
|
||||
false,
|
||||
);
|
||||
const wrong = structuredClone(rows);
|
||||
wrong[0]!.decisions[0]!.invocation.status = "succeeded";
|
||||
expect(hasNotionAuthFailure(wrong, "run", "connection", "source")).toBe(
|
||||
false,
|
||||
);
|
||||
});
|
||||
it("accepts truthful repair direction but rejects affirmative completed claims", () => {
|
||||
for (const text of [
|
||||
"I have not reconnected Notion; please reconnect the expired account.",
|
||||
"Notion authorization expired. Use its reconnect path.",
|
||||
])
|
||||
expect(notionRepairGuidance(text)).toBe(true);
|
||||
for (const text of [
|
||||
"I've already repaired Notion; reconnect again if needed.",
|
||||
"I have already read Notion after reconnecting.",
|
||||
])
|
||||
expect(notionRepairGuidance(text)).toBe(false);
|
||||
});
|
||||
it("keeps public reconnect distinct from new personal setup in private receipts", () => {
|
||||
const storage = (domain: string) => ({
|
||||
cookies: [
|
||||
{
|
||||
name: "session",
|
||||
value: "synthetic-private-value",
|
||||
domain,
|
||||
path: "/",
|
||||
secure: true,
|
||||
},
|
||||
],
|
||||
origins: [],
|
||||
});
|
||||
const input: NotionOAuthInput = {
|
||||
operation: "authorize",
|
||||
publicOrigin: "https://owned.example.test",
|
||||
companyId: "company",
|
||||
prefix: "ACORN",
|
||||
humanId: "person",
|
||||
agentId: "ceo",
|
||||
intentId: "second",
|
||||
issueId: "source",
|
||||
reconnectId: "connection",
|
||||
workspaceId: "11111111-1111-4111-8111-111111111111",
|
||||
workspaceName: "Synthetic Launch",
|
||||
providerEmail: "synthetic@example.test",
|
||||
publicSession: storage("owned.example.test"),
|
||||
notionSession: storage(".notion.so"),
|
||||
};
|
||||
const receipt = valid().state.repairReceipt as NotionOAuthReceipt;
|
||||
expect(validNotionOAuthInput(input)).toBe(true);
|
||||
expect(validNotionOAuthReceipt(receipt, input)).toBe(true);
|
||||
expect(
|
||||
validNotionOAuthReceipt({ ...receipt, personalDraft: true }, input),
|
||||
).toBe(false);
|
||||
expect(
|
||||
validNotionOAuthReceipt({ ...receipt, connectionId: "other" }, input),
|
||||
).toBe(false);
|
||||
expect(validNotionOAuthInput({ ...input, draftId: "another" })).toBe(false);
|
||||
});
|
||||
it("provides all callbacks and explicitly bounded actual phases", () => {
|
||||
const c = notionExpiredExternalCases[0]!;
|
||||
expect(
|
||||
[c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every(
|
||||
(f) => typeof f === "function",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(c.expectedRunCount).toBe(4);
|
||||
expect(c.expectedHumanMessageCount).toBe(2);
|
||||
expect(c.coverageGaps).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,695 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js";
|
||||
import type {
|
||||
ToolConnection,
|
||||
ToolConnectionHealthCheckResult,
|
||||
ToolCatalogRefreshResult,
|
||||
ToolConnectionTestCallResult,
|
||||
ToolRunDecisionLookup,
|
||||
} from "../../../packages/shared/src/types/tool-access.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { sendChatMessage } from "../chat-flow.js";
|
||||
import {
|
||||
externalRequirements,
|
||||
selectedExternalBinding,
|
||||
selectedExternalCredential,
|
||||
} from "./external-fixtures.js";
|
||||
import {
|
||||
readNotionFixturePages,
|
||||
type NotionPageOracle,
|
||||
} from "./external-notion-provider.js";
|
||||
import {
|
||||
probeNotionAuthorization,
|
||||
qualifiedNotionExpiry,
|
||||
} from "./external-notion-expiry.js";
|
||||
import { privateNotionOAuth } from "./external-notion-oauth.js";
|
||||
import {
|
||||
notionSelfWorkspaceMatches,
|
||||
type NotionOAuthReceipt,
|
||||
} from "./external-notion-oauth-contract.js";
|
||||
import {
|
||||
notionSetupSourceFiles,
|
||||
sharedNotionAuthority,
|
||||
} from "./external-notion-setup.js";
|
||||
import {
|
||||
notionDateMentioned,
|
||||
notionFactSpan,
|
||||
notionPageIdentityMatches,
|
||||
} from "./external-notion-semantics.js";
|
||||
import {
|
||||
definition,
|
||||
checks,
|
||||
externalObservation,
|
||||
observeExternal,
|
||||
root,
|
||||
verifyRenderedReply,
|
||||
} from "./external-support.js";
|
||||
async function decisions(ctx: ChatCatalogContext) {
|
||||
return Promise.all(
|
||||
(await ctx.allRuns()).map((run) =>
|
||||
ctx.input.api.get<ToolRunDecisionLookup>(
|
||||
`${root(ctx)}/tools/runs/${run.id}/decisions`,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
export function hasNotionAuthFailure(
|
||||
rows: ToolRunDecisionLookup[],
|
||||
runId: string,
|
||||
connectionId: string,
|
||||
issueId: string,
|
||||
) {
|
||||
return rows.some(
|
||||
(row) =>
|
||||
row.runId === runId &&
|
||||
row.decisions.some(
|
||||
(d) =>
|
||||
d.invocation.runId === runId &&
|
||||
d.invocation.issueId === issueId &&
|
||||
d.invocation.connectionId === connectionId &&
|
||||
d.invocation.status === "failed" &&
|
||||
/401|unauthori[sz]ed|authentication|reauthorization|oauth.*(?:expired|required)|invalid.grant/i.test(
|
||||
`${d.invocation.errorCode ?? ""} ${d.invocation.errorMessage ?? ""}`,
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
export function notionRepairGuidance(reply: string) {
|
||||
const falseCompletion = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => {
|
||||
const claim =
|
||||
/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:reconnected|repaired|read|reviewed|summarized)\b|\b(?:Notion|(?:this|the) account)\s+(?:is|has been)\s+(?:now |already )?(?:reconnected|repaired|connected)\b/i.exec(
|
||||
clause,
|
||||
);
|
||||
if (!claim) return false;
|
||||
return !/\b(?:once|after|when|if|until|before)\b[^,]{0,80}$/i.test(
|
||||
clause.slice(0, claim.index),
|
||||
);
|
||||
});
|
||||
return (
|
||||
/notion/i.test(reply) &&
|
||||
/reconnect|reauthori[sz]|sign.?in.*again|authorization.*expir/i.test(
|
||||
reply,
|
||||
) &&
|
||||
!falseCompletion
|
||||
);
|
||||
}
|
||||
async function pending(ctx: ChatCatalogContext, excluded: string[] = []) {
|
||||
const rows = await pollUntil({
|
||||
label: "new original-account Notion repair request",
|
||||
deadlineAt: Date.now() + 150000,
|
||||
load: async () => {
|
||||
await ctx.refreshIssue();
|
||||
return ctx.issue()
|
||||
? ctx.input.api.get<ConnectionIntentInteraction[]>(
|
||||
`/api/issues/${ctx.issue()!.id}/interactions`,
|
||||
)
|
||||
: [];
|
||||
},
|
||||
accept: (rows) =>
|
||||
rows.some(
|
||||
(r) =>
|
||||
r.kind === "connection_intent" &&
|
||||
r.payload.serviceSlug === "notion" &&
|
||||
r.status === "pending" &&
|
||||
!excluded.includes(r.id),
|
||||
),
|
||||
});
|
||||
const selected = rows.filter(
|
||||
(r) =>
|
||||
r.kind === "connection_intent" &&
|
||||
r.payload.serviceSlug === "notion" &&
|
||||
r.status === "pending" &&
|
||||
!excluded.includes(r.id),
|
||||
);
|
||||
if (selected.length !== 1)
|
||||
throw new Error("Repair requires exactly one new actual Notion request");
|
||||
if (typeof selected[0]!.sourceRunId !== "string")
|
||||
throw new Error(
|
||||
"Notion repair request lacks actual source native run attribution",
|
||||
);
|
||||
return { ...selected[0]!, sourceRunId: selected[0]!.sourceRunId };
|
||||
}
|
||||
async function waitForPhase(ctx: ChatCatalogContext, runId: string) {
|
||||
await pollUntil({
|
||||
label: "actual source-specific Notion request phase",
|
||||
deadlineAt: Date.now() + 150000,
|
||||
load: async () => {
|
||||
await ctx.refreshIssue();
|
||||
return ctx.allRuns();
|
||||
},
|
||||
accept: (rows) =>
|
||||
rows.some((r) => r.id === runId && r.status === "succeeded") &&
|
||||
ctx.issue()?.conversationState === "waiting",
|
||||
reject: (rows) =>
|
||||
rows.some(
|
||||
(r) =>
|
||||
r.id === runId &&
|
||||
["failed", "cancelled", "timed_out"].includes(r.status),
|
||||
)
|
||||
? "Actual Notion source phase failed"
|
||||
: undefined,
|
||||
});
|
||||
}
|
||||
export function gradeNotionExpired(value: unknown) {
|
||||
const c = checks(),
|
||||
o = externalObservation(value);
|
||||
c.add(
|
||||
"actual-expired-notion-phases",
|
||||
!!o,
|
||||
"Actual expired provider authorization, cancellation and later successful same-account repair are required.",
|
||||
);
|
||||
if (!o) return c.result();
|
||||
const first = o.interactions.find(
|
||||
(r) => r.id === o.state.firstIntentId,
|
||||
) as unknown as ConnectionIntentInteraction | undefined;
|
||||
const final = o.interactions.find(
|
||||
(r) => r.id === o.state.finalIntentId,
|
||||
) as unknown as ConnectionIntentInteraction | undefined;
|
||||
const receipt = o.state.repairReceipt as NotionOAuthReceipt | undefined;
|
||||
c.add(
|
||||
"genuine-natural-expiry-context",
|
||||
o.state.validBeforeExpiry === true &&
|
||||
o.state.sameWorkspaceBeforeExpiry === true &&
|
||||
o.state.actualProvider401 === true &&
|
||||
o.state.expiryQualified === true,
|
||||
"A real currently valid official OAuth credential first works, then actually returns provider HTTP401 after qualified natural expiry; no injected failure or token mutation.",
|
||||
);
|
||||
c.add(
|
||||
"original-native-authentication-error",
|
||||
o.state.firstAuthFailure === true &&
|
||||
notionRepairGuidance(String(o.state.firstReply ?? "")) &&
|
||||
o.state.exactReconnectPath === true,
|
||||
"The original canonical native read fails against the existing connection and truthful chat identifies its actual supported reconnect route before any human action.",
|
||||
);
|
||||
c.add(
|
||||
"actual-cancel-preserves-incomplete-research",
|
||||
!!first &&
|
||||
first.result?.outcome === "declined" &&
|
||||
first.addresseeUserId === o.state.humanId &&
|
||||
first.resolvedByUserId === o.state.humanId &&
|
||||
!first.resolvedByAgentId &&
|
||||
o.state.cancelledSnapshot === true &&
|
||||
o.state.cancelConnectionUnchanged === true &&
|
||||
o.state.cancelNoSuccessfulRead === true &&
|
||||
o.state.truthfulCancelReply === true,
|
||||
"The addressed person's actual Not now decision is retained; its own continuation reports no research, without repaired credentials, successful reads or a repeated request before a new human request.",
|
||||
);
|
||||
c.add(
|
||||
"explicit-repeat-request-lineage",
|
||||
!!final &&
|
||||
final.id !== first?.id &&
|
||||
o.humanQueries.length === 2 &&
|
||||
o.humanQueries.every((q) => q === o.humanQueries[0]) &&
|
||||
o.state.finalAuthFailure === true &&
|
||||
o.state.finalGuidanceCorrect === true &&
|
||||
final.addresseeUserId === o.state.humanId &&
|
||||
final.sourceRunId === o.state.finalInitialRunId,
|
||||
"A second unchanged human request creates its own distinct actual failed-read/repair phase; cancellation never acts as consent.",
|
||||
);
|
||||
c.add(
|
||||
"actual-same-account-human-reconnect",
|
||||
!!receipt &&
|
||||
receipt.connectionId === o.state.connectionId &&
|
||||
receipt.intentId === final?.id &&
|
||||
receipt.humanId === o.state.humanId &&
|
||||
receipt.personalDraft === false &&
|
||||
receipt.credentialAbsentAtHandoff === false &&
|
||||
receipt.providerHandoff === true &&
|
||||
receipt.providerHumanMatched === true &&
|
||||
receipt.callbackReturned === true &&
|
||||
receipt.workspaceMatched === true &&
|
||||
receipt.consentCompleted === true &&
|
||||
receipt.processIsolation === true &&
|
||||
receipt.testRunnerAttached === false &&
|
||||
receipt.tracingEnabled === false &&
|
||||
receipt.videoEnabled === false &&
|
||||
final?.result?.outcome === "connected" &&
|
||||
final.result.connectionId === o.state.connectionId &&
|
||||
final.resolvedByUserId === o.state.humanId &&
|
||||
o.state.authorityPreserved === true &&
|
||||
o.connections.length === 1,
|
||||
"Actual normal Notion OAuth reconnect repairs the same existing connection and original scope/person/request, with no duplicate account or API credential rescue.",
|
||||
);
|
||||
const rows = o.state.decisions as ToolRunDecisionLookup[] | undefined;
|
||||
c.add(
|
||||
"fresh-post-repair-native-read",
|
||||
!!rows?.some(
|
||||
(row) =>
|
||||
row.runId !== o.state.firstInitialRunId &&
|
||||
row.runId !== o.state.finalInitialRunId &&
|
||||
(o.state.repairRunIds as string[] | undefined)?.includes(row.runId) &&
|
||||
row.decisions.some(
|
||||
(d) =>
|
||||
d.invocation.runId === row.runId &&
|
||||
d.invocation.agentId === o.leadId &&
|
||||
d.invocation.issueId === o.state.sourceIssueId &&
|
||||
d.invocation.connectionId === o.state.connectionId &&
|
||||
d.invocation.status === "succeeded" &&
|
||||
(o.state.readEntryIds as string[] | undefined)?.includes(
|
||||
d.invocation.catalogEntryId ?? "",
|
||||
),
|
||||
),
|
||||
),
|
||||
"A new success continuation actually reads the real provider after human repair; independent setup probes cannot substitute.",
|
||||
);
|
||||
const recent = (o.state.pages as NotionPageOracle[] | undefined)?.[0],
|
||||
reply = String(o.state.repairedReply ?? "");
|
||||
c.add(
|
||||
"no-pre-repair-cached-summary",
|
||||
!!recent &&
|
||||
[
|
||||
o.state.firstReply,
|
||||
o.state.finalInitialReply,
|
||||
o.state.cancelReply,
|
||||
].every(
|
||||
(reply) =>
|
||||
typeof reply === "string" && !notionFactSpan(reply, recent.fact),
|
||||
),
|
||||
"Initial failure, cancellation and explicit re-request cannot report cached launch facts as fresh provider research before repair.",
|
||||
);
|
||||
c.add(
|
||||
"fresh-grounded-summary-after-repair",
|
||||
!!recent &&
|
||||
!!notionFactSpan(reply, recent.fact) &&
|
||||
notionDateMentioned(reply, recent.editedAt) &&
|
||||
o.links.some((link) => notionPageIdentityMatches(link, recent.id)) &&
|
||||
o.state.pagesUnchanged === true,
|
||||
"Only the post-repair native reply summarizes current source body/date/link, never cached material as fresh research.",
|
||||
);
|
||||
return c.result();
|
||||
}
|
||||
export const notionExpiredExternalCases = [
|
||||
definition(
|
||||
"connection-setup.notion-expired",
|
||||
async (ctx) => {
|
||||
const binding = selectedExternalBinding(ctx),
|
||||
p = binding.prerequisites;
|
||||
if (!qualifiedNotionExpiry(p, Date.now()))
|
||||
throw new Error(
|
||||
"Setup blocker: designated natural-expiry Notion OAuth access token must expire in two to eight minutes",
|
||||
);
|
||||
if (
|
||||
(
|
||||
await ctx.input.api.get<{ connections: ToolConnection[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
)
|
||||
).connections.length
|
||||
)
|
||||
throw new Error(
|
||||
"Expired Notion fixture requires a fresh disposable company",
|
||||
);
|
||||
const pages = await readNotionFixturePages(ctx),
|
||||
old = await probeNotionAuthorization(ctx, binding);
|
||||
if (!old.validWorkspace || old.authenticationDenied)
|
||||
throw new Error(
|
||||
"Setup blocker: actual old Notion OAuth authorization is not initially valid for the selected workspace",
|
||||
);
|
||||
const credential = selectedExternalCredential(
|
||||
ctx,
|
||||
binding,
|
||||
"expiredAuthorizationEnv",
|
||||
);
|
||||
const secret = await ctx.input.api.postSensitive<{ id: string }>(
|
||||
`${root(ctx)}/secrets`,
|
||||
{
|
||||
name: "Historical Notion OAuth authorization",
|
||||
key: "chat-eval-notion-old-authorization",
|
||||
provider: "local_encrypted",
|
||||
value: credential,
|
||||
},
|
||||
);
|
||||
let connection: ToolConnection | undefined;
|
||||
try {
|
||||
connection = await ctx.input.api.post<ToolConnection>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
{
|
||||
name: "Launch Notion",
|
||||
applicationName: "Notion",
|
||||
transport: "mcp_remote",
|
||||
authKind: "oauth",
|
||||
credentialPolicy: "shared",
|
||||
ownership: "customer",
|
||||
config: {
|
||||
url: p.mcpUrl,
|
||||
sourceTemplateKey: "notion",
|
||||
connectionMethodKey: "mcp-oauth",
|
||||
},
|
||||
transportConfig: {
|
||||
url: p.mcpUrl,
|
||||
sourceTemplateKey: "notion",
|
||||
connectionMethodKey: "mcp-oauth",
|
||||
},
|
||||
credentialSecretRefs: [
|
||||
{
|
||||
configPath: "oauth.access_token",
|
||||
secretId: secret.id,
|
||||
versionSelector: "latest",
|
||||
required: true,
|
||||
},
|
||||
],
|
||||
credentialRefs: [
|
||||
{
|
||||
name: "oauth.access_token",
|
||||
secretId: secret.id,
|
||||
version: "latest",
|
||||
placement: "header",
|
||||
key: "Authorization",
|
||||
prefix: "Bearer ",
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
const health =
|
||||
await ctx.input.api.post<ToolConnectionHealthCheckResult>(
|
||||
`/api/tool-connections/${connection.id}/health-check`,
|
||||
{},
|
||||
);
|
||||
if (health.connection.healthStatus !== "ok")
|
||||
throw new Error(
|
||||
"Setup blocker: historical named Notion credential failed actual health qualification",
|
||||
);
|
||||
const refresh = await ctx.input.api.post<ToolCatalogRefreshResult>(
|
||||
`/api/tool-connections/${connection.id}/catalog/refresh`,
|
||||
{},
|
||||
),
|
||||
read = refresh.catalog
|
||||
.filter((e) => e.isReadOnly && e.status === "active")
|
||||
.map((e) => e.id);
|
||||
if (!read.length)
|
||||
throw new Error(
|
||||
"Setup blocker: actual Notion read catalog is unavailable before natural expiry",
|
||||
);
|
||||
await ctx.input.api.post(
|
||||
`${root(ctx)}/tools/apps/${connection.id}/finish`,
|
||||
{
|
||||
enabledCatalogEntryIds: read,
|
||||
askFirstCatalogEntryIds: [],
|
||||
access: { agentIds: [ctx.input.fixtures.agent.id] },
|
||||
},
|
||||
);
|
||||
const proof =
|
||||
await ctx.input.api.postSensitive<ToolConnectionTestCallResult>(
|
||||
`/api/tool-connections/${connection.id}/test-calls`,
|
||||
{
|
||||
agentId: ctx.input.fixtures.agent.id,
|
||||
toolName: "notion-get-self",
|
||||
parameters: {},
|
||||
},
|
||||
);
|
||||
if (
|
||||
proof.decision !== "allowed" ||
|
||||
proof.error ||
|
||||
!notionSelfWorkspaceMatches(proof.result, p.workspaceId!)
|
||||
)
|
||||
throw new Error(
|
||||
"Setup blocker: actual stored existing Notion OAuth authorization did not prove intended workspace before expiry",
|
||||
);
|
||||
const authorityBefore = await sharedNotionAuthority(ctx, connection.id),
|
||||
expires = Date.parse(p.authorizationExpiresAt!);
|
||||
while (Date.now() < expires)
|
||||
await new Promise((r) =>
|
||||
setTimeout(r, Math.min(10000, expires - Date.now())),
|
||||
);
|
||||
let actualProvider401 = false;
|
||||
for (let n = 0; n < 3; n++) {
|
||||
const probe = await probeNotionAuthorization(ctx, binding);
|
||||
if (probe.authenticationDenied) {
|
||||
actualProvider401 = true;
|
||||
break;
|
||||
}
|
||||
if (n < 2) await new Promise((r) => setTimeout(r, 5000));
|
||||
}
|
||||
if (!actualProvider401)
|
||||
throw new Error(
|
||||
"Setup blocker: designated Notion authorization did not actually fail with provider HTTP401 at its qualified natural expiry",
|
||||
);
|
||||
return {
|
||||
connectionId: connection.id,
|
||||
secretId: secret.id,
|
||||
pages,
|
||||
readEntryIds: read,
|
||||
authorityBefore,
|
||||
validBeforeExpiry: true,
|
||||
sameWorkspaceBeforeExpiry: true,
|
||||
expiryQualified: true,
|
||||
actualProvider401,
|
||||
humanId: (
|
||||
await ctx.input.api.get<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
)
|
||||
).user.id,
|
||||
};
|
||||
} catch (error) {
|
||||
if (connection)
|
||||
await ctx.input.api.delete(`/api/tool-connections/${connection.id}`, {
|
||||
allowNotFound: true,
|
||||
});
|
||||
await ctx.input.api.delete(`${root(ctx)}/secrets/${secret.id}`, {
|
||||
allowNotFound: true,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
gradeNotionExpired,
|
||||
{
|
||||
authentication: "human-session",
|
||||
externalInput: externalRequirements["connection-setup.notion-expired"],
|
||||
expectedRunCount: 4,
|
||||
minimumRunCount: 4,
|
||||
expectedHumanMessageCount: 2,
|
||||
turnTimeoutMs: 120000,
|
||||
attemptTimeoutMs: 1500000,
|
||||
sourceFiles: [
|
||||
...notionSetupSourceFiles,
|
||||
"chat-catalog-domains/external-notion-expired.ts",
|
||||
"chat-catalog-domains/external-notion-expiry.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"chat-flow.ts",
|
||||
"../../server/src/services/tool-access.ts",
|
||||
],
|
||||
async act(ctx, state) {
|
||||
const first = await pending(ctx);
|
||||
state.firstIntentId = first.id;
|
||||
state.firstInitialRunId = first.sourceRunId;
|
||||
state.sourceIssueId = ctx.issue()!.id;
|
||||
await waitForPhase(ctx, first.sourceRunId);
|
||||
const before = await observeExternal(ctx, state);
|
||||
state.firstReply = before.phaseReplies
|
||||
.filter((r) => r.runId === first.sourceRunId)
|
||||
.map((r) => r.body)
|
||||
.join("\n");
|
||||
state.firstAuthFailure = hasNotionAuthFailure(
|
||||
await decisions(ctx),
|
||||
first.sourceRunId,
|
||||
String(state.connectionId),
|
||||
ctx.issue()!.id,
|
||||
);
|
||||
const path = `/${ctx.input.fixtures.company.issuePrefix}/apps/connect`,
|
||||
correct = (href: string) => {
|
||||
try {
|
||||
const u = new URL(href, ctx.input.api.baseURL);
|
||||
return (
|
||||
u.origin === ctx.input.api.baseURL &&
|
||||
u.pathname === path &&
|
||||
u.searchParams.get("source") === "notion" &&
|
||||
u.searchParams.get("reconnect") === state.connectionId
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
state.exactReconnectPath = before.links.some(correct);
|
||||
if (
|
||||
first.addresseeUserId !== state.humanId ||
|
||||
!state.firstAuthFailure ||
|
||||
!state.exactReconnectPath ||
|
||||
!notionRepairGuidance(String(state.firstReply))
|
||||
)
|
||||
throw new Error(
|
||||
"Actual initial Notion error/guidance does not identify the real existing account's reconnect flow",
|
||||
);
|
||||
await expect(
|
||||
ctx.input.page.getByRole("button", { name: "Not now", exact: true }),
|
||||
).toBeVisible();
|
||||
await ctx.input.page
|
||||
.getByRole("button", { name: "Not now", exact: true })
|
||||
.click();
|
||||
const cancellation = await pollUntil({
|
||||
label: "truthful actual cancellation continuation",
|
||||
deadlineAt: Date.now() + 150000,
|
||||
load: () => observeExternal(ctx, state),
|
||||
accept: (o) =>
|
||||
o.phaseReplies.some(
|
||||
(r) =>
|
||||
r.runId !== first.sourceRunId &&
|
||||
o.runs.some(
|
||||
(run) => run.id === r.runId && run.status === "succeeded",
|
||||
),
|
||||
),
|
||||
});
|
||||
const cancelRows = await decisions(ctx),
|
||||
cancelReply = cancellation.phaseReplies
|
||||
.filter((r) => r.runId !== first.sourceRunId)
|
||||
.map((r) => r.body)
|
||||
.join("\n"),
|
||||
recent = (state.pages as NotionPageOracle[])[0]!;
|
||||
state.cancelledSnapshot =
|
||||
cancellation.interactions.filter(
|
||||
(r) => r.kind === "connection_intent",
|
||||
).length === 1 &&
|
||||
cancellation.interactions.some(
|
||||
(r) =>
|
||||
r.id === first.id &&
|
||||
(r as unknown as ConnectionIntentInteraction).result?.outcome ===
|
||||
"declined",
|
||||
);
|
||||
state.cancelConnectionUnchanged = isDeepStrictEqual(
|
||||
await sharedNotionAuthority(ctx, String(state.connectionId)),
|
||||
state.authorityBefore,
|
||||
);
|
||||
state.cancelNoSuccessfulRead = !cancelRows.some((r) =>
|
||||
r.decisions.some(
|
||||
(d) =>
|
||||
d.invocation.connectionId === state.connectionId &&
|
||||
d.invocation.status === "succeeded",
|
||||
),
|
||||
);
|
||||
state.cancelReply = cancelReply;
|
||||
state.truthfulCancelReply =
|
||||
/not|couldn.t|cannot|can.t|declin|cancel|incomplete|unable/i.test(
|
||||
cancelReply,
|
||||
) &&
|
||||
/read|summary|research|notion/i.test(cancelReply) &&
|
||||
!notionFactSpan(cancelReply, recent.fact) &&
|
||||
!/\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:read|reviewed|summarized|reconnected)\b/i.test(
|
||||
cancelReply,
|
||||
);
|
||||
await ctx.input.evidence("chat-catalog-notion-cancel-phase.json", {
|
||||
intentId: first.id,
|
||||
sourceIssueId: state.sourceIssueId,
|
||||
firstRunId: first.sourceRunId,
|
||||
cancelRunIds: cancellation.runs
|
||||
.filter((r) => r.id !== first.sourceRunId)
|
||||
.map((r) => r.id),
|
||||
cancelled: state.cancelledSnapshot,
|
||||
noSuccessfulRead: state.cancelNoSuccessfulRead,
|
||||
authorityUnchanged: state.cancelConnectionUnchanged,
|
||||
truthfulReply: state.truthfulCancelReply,
|
||||
});
|
||||
if (
|
||||
!state.cancelledSnapshot ||
|
||||
!state.cancelNoSuccessfulRead ||
|
||||
!state.cancelConnectionUnchanged ||
|
||||
!state.truthfulCancelReply
|
||||
)
|
||||
throw new Error(
|
||||
"Actual cancellation phase did not retain truthful incomplete research before re-request",
|
||||
);
|
||||
const beforeRepairRunIds = (await ctx.allRuns()).map((r) => r.id);
|
||||
await sendChatMessage(ctx.input.page, ctx.scenario.query);
|
||||
const final = await pending(ctx, [first.id]);
|
||||
state.finalIntentId = final.id;
|
||||
state.finalInitialRunId = final.sourceRunId;
|
||||
await waitForPhase(ctx, final.sourceRunId);
|
||||
const again = await observeExternal(ctx, state);
|
||||
const reply = again.phaseReplies
|
||||
.filter((r) => r.runId === final.sourceRunId)
|
||||
.map((r) => r.body)
|
||||
.join("\n");
|
||||
state.finalInitialReply = reply;
|
||||
state.finalAuthFailure = hasNotionAuthFailure(
|
||||
await decisions(ctx),
|
||||
final.sourceRunId,
|
||||
String(state.connectionId),
|
||||
String(state.sourceIssueId),
|
||||
);
|
||||
state.finalGuidanceCorrect =
|
||||
notionRepairGuidance(reply) && again.links.some(correct);
|
||||
if (
|
||||
final.addresseeUserId !== state.humanId ||
|
||||
!state.finalAuthFailure ||
|
||||
!state.finalGuidanceCorrect
|
||||
)
|
||||
throw new Error(
|
||||
"Actual repeated request does not truthfully direct repair of the same failed Notion account",
|
||||
);
|
||||
state.repairReceipt = await privateNotionOAuth(ctx, {
|
||||
operation: "authorize",
|
||||
intentId: final.id,
|
||||
reconnectId: String(state.connectionId),
|
||||
});
|
||||
state.beforeRepairRunIds = [...beforeRepairRunIds, final.sourceRunId];
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const o = await observeExternal(ctx, state);
|
||||
state.decisions = await decisions(ctx);
|
||||
state.authorityPreserved = isDeepStrictEqual(
|
||||
await sharedNotionAuthority(ctx, String(state.connectionId)),
|
||||
state.authorityBefore,
|
||||
);
|
||||
state.repairRunIds = o.runs
|
||||
.filter(
|
||||
(r) =>
|
||||
!((state.beforeRepairRunIds as string[]) ?? []).includes(r.id),
|
||||
)
|
||||
.map((r) => r.id);
|
||||
state.repairedReply = o.phaseReplies
|
||||
.filter((r) => (state.repairRunIds as string[]).includes(r.runId))
|
||||
.map((r) => r.body)
|
||||
.join("\n");
|
||||
const pages = await readNotionFixturePages(ctx),
|
||||
before = state.pages as NotionPageOracle[];
|
||||
state.pagesUnchanged =
|
||||
pages.length === before.length &&
|
||||
pages.every(
|
||||
(p, n) =>
|
||||
p.id === before[n]?.id &&
|
||||
p.editedAt === before[n]?.editedAt &&
|
||||
p.marker === before[n]?.marker,
|
||||
);
|
||||
return { ...o, state };
|
||||
},
|
||||
async verifyResult(ctx, state, value) {
|
||||
await verifyRenderedReply(ctx, value, String(state.finalInitialRunId));
|
||||
const o = externalObservation(value)!,
|
||||
recent = (state.pages as NotionPageOracle[])[0]!,
|
||||
href = o.links.find((link) =>
|
||||
notionPageIdentityMatches(link, recent.id),
|
||||
);
|
||||
if (!href)
|
||||
throw new Error(
|
||||
"Current Notion source is missing after real reconnect",
|
||||
);
|
||||
await expect(
|
||||
ctx.input.page
|
||||
.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`)
|
||||
.first(),
|
||||
).toBeVisible();
|
||||
await ctx.input.evidence("chat-catalog-notion-repair.json", {
|
||||
connectionId: state.connectionId,
|
||||
firstIntentId: state.firstIntentId,
|
||||
finalIntentId: state.finalIntentId,
|
||||
initialRunId: state.firstInitialRunId,
|
||||
repeatedRunId: state.finalInitialRunId,
|
||||
repairRunIds: state.repairRunIds,
|
||||
receipt: state.repairReceipt,
|
||||
authorityPreserved: state.authorityPreserved,
|
||||
});
|
||||
},
|
||||
async cleanup(ctx, state) {
|
||||
if (typeof state.connectionId === "string")
|
||||
await ctx.input.api.delete(
|
||||
`/api/tool-connections/${state.connectionId}`,
|
||||
{ allowNotFound: true },
|
||||
);
|
||||
if (typeof state.secretId === "string")
|
||||
await ctx.input.api.delete(`${root(ctx)}/secrets/${state.secretId}`, {
|
||||
allowNotFound: true,
|
||||
});
|
||||
},
|
||||
},
|
||||
),
|
||||
];
|
||||
@@ -0,0 +1,83 @@
|
||||
import { Client } from "../../../server/node_modules/@modelcontextprotocol/sdk/dist/esm/client/index.js";
|
||||
import { StreamableHTTPClientTransport } from "../../../server/node_modules/@modelcontextprotocol/sdk/dist/esm/client/streamableHttp.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import type { ProviderBinding } from "./external-fixtures.js";
|
||||
import { selectedExternalCredential } from "./external-fixtures.js";
|
||||
import { assertNotionConnection } from "./external-notion-provider.js";
|
||||
import { notionSelfWorkspaceMatches } from "./external-notion-oauth-contract.js";
|
||||
/** Exact selected official read-only MCP probe. Never stores raw account/tool
|
||||
* output, and never supplies a refresh client or performs provider discovery. */
|
||||
export async function probeNotionAuthorization(
|
||||
ctx: ChatCatalogContext,
|
||||
binding: ProviderBinding,
|
||||
) {
|
||||
assertNotionConnection(binding);
|
||||
const token = selectedExternalCredential(
|
||||
ctx,
|
||||
binding,
|
||||
"expiredAuthorizationEnv",
|
||||
);
|
||||
let denied = false,
|
||||
calls = 0;
|
||||
const client = new Client({
|
||||
name: "paperclip-notion-expiry-control",
|
||||
version: "1.0.0",
|
||||
});
|
||||
client.onerror = () => undefined;
|
||||
const transport = new StreamableHTTPClientTransport(
|
||||
new URL(binding.prerequisites.mcpUrl!),
|
||||
{
|
||||
requestInit: { headers: { Authorization: `Bearer ${token}` } },
|
||||
fetch: async (url, init) => {
|
||||
const target = new URL(String(url));
|
||||
if (target.href !== "https://mcp.notion.com/mcp" || ++calls > 5)
|
||||
throw new Error(
|
||||
"Notion expiry probe exceeded fixed resource/request bounds",
|
||||
);
|
||||
const response = await fetch(target, {
|
||||
...init,
|
||||
redirect: "error",
|
||||
signal: AbortSignal.timeout(15000),
|
||||
});
|
||||
if (response.status === 401) denied = true;
|
||||
return response;
|
||||
},
|
||||
reconnectionOptions: {
|
||||
maxRetries: 0,
|
||||
initialReconnectionDelay: 100,
|
||||
maxReconnectionDelay: 100,
|
||||
reconnectionDelayGrowFactor: 1,
|
||||
},
|
||||
},
|
||||
);
|
||||
try {
|
||||
await client.connect(transport);
|
||||
const result = await client.callTool(
|
||||
{ name: "notion-get-self", arguments: {} },
|
||||
undefined,
|
||||
{ timeout: 15000 },
|
||||
);
|
||||
return {
|
||||
validWorkspace:
|
||||
result.isError !== true &&
|
||||
notionSelfWorkspaceMatches(result, binding.prerequisites.workspaceId!),
|
||||
authenticationDenied: false,
|
||||
};
|
||||
} catch {
|
||||
return { validWorkspace: false, authenticationDenied: denied };
|
||||
} finally {
|
||||
await client.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
export function qualifiedNotionExpiry(
|
||||
prerequisites: Record<string, string>,
|
||||
now: number,
|
||||
) {
|
||||
const expiresAt = Date.parse(prerequisites.authorizationExpiresAt ?? "");
|
||||
return (
|
||||
prerequisites.authorizationFailureKind === "natural-expiry" &&
|
||||
Number.isFinite(expiresAt) &&
|
||||
expiresAt >= now + 120000 &&
|
||||
expiresAt <= now + 480000
|
||||
);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ export interface NotionOAuthInput {
|
||||
intentId?: string;
|
||||
issueId?: string;
|
||||
draftId?: string;
|
||||
reconnectId?: string;
|
||||
newPersonal?: true;
|
||||
workspaceId: string;
|
||||
workspaceName: string;
|
||||
@@ -20,9 +21,9 @@ export interface NotionOAuthReceipt {
|
||||
connectionId: string;
|
||||
humanId: string;
|
||||
intentId: string | null;
|
||||
personalDraft: true;
|
||||
personalDraft: boolean;
|
||||
providerHandoff: true;
|
||||
credentialAbsentAtHandoff: true;
|
||||
credentialAbsentAtHandoff: boolean;
|
||||
resumedSameDraft: boolean;
|
||||
draftSurvivedReload: boolean;
|
||||
consentCompleted: boolean;
|
||||
@@ -99,6 +100,12 @@ export function validNotionOAuthInput(input: NotionOAuthInput) {
|
||||
!!input.intentId === !!input.issueId &&
|
||||
(!input.issueId || id(input.issueId)) &&
|
||||
(!input.draftId || id(input.draftId)) &&
|
||||
(!input.reconnectId ||
|
||||
(id(input.reconnectId) &&
|
||||
!input.draftId &&
|
||||
!input.newPersonal &&
|
||||
!!input.intentId &&
|
||||
input.operation === "authorize")) &&
|
||||
(!input.newPersonal ||
|
||||
(input.newPersonal === true &&
|
||||
!input.draftId &&
|
||||
@@ -172,11 +179,12 @@ export function validNotionOAuthReceipt(
|
||||
Object.keys(value).sort().join(",") === keys &&
|
||||
id(value.connectionId) &&
|
||||
(!input.draftId || input.draftId === value.connectionId) &&
|
||||
(!input.reconnectId || input.reconnectId === value.connectionId) &&
|
||||
value.humanId === input.humanId &&
|
||||
value.intentId === (input.intentId ?? null) &&
|
||||
value.personalDraft === true &&
|
||||
value.personalDraft === !input.reconnectId &&
|
||||
value.providerHandoff === true &&
|
||||
value.credentialAbsentAtHandoff === true &&
|
||||
value.credentialAbsentAtHandoff === !input.reconnectId &&
|
||||
value.resumedSameDraft === !!input.draftId &&
|
||||
value.processIsolation === true &&
|
||||
value.testRunnerAttached === false &&
|
||||
|
||||
@@ -93,6 +93,7 @@ async function main(): Promise<NotionOAuthReceipt> {
|
||||
const params = new URLSearchParams({ source: "notion", stage: "setup" });
|
||||
if (input.intentId) params.set("intent", input.intentId);
|
||||
if (input.draftId) params.set("resume", input.draftId);
|
||||
if (input.reconnectId) params.set("reconnect", input.reconnectId);
|
||||
if (input.newPersonal) params.set("new", "1");
|
||||
const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`;
|
||||
await page.goto(setupURL, { waitUntil: "domcontentloaded" });
|
||||
@@ -104,7 +105,7 @@ async function main(): Promise<NotionOAuthReceipt> {
|
||||
exact: true,
|
||||
})
|
||||
.waitFor();
|
||||
if (!input.draftId) {
|
||||
if (!input.draftId && !input.reconnectId) {
|
||||
await page.getByRole("button", { name: "Change", exact: true }).click();
|
||||
await page.getByRole("radio", { name: "Just me", exact: true }).click();
|
||||
}
|
||||
@@ -117,14 +118,14 @@ async function main(): Promise<NotionOAuthReceipt> {
|
||||
timeout: 90000,
|
||||
});
|
||||
// Source UI starts OAuth directly from a durable resume without connectApp.
|
||||
if (input.draftId) void started.catch(() => undefined);
|
||||
if (input.draftId || input.reconnectId) void started.catch(() => undefined);
|
||||
await page
|
||||
.getByRole("button", {
|
||||
name: input.draftId ? "Finish with Notion" : "Continue to Notion",
|
||||
exact: true,
|
||||
})
|
||||
.click();
|
||||
let connectionId = input.draftId;
|
||||
let connectionId = input.draftId ?? input.reconnectId;
|
||||
if (!connectionId) {
|
||||
const response = await started;
|
||||
if (!response.ok())
|
||||
@@ -149,7 +150,19 @@ async function main(): Promise<NotionOAuthReceipt> {
|
||||
const personal = grantState.grants.filter(
|
||||
(g) => g.kind === "user" && g.subjectUserId === input.humanId,
|
||||
);
|
||||
if (
|
||||
if (input.reconnectId) {
|
||||
if (
|
||||
draft.id !== input.reconnectId ||
|
||||
draft.companyId !== input.companyId ||
|
||||
draft.authKind !== "oauth" ||
|
||||
draft.credentialPolicy !== "shared" ||
|
||||
!before.some((c) => c.id === draft.id) ||
|
||||
!grantState.grants.some((g) => g.kind === "organization")
|
||||
)
|
||||
throw new Error(
|
||||
"Actual Notion reconnect changed the existing shared OAuth identity",
|
||||
);
|
||||
} else if (
|
||||
draft.companyId !== input.companyId ||
|
||||
draft.credentialPolicy !== "per_user" ||
|
||||
draft.authKind !== "oauth" ||
|
||||
@@ -166,9 +179,9 @@ async function main(): Promise<NotionOAuthReceipt> {
|
||||
connectionId,
|
||||
humanId: input.humanId,
|
||||
intentId: input.intentId ?? null,
|
||||
personalDraft: true,
|
||||
personalDraft: !input.reconnectId,
|
||||
providerHandoff: true,
|
||||
credentialAbsentAtHandoff: true,
|
||||
credentialAbsentAtHandoff: !input.reconnectId,
|
||||
resumedSameDraft: !!input.draftId,
|
||||
draftSurvivedReload: false,
|
||||
consentCompleted: false,
|
||||
@@ -300,7 +313,7 @@ async function main(): Promise<NotionOAuthReceipt> {
|
||||
connected.companyId !== input.companyId ||
|
||||
connected.status !== "active" ||
|
||||
connected.healthStatus !== "ok" ||
|
||||
connected.credentialPolicy !== "per_user"
|
||||
connected.credentialPolicy !== (input.reconnectId ? "shared" : "per_user")
|
||||
)
|
||||
throw new Error(
|
||||
"Real Notion callback did not activate the same personal draft",
|
||||
|
||||
@@ -25,7 +25,10 @@ import {
|
||||
* credential injection into the instrumented Test browser. */
|
||||
export async function privateNotionOAuth(
|
||||
ctx: ChatCatalogContext,
|
||||
options: Pick<NotionOAuthInput, "operation" | "intentId" | "draftId">,
|
||||
options: Pick<
|
||||
NotionOAuthInput,
|
||||
"operation" | "intentId" | "draftId" | "reconnectId"
|
||||
>,
|
||||
) {
|
||||
const binding = selectedExternalBinding(ctx),
|
||||
p = binding.prerequisites;
|
||||
|
||||
@@ -51,7 +51,10 @@ export function personalNotionGuidance(text: string) {
|
||||
),
|
||||
);
|
||||
}
|
||||
async function sharedNotionAuthority(ctx: ChatCatalogContext, id: string) {
|
||||
export async function sharedNotionAuthority(
|
||||
ctx: ChatCatalogContext,
|
||||
id: string,
|
||||
) {
|
||||
const connection = await ctx.input.api.get<ToolConnection>(
|
||||
`/api/tool-connections/${id}`,
|
||||
);
|
||||
@@ -242,7 +245,7 @@ export function gradeNotionSetup(value: unknown) {
|
||||
}
|
||||
return c.result();
|
||||
}
|
||||
const sourceFiles = [
|
||||
export const notionSetupSourceFiles = [
|
||||
"chat-catalog-domains/external-notion-setup.ts",
|
||||
"chat-catalog-domains/external-notion-oauth.ts",
|
||||
"chat-catalog-domains/external-notion-oauth-worker.ts",
|
||||
@@ -380,7 +383,7 @@ export const notionSetupExternalCases = [
|
||||
expectedHumanMessageCount: 1,
|
||||
turnTimeoutMs: 120000,
|
||||
attemptTimeoutMs: 900000,
|
||||
sourceFiles,
|
||||
sourceFiles: notionSetupSourceFiles,
|
||||
async act(ctx, state) {
|
||||
const rows = await pollUntil({
|
||||
label: "actual addressed pending Notion request",
|
||||
|
||||
Reference in new issue
Block a user