mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 19:28:46 +01:00
test(evals): author real Notion OAuth handoff and draft recovery
This commit is contained in:
Dotta committed
1 parent
2e287f0f61
commit
77d64e17cc
7 files changed
+1532
-3
No files matched your search
@@ -260,3 +260,29 @@ calibration. A visible real form rejection is retained as a nonpassing product
|
||||
observation; the harness does not pre-reject an actual rotating-token format or
|
||||
bypass that form. Slack's documented rotating access-token format is described at
|
||||
https://docs.slack.dev/authentication/using-token-rotation/.
|
||||
|
||||
The named human Notion setup cases (`notion-missing`, `return-from-login`, and
|
||||
`save-and-resume`) use the real legacy catalog OAuth UI and official Notion MCP,
|
||||
not a supplied bearer connection as a substitute. In addition to the independent
|
||||
REST workspace/pages/facts above, they require `workspaceName`,
|
||||
`oauthBrowserSessionEnv` (a selected namespaced reference containing bounded
|
||||
Playwright storage JSON for the authorized synthetic Notion human), and
|
||||
`oauthHumanEmailEnv` (that human's private email reference). Only official secure
|
||||
Notion-domain cookies/storage are accepted. A qualified session must expose the
|
||||
actual account email, selected workspace, and authorization controls; missing or
|
||||
changed controls fail explicitly. No password, OTP mailbox, or generic browser
|
||||
execution fallback is inferred. Notion's [official MCP flow](https://developers.notion.com/guides/mcp/get-started-with-mcp)
|
||||
grants the selected workspace; a private actual `notion-get-self` call verifies
|
||||
that workspace after callback, separately from the model's required launch read.
|
||||
|
||||
These cases require the selected owned HTTPS-origin receipt and actual public
|
||||
sign-in of the same generated fixture person. They do not launch a tunnel or copy
|
||||
loopback cookies. Full-page provider navigation/consent runs in the finite
|
||||
standalone private child, with no Test tracing/video/report hooks. The callback
|
||||
must actually return to the original issue and resolve the exact pending request
|
||||
for its addressee. Save-and-resume uses actual **Continue to Notion** to persist a
|
||||
personal draft before provider consent, leaves through supported navigation,
|
||||
reloads that same draft, then requires its eventual authorization to reuse the
|
||||
same connection ID. No literal Save & exit control or API-created draft is
|
||||
invented. All of this remains authored/unmeasured until a reserved execution has
|
||||
a real authorized disposable Notion session, workspace/pages, and owned origin.
|
||||
@@ -20,12 +20,12 @@ export const externalRequirements: Record<string, { provider: ExternalProvider;
|
||||
"app-permissions.grant-one-agent": { provider: "notion", fields: ["workspaceId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv"] },
|
||||
"gateways-profiles.define-profile": { provider: "notion", fields: ["workspaceId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv"] },
|
||||
"connection-setup.notion-connected": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
|
||||
"connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "recentPageId", "oauthBrowserSession"] },
|
||||
"connection-setup.notion-missing": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
|
||||
"connection-setup.notion-wrong-account": { provider: "notion", fields: ["sharedWorkspaceId", "personalWorkspaceId", "personalPageId", "oauthBrowserSession"] },
|
||||
"connection-setup.notion-agent-unassigned": { provider: "notion", fields: ["workspaceId", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "mcpTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson"] },
|
||||
"connection-setup.notion-expired": { provider: "notion", fields: ["workspaceId", "recentPageId", "expiredAuthorizationEnv", "oauthBrowserSession"] },
|
||||
"connection-setup.return-from-login": { provider: "notion", fields: ["workspaceId", "recentPageId", "oauthBrowserSession", "registeredCallbackOrigin"] },
|
||||
"connection-setup.save-and-resume": { provider: "notion", fields: ["workspaceId", "recentPageId", "oauthBrowserSession"] },
|
||||
"connection-setup.return-from-login": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
|
||||
"connection-setup.save-and-resume": { provider: "notion", fields: ["workspaceId", "workspaceName", "recentPageId", "olderPageId", "mcpUrl", "apiVersion", "apiTokenEnv", "recentMarker", "olderMarker", "recentFactJson", "olderFactJson", "oauthBrowserSessionEnv", "oauthHumanEmailEnv"] },
|
||||
"connection-setup.channel-prerequisites": { provider: "slack", fields: ["workspaceId", "channelId", "senderUserId", "botUserId", "appId", "registeredWebhookOrigin", "webhookRelayPort", "botTokenEnv", "senderTokenEnv", "signingSecretEnv", "configurationTokenEnv", "slackBrowserSessionEnv"] },
|
||||
"chat-channels.connect-slack": { provider: "slack", fields: ["workspaceId", "channelId", "senderUserId", "botUserId", "appId", "registeredWebhookOrigin", "webhookRelayPort", "botTokenEnv", "senderTokenEnv", "signingSecretEnv", "configurationTokenEnv", "slackBrowserSessionEnv"] },
|
||||
"chat-channels.link-sender": { provider: "slack", fields: ["workspaceId", "channelId", "senderUserId", "botUserId", "appId", "registeredWebhookOrigin", "webhookRelayPort", "botTokenEnv", "senderTokenEnv", "signingSecretEnv", "configurationTokenEnv"] },
|
||||
|
||||
@@ -0,0 +1,190 @@
|
||||
import type { PublicSignInStorage } from "./external-public-signin-contract.js";
|
||||
export interface NotionOAuthInput {
|
||||
operation: "save-draft" | "authorize";
|
||||
publicOrigin: string;
|
||||
companyId: string;
|
||||
prefix: string;
|
||||
humanId: string;
|
||||
agentId: string;
|
||||
intentId?: string;
|
||||
issueId?: string;
|
||||
draftId?: string;
|
||||
workspaceId: string;
|
||||
workspaceName: string;
|
||||
providerEmail: string;
|
||||
publicSession: PublicSignInStorage;
|
||||
notionSession: PublicSignInStorage;
|
||||
}
|
||||
export interface NotionOAuthReceipt {
|
||||
connectionId: string;
|
||||
humanId: string;
|
||||
intentId: string | null;
|
||||
personalDraft: true;
|
||||
providerHandoff: true;
|
||||
credentialAbsentAtHandoff: true;
|
||||
resumedSameDraft: boolean;
|
||||
draftSurvivedReload: boolean;
|
||||
consentCompleted: boolean;
|
||||
callbackReturned: boolean;
|
||||
workspaceMatched: boolean;
|
||||
providerHumanMatched: boolean;
|
||||
processIsolation: true;
|
||||
testRunnerAttached: false;
|
||||
tracingEnabled: false;
|
||||
videoEnabled: false;
|
||||
}
|
||||
const id = (s: string | undefined) =>
|
||||
typeof s === "string" && /^[a-zA-Z0-9_-]{1,100}$/.test(s);
|
||||
const notionHost = (host: string) =>
|
||||
/^(?:[a-z0-9-]+\.)*notion\.(?:so|com)$/.test(host);
|
||||
export function notionStorageMatches(
|
||||
storage: PublicSignInStorage,
|
||||
host: (h: string) => boolean,
|
||||
) {
|
||||
return (
|
||||
!!storage &&
|
||||
Array.isArray(storage.cookies) &&
|
||||
storage.cookies.length > 0 &&
|
||||
storage.cookies.length <= 100 &&
|
||||
storage.cookies.every(
|
||||
(c) =>
|
||||
!!c &&
|
||||
typeof c.name === "string" &&
|
||||
!!c.name &&
|
||||
typeof c.value === "string" &&
|
||||
c.value.length > 0 &&
|
||||
c.value.length <= 65536 &&
|
||||
c.secure === true &&
|
||||
host(c.domain.replace(/^\./, "")) &&
|
||||
c.path.startsWith("/"),
|
||||
) &&
|
||||
Array.isArray(storage.origins) &&
|
||||
storage.origins.length <= 10 &&
|
||||
storage.origins.every((o) => {
|
||||
try {
|
||||
const u = new URL(o.origin);
|
||||
return (
|
||||
u.origin === o.origin &&
|
||||
u.protocol === "https:" &&
|
||||
!u.port &&
|
||||
host(u.hostname) &&
|
||||
Array.isArray(o.localStorage) &&
|
||||
o.localStorage.length <= 200 &&
|
||||
o.localStorage.every(
|
||||
(x) =>
|
||||
typeof x.name === "string" &&
|
||||
typeof x.value === "string" &&
|
||||
x.value.length <= 262144,
|
||||
)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
})
|
||||
);
|
||||
}
|
||||
export function validNotionOAuthInput(input: NotionOAuthInput) {
|
||||
try {
|
||||
const u = new URL(input.publicOrigin);
|
||||
return (
|
||||
["save-draft", "authorize"].includes(input.operation) &&
|
||||
u.origin === input.publicOrigin &&
|
||||
u.protocol === "https:" &&
|
||||
!u.username &&
|
||||
!u.password &&
|
||||
!u.port &&
|
||||
[input.companyId, input.humanId, input.agentId, input.prefix].every(id) &&
|
||||
(!input.intentId || id(input.intentId)) &&
|
||||
!!input.intentId === !!input.issueId &&
|
||||
(!input.issueId || id(input.issueId)) &&
|
||||
(!input.draftId || id(input.draftId)) &&
|
||||
/^[a-f0-9-]{32,36}$/i.test(input.workspaceId) &&
|
||||
typeof input.workspaceName === "string" &&
|
||||
input.workspaceName.length > 0 &&
|
||||
input.workspaceName.length <= 200 &&
|
||||
!/[\r\n\0]/.test(input.workspaceName) &&
|
||||
/^[^\s@]+@[^\s@]+\.[^\s@]+$/.test(input.providerEmail) &&
|
||||
input.providerEmail.length <= 254 &&
|
||||
notionStorageMatches(input.notionSession, notionHost) &&
|
||||
notionStorageMatches(input.publicSession, (h) => h === u.hostname)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
/** A self-tool receipt is read privately; no account names, emails or token
|
||||
* material enters the parent. Match ONLY the documented workspace object. */
|
||||
export function notionSelfWorkspaceMatches(
|
||||
value: unknown,
|
||||
workspaceId: string,
|
||||
): boolean {
|
||||
let v = value;
|
||||
for (let n = 0; n < 4; n++) {
|
||||
if (!v || typeof v !== "object") return false;
|
||||
const r = v as Record<string, unknown>;
|
||||
const workspace = r.workspace as { id?: unknown } | undefined;
|
||||
const bot = r.bot as { workspace_id?: unknown } | undefined;
|
||||
if (r.type === "bot" && bot && typeof bot.workspace_id === "string")
|
||||
return (
|
||||
bot.workspace_id.replaceAll("-", "").toLowerCase() ===
|
||||
workspaceId.replaceAll("-", "").toLowerCase()
|
||||
);
|
||||
if (workspace && typeof workspace.id === "string")
|
||||
return (
|
||||
workspace.id.replaceAll("-", "").toLowerCase() ===
|
||||
workspaceId.replaceAll("-", "").toLowerCase()
|
||||
);
|
||||
if (r.result) {
|
||||
v = r.result;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
Array.isArray(r.content) &&
|
||||
r.content.length === 1 &&
|
||||
r.content[0]?.type === "text" &&
|
||||
typeof r.content[0]?.text === "string"
|
||||
) {
|
||||
try {
|
||||
v = JSON.parse(r.content[0].text);
|
||||
continue;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
export function validNotionOAuthReceipt(
|
||||
value: NotionOAuthReceipt,
|
||||
input: NotionOAuthInput,
|
||||
) {
|
||||
const keys =
|
||||
"callbackReturned,connectionId,consentCompleted,credentialAbsentAtHandoff,draftSurvivedReload,humanId,intentId,personalDraft,processIsolation,providerHandoff,providerHumanMatched,resumedSameDraft,testRunnerAttached,tracingEnabled,videoEnabled,workspaceMatched";
|
||||
return (
|
||||
!!value &&
|
||||
Object.keys(value).sort().join(",") === keys &&
|
||||
id(value.connectionId) &&
|
||||
(!input.draftId || input.draftId === value.connectionId) &&
|
||||
value.humanId === input.humanId &&
|
||||
value.intentId === (input.intentId ?? null) &&
|
||||
value.personalDraft === true &&
|
||||
value.providerHandoff === true &&
|
||||
value.credentialAbsentAtHandoff === true &&
|
||||
value.resumedSameDraft === !!input.draftId &&
|
||||
value.processIsolation === true &&
|
||||
value.testRunnerAttached === false &&
|
||||
value.tracingEnabled === false &&
|
||||
value.videoEnabled === false &&
|
||||
(input.operation === "save-draft"
|
||||
? value.draftSurvivedReload === true &&
|
||||
!value.consentCompleted &&
|
||||
!value.callbackReturned &&
|
||||
!value.workspaceMatched &&
|
||||
!value.providerHumanMatched
|
||||
: value.consentCompleted === true &&
|
||||
value.callbackReturned === true &&
|
||||
value.workspaceMatched === true &&
|
||||
value.providerHumanMatched === true)
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,360 @@
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { chromium, test } from "@playwright/test";
|
||||
import type {
|
||||
ToolConnection,
|
||||
ToolCatalogEntry,
|
||||
ToolConnectionTestCallResult,
|
||||
ConnectionGrantsResponse,
|
||||
} from "../../../packages/shared/src/types/tool-access.js";
|
||||
import {
|
||||
validNotionOAuthInput,
|
||||
notionSelfWorkspaceMatches,
|
||||
type NotionOAuthInput,
|
||||
type NotionOAuthReceipt,
|
||||
} from "./external-notion-oauth-contract.js";
|
||||
/** Finite real Notion OAuth operation. Never imported into Playwright Test;
|
||||
* private redirects/session/API bodies cannot enter its step/report hooks. */
|
||||
async function main(): Promise<NotionOAuthReceipt> {
|
||||
let attached = false;
|
||||
try {
|
||||
test.info();
|
||||
attached = true;
|
||||
} catch {
|
||||
/* standalone */
|
||||
}
|
||||
if (attached) throw new Error("Private Notion OAuth must run outside Test");
|
||||
const input = JSON.parse(
|
||||
await readFile(process.argv[2]!, "utf8"),
|
||||
) as NotionOAuthInput & { executablePath?: string; channel?: string };
|
||||
if (!validNotionOAuthInput(input))
|
||||
throw new Error("Invalid private Notion request");
|
||||
const browser = await chromium.launch({
|
||||
headless: true,
|
||||
...(input.executablePath ? { executablePath: input.executablePath } : {}),
|
||||
...(input.channel ? { channel: input.channel } : {}),
|
||||
});
|
||||
try {
|
||||
const storage = path.join(path.dirname(process.argv[2]!), "storage.json");
|
||||
await writeFile(
|
||||
storage,
|
||||
JSON.stringify({
|
||||
cookies: [
|
||||
...input.publicSession.cookies,
|
||||
...input.notionSession.cookies,
|
||||
],
|
||||
origins: [
|
||||
...input.publicSession.origins,
|
||||
...input.notionSession.origins,
|
||||
],
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
const context = await browser.newContext({ storageState: storage });
|
||||
const page = await context.newPage();
|
||||
page.setDefaultTimeout(20000);
|
||||
const api = async <T>(pathname: string): Promise<T> => {
|
||||
const response = await context.request.get(input.publicOrigin + pathname);
|
||||
if (!response.ok()) throw new Error("Private public-state read failed");
|
||||
return (await response.json()) as T;
|
||||
};
|
||||
const session = await api<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
);
|
||||
if (session.user?.id !== input.humanId)
|
||||
throw new Error(
|
||||
"Public Notion setup person differs from addressed human",
|
||||
);
|
||||
const root = `/api/companies/${input.companyId}/tools`;
|
||||
if (input.intentId) {
|
||||
const cards = await api<
|
||||
Array<{
|
||||
id: string;
|
||||
status: string;
|
||||
addresseeUserId?: string;
|
||||
payload?: { serviceSlug: string; requestingAgentId: string };
|
||||
}>
|
||||
>(`/api/issues/${input.issueId}/interactions`);
|
||||
const card = cards.find((c) => c.id === input.intentId);
|
||||
if (
|
||||
!card ||
|
||||
card.status !== "pending" ||
|
||||
card.addresseeUserId !== input.humanId ||
|
||||
card.payload?.serviceSlug !== "notion" ||
|
||||
card.payload.requestingAgentId !== input.agentId
|
||||
)
|
||||
throw new Error(
|
||||
"Notion full-page handoff lost its actual addressed source intent",
|
||||
);
|
||||
}
|
||||
const before = (
|
||||
await api<{ connections: ToolConnection[] }>(root + "/connections")
|
||||
).connections;
|
||||
const params = new URLSearchParams({ source: "notion", stage: "setup" });
|
||||
if (input.intentId) params.set("intent", input.intentId);
|
||||
if (input.draftId) params.set("resume", input.draftId);
|
||||
const setupURL = `${input.publicOrigin}/${input.prefix}/apps/connect?${params}`;
|
||||
await page.goto(setupURL, { waitUntil: "domcontentloaded" });
|
||||
await page
|
||||
.getByRole("heading", {
|
||||
name: input.draftId
|
||||
? "Finish connecting Notion"
|
||||
: "Connect Notion to Paperclip",
|
||||
exact: true,
|
||||
})
|
||||
.waitFor();
|
||||
if (!input.draftId) {
|
||||
await page.getByRole("button", { name: "Change", exact: true }).click();
|
||||
await page.getByRole("radio", { name: "Just me", exact: true }).click();
|
||||
}
|
||||
const started = page.waitForResponse(
|
||||
(r) =>
|
||||
r.url().startsWith(input.publicOrigin + root + "/apps/connect") &&
|
||||
r.request().method() === "POST",
|
||||
);
|
||||
const navigated = page.waitForURL((u) => u.origin !== input.publicOrigin, {
|
||||
timeout: 90000,
|
||||
});
|
||||
// Source UI starts OAuth directly from a durable resume without connectApp.
|
||||
if (input.draftId) void started.catch(() => undefined);
|
||||
await page
|
||||
.getByRole("button", {
|
||||
name: input.draftId ? "Finish with Notion" : "Continue to Notion",
|
||||
exact: true,
|
||||
})
|
||||
.click();
|
||||
let connectionId = input.draftId;
|
||||
if (!connectionId) {
|
||||
const response = await started;
|
||||
if (!response.ok())
|
||||
throw new Error("Actual Notion draft creation failed");
|
||||
connectionId = ((await response.json()) as { connectionId: string })
|
||||
.connectionId;
|
||||
}
|
||||
if (!connectionId) throw new Error("Actual Notion draft ID missing");
|
||||
await navigated;
|
||||
const provider = new URL(page.url());
|
||||
if (
|
||||
provider.protocol !== "https:" ||
|
||||
!/^(?:[a-z0-9-]+\.)*notion\.(?:so|com)$/.test(provider.hostname)
|
||||
)
|
||||
throw new Error("Notion handoff escaped official provider");
|
||||
const draft = await api<ToolConnection>(
|
||||
`/api/tool-connections/${connectionId}`,
|
||||
);
|
||||
const grantState = await api<ConnectionGrantsResponse>(
|
||||
`/api/tool-connections/${connectionId}/grants`,
|
||||
);
|
||||
const personal = grantState.grants.filter(
|
||||
(g) => g.kind === "user" && g.subjectUserId === input.humanId,
|
||||
);
|
||||
if (
|
||||
draft.companyId !== input.companyId ||
|
||||
draft.credentialPolicy !== "per_user" ||
|
||||
draft.authKind !== "oauth" ||
|
||||
draft.status !== "draft" ||
|
||||
draft.credentialSecretRefs.length ||
|
||||
draft.credentialRefs?.length ||
|
||||
personal.length !== 1 ||
|
||||
before.some((c) => c.id === connectionId) !== !!input.draftId
|
||||
)
|
||||
throw new Error(
|
||||
"Notion handoff did not preserve one actual credential-free personal draft",
|
||||
);
|
||||
const receipt: NotionOAuthReceipt = {
|
||||
connectionId,
|
||||
humanId: input.humanId,
|
||||
intentId: input.intentId ?? null,
|
||||
personalDraft: true,
|
||||
providerHandoff: true,
|
||||
credentialAbsentAtHandoff: true,
|
||||
resumedSameDraft: !!input.draftId,
|
||||
draftSurvivedReload: false,
|
||||
consentCompleted: false,
|
||||
callbackReturned: false,
|
||||
workspaceMatched: false,
|
||||
providerHumanMatched: false,
|
||||
processIsolation: true,
|
||||
testRunnerAttached: false,
|
||||
tracingEnabled: false,
|
||||
videoEnabled: false,
|
||||
};
|
||||
if (input.operation === "save-draft") {
|
||||
// Leave before any provider consent; supported setup navigation and actual
|
||||
// reload must hydrate the original durable choices, without a fake draft.
|
||||
const resume = new URL(setupURL);
|
||||
resume.searchParams.set("resume", connectionId);
|
||||
await page.goto(`${input.publicOrigin}/${input.prefix}/apps`, {
|
||||
waitUntil: "domcontentloaded",
|
||||
});
|
||||
await page.goto(resume.href, { waitUntil: "domcontentloaded" });
|
||||
await page.reload();
|
||||
await page
|
||||
.getByRole("button", { name: "Finish with Notion", exact: true })
|
||||
.waitFor();
|
||||
await page.getByRole("button", { name: "Change", exact: true }).click();
|
||||
if (
|
||||
!(await page
|
||||
.getByRole("radio", { name: "Just me", exact: true })
|
||||
.isChecked())
|
||||
)
|
||||
throw new Error("Reloaded Notion draft lost personal identity choice");
|
||||
const current = await api<ToolConnection>(
|
||||
`/api/tool-connections/${connectionId}`,
|
||||
);
|
||||
const rows = (
|
||||
await api<{ connections: ToolConnection[] }>(root + "/connections")
|
||||
).connections;
|
||||
if (
|
||||
current.status !== "draft" ||
|
||||
current.credentialPolicy !== "per_user" ||
|
||||
current.credentialSecretRefs.length ||
|
||||
current.credentialRefs?.length ||
|
||||
rows.length !== before.length + (input.draftId ? 0 : 1)
|
||||
)
|
||||
throw new Error(
|
||||
"Saved Notion draft became usable or duplicated before credentials",
|
||||
);
|
||||
receipt.draftSurvivedReload = true;
|
||||
return receipt;
|
||||
}
|
||||
// Provider UI identity is checked before consent; a session/display name
|
||||
// alone never proves the workspace that was actually granted. The private
|
||||
// get-self call below supplies independent exact workspace authority.
|
||||
await page.getByText(input.providerEmail, { exact: true }).waitFor();
|
||||
const workspace = page.getByRole("combobox");
|
||||
if ((await workspace.count()) === 1) {
|
||||
if (await workspace.evaluate((e) => e.tagName === "SELECT"))
|
||||
await workspace.selectOption({ label: input.workspaceName });
|
||||
else {
|
||||
await workspace.click();
|
||||
await page
|
||||
.getByRole("option", { name: input.workspaceName, exact: true })
|
||||
.click();
|
||||
}
|
||||
}
|
||||
await page
|
||||
.getByText(input.workspaceName, { exact: true })
|
||||
.first()
|
||||
.waitFor();
|
||||
receipt.providerHumanMatched = true;
|
||||
let originalConversationReturned = false;
|
||||
page.on("framenavigated", (frame) => {
|
||||
if (frame !== page.mainFrame() || !input.issueId) return;
|
||||
const u = new URL(frame.url());
|
||||
if (
|
||||
u.origin === input.publicOrigin &&
|
||||
[
|
||||
`/issues/${input.issueId}`,
|
||||
`/${input.prefix}/issues/${input.issueId}`,
|
||||
].includes(u.pathname)
|
||||
)
|
||||
originalConversationReturned = true;
|
||||
});
|
||||
await page
|
||||
.getByRole("button", { name: /^(Allow access|Authorize|Allow)$/i })
|
||||
.click();
|
||||
await page.waitForURL((u) => u.origin === input.publicOrigin, {
|
||||
timeout: 90000,
|
||||
});
|
||||
if (input.issueId) {
|
||||
for (let n = 0; n < 40 && !originalConversationReturned; n++)
|
||||
await new Promise((r) => setTimeout(r, 250));
|
||||
if (!originalConversationReturned)
|
||||
throw new Error(
|
||||
"Notion callback did not actually return to its original source conversation",
|
||||
);
|
||||
const cards = await api<
|
||||
Array<{
|
||||
id: string;
|
||||
status: string;
|
||||
resolvedByUserId?: string;
|
||||
result?: { outcome: string; connectionId: string };
|
||||
}>
|
||||
>(`/api/issues/${input.issueId}/interactions`);
|
||||
const card = cards.find((c) => c.id === input.intentId);
|
||||
if (
|
||||
!card ||
|
||||
card.status === "pending" ||
|
||||
card.resolvedByUserId !== input.humanId ||
|
||||
card.result?.outcome !== "connected" ||
|
||||
card.result.connectionId !== connectionId
|
||||
)
|
||||
throw new Error(
|
||||
"Notion callback did not resolve the exact original addressed conversation request",
|
||||
);
|
||||
}
|
||||
receipt.callbackReturned = true;
|
||||
let connected: ToolConnection | undefined;
|
||||
for (let n = 0; n < 40; n++) {
|
||||
connected = await api<ToolConnection>(
|
||||
`/api/tool-connections/${connectionId}`,
|
||||
);
|
||||
if (connected.status === "active" && connected.healthStatus === "ok")
|
||||
break;
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
if (
|
||||
!connected ||
|
||||
connected.companyId !== input.companyId ||
|
||||
connected.status !== "active" ||
|
||||
connected.healthStatus !== "ok" ||
|
||||
connected.credentialPolicy !== "per_user"
|
||||
)
|
||||
throw new Error(
|
||||
"Real Notion callback did not activate the same personal draft",
|
||||
);
|
||||
const catalog = (
|
||||
await api<{ catalog: ToolCatalogEntry[] }>(
|
||||
`/api/tool-connections/${connectionId}/catalog`,
|
||||
)
|
||||
).catalog;
|
||||
const self = catalog.filter(
|
||||
(e) =>
|
||||
e.toolName === "notion-get-self" &&
|
||||
e.isReadOnly &&
|
||||
e.status === "active",
|
||||
);
|
||||
if (self.length !== 1)
|
||||
throw new Error("Official Notion workspace identity tool is unavailable");
|
||||
const testCall = await context.request.post(
|
||||
`${input.publicOrigin}/api/tool-connections/${connectionId}/test-calls`,
|
||||
{
|
||||
data: {
|
||||
agentId: input.agentId,
|
||||
toolName: self[0]!.toolName,
|
||||
parameters: {},
|
||||
},
|
||||
},
|
||||
);
|
||||
if (!testCall.ok())
|
||||
throw new Error("Independent Notion get-self could not execute");
|
||||
const result = (await testCall.json()) as ToolConnectionTestCallResult;
|
||||
if (
|
||||
result.decision !== "allowed" ||
|
||||
result.error ||
|
||||
!notionSelfWorkspaceMatches(result.result, input.workspaceId)
|
||||
)
|
||||
throw new Error(
|
||||
"Actual granted Notion workspace differs from selected personal fixture",
|
||||
);
|
||||
if (
|
||||
(await api<{ user: { id: string } }>("/api/auth/get-session")).user.id !==
|
||||
input.humanId
|
||||
)
|
||||
throw new Error("Notion callback changed the signed-in person");
|
||||
receipt.workspaceMatched = true;
|
||||
receipt.consentCompleted = true;
|
||||
return receipt;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
main()
|
||||
.then((value) => process.stdout.write(JSON.stringify(value)))
|
||||
.catch(() => {
|
||||
process.stderr.write(
|
||||
"Private Notion OAuth operation failed; provider/session/redirect details withheld\n",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,181 @@
|
||||
import { mkdtemp, writeFile, rm, chmod } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
import { chromium } from "@playwright/test";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
import { signInPublicFixturePerson } from "../public-mcp-flow.js";
|
||||
import {
|
||||
readValidatedPublicOriginReceipt,
|
||||
PUBLIC_ORIGIN_RECEIPT_ENV,
|
||||
} from "../chat-public-origin.js";
|
||||
import {
|
||||
selectedExternalBinding,
|
||||
selectedExternalCredential,
|
||||
} from "./external-fixtures.js";
|
||||
import {
|
||||
validNotionOAuthInput,
|
||||
validNotionOAuthReceipt,
|
||||
type NotionOAuthInput,
|
||||
type NotionOAuthReceipt,
|
||||
} from "./external-notion-oauth-contract.js";
|
||||
/** Selected actual human/provider session only; no generic navigation or private
|
||||
* credential injection into the instrumented Test browser. */
|
||||
export async function privateNotionOAuth(
|
||||
ctx: ChatCatalogContext,
|
||||
options: Pick<NotionOAuthInput, "operation" | "intentId" | "draftId">,
|
||||
) {
|
||||
const binding = selectedExternalBinding(ctx),
|
||||
p = binding.prerequisites;
|
||||
const secrets = ctx.input.privateSecrets;
|
||||
if (!secrets)
|
||||
throw new Error(
|
||||
"Private Notion OAuth requires the owning attempt redaction sink",
|
||||
);
|
||||
const sessionJson = selectedExternalCredential(
|
||||
ctx,
|
||||
binding,
|
||||
"oauthBrowserSessionEnv",
|
||||
);
|
||||
const providerEmail = selectedExternalCredential(
|
||||
ctx,
|
||||
binding,
|
||||
"oauthHumanEmailEnv",
|
||||
);
|
||||
secrets.push(sessionJson, providerEmail);
|
||||
if (Buffer.byteLength(sessionJson) > 2 * 1024 * 1024)
|
||||
throw new Error(
|
||||
"Setup blocker: bounded selected Notion human session is required",
|
||||
);
|
||||
let notionSession: NotionOAuthInput["notionSession"];
|
||||
try {
|
||||
notionSession = JSON.parse(sessionJson);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Setup blocker: selected private Notion human session JSON is invalid",
|
||||
);
|
||||
}
|
||||
for (const cookie of notionSession?.cookies ?? [])
|
||||
if (typeof cookie.value === "string") secrets.push(cookie.value);
|
||||
for (const origin of notionSession?.origins ?? [])
|
||||
for (const item of origin.localStorage ?? [])
|
||||
if (typeof item.value === "string" && item.value)
|
||||
secrets.push(item.value);
|
||||
await signInPublicFixturePerson(ctx.input.api, ctx.input.page.context());
|
||||
const receiptPath = process.env[PUBLIC_ORIGIN_RECEIPT_ENV],
|
||||
attemptId = process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID;
|
||||
const executions = JSON.parse(
|
||||
process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS ?? "[]",
|
||||
) as string[];
|
||||
if (!receiptPath || !attemptId || executions.length !== 1)
|
||||
throw new Error(
|
||||
"Setup blocker: selected Notion attempt requires its owned validated public origin",
|
||||
);
|
||||
const origin = await readValidatedPublicOriginReceipt({
|
||||
receiptPath,
|
||||
attemptId,
|
||||
executionId: executions[0]!,
|
||||
backendOrigin: ctx.input.api.baseURL,
|
||||
});
|
||||
const publicStorage = await ctx.input.page.context().storageState();
|
||||
const host = new URL(origin.publicOrigin).hostname;
|
||||
const publicSession = {
|
||||
cookies: publicStorage.cookies.filter(
|
||||
(c) => c.domain.replace(/^\./, "") === host,
|
||||
),
|
||||
origins: publicStorage.origins.filter(
|
||||
(o) => o.origin === origin.publicOrigin,
|
||||
),
|
||||
};
|
||||
const human = await ctx.input.api.get<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
);
|
||||
const input: NotionOAuthInput = {
|
||||
...options,
|
||||
...(options.intentId ? { issueId: ctx.issue()?.id } : {}),
|
||||
publicOrigin: origin.publicOrigin,
|
||||
companyId: ctx.input.fixtures.company.id,
|
||||
prefix: ctx.input.fixtures.company.issuePrefix ?? "",
|
||||
humanId: human.user.id,
|
||||
agentId: ctx.input.fixtures.agent.id,
|
||||
workspaceId: p.workspaceId!,
|
||||
workspaceName: p.workspaceName!,
|
||||
providerEmail,
|
||||
notionSession,
|
||||
publicSession,
|
||||
};
|
||||
if (!validNotionOAuthInput(input))
|
||||
throw new Error(
|
||||
"Setup blocker: Notion human/workspace/session prerequisite is outside the finite named-provider contract",
|
||||
);
|
||||
const directory = await mkdtemp(
|
||||
path.join(tmpdir(), "paperclip-notion-oauth-"),
|
||||
);
|
||||
await chmod(directory, 0o700);
|
||||
try {
|
||||
const request = path.join(directory, "request.json"),
|
||||
channel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL;
|
||||
await writeFile(
|
||||
request,
|
||||
JSON.stringify({
|
||||
...input,
|
||||
executablePath:
|
||||
process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ??
|
||||
(channel ? undefined : chromium.executablePath()),
|
||||
channel,
|
||||
}),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
let stdout: string;
|
||||
try {
|
||||
stdout = (
|
||||
await promisify(execFile)(
|
||||
process.execPath,
|
||||
[
|
||||
"--import",
|
||||
fileURLToPath(
|
||||
new URL(
|
||||
"../../../server/node_modules/tsx/dist/loader.mjs",
|
||||
import.meta.url,
|
||||
),
|
||||
),
|
||||
fileURLToPath(
|
||||
new URL("./external-notion-oauth-worker.ts", import.meta.url),
|
||||
),
|
||||
request,
|
||||
],
|
||||
{
|
||||
timeout: 240000,
|
||||
maxBuffer: 16384,
|
||||
env: {
|
||||
PATH: process.env.PATH ?? "",
|
||||
HOME: directory,
|
||||
TMPDIR: directory,
|
||||
NODE_ENV: "test",
|
||||
DO_NOT_TRACK: "1",
|
||||
},
|
||||
},
|
||||
)
|
||||
).stdout;
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Private named Notion setup failed; session/provider/redirect details withheld",
|
||||
);
|
||||
}
|
||||
let receipt: NotionOAuthReceipt;
|
||||
try {
|
||||
receipt = JSON.parse(stdout);
|
||||
} catch {
|
||||
throw new Error("Private Notion setup did not return its closed receipt");
|
||||
}
|
||||
if (!validNotionOAuthReceipt(receipt, input))
|
||||
throw new Error(
|
||||
"Private Notion setup receipt failed its exact identity/draft/isolation contract",
|
||||
);
|
||||
return receipt;
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,330 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
gradeNotionSetup,
|
||||
notionSetupExternalCases,
|
||||
} from "./external-notion-setup.js";
|
||||
import {
|
||||
validNotionOAuthInput,
|
||||
validNotionOAuthReceipt,
|
||||
notionSelfWorkspaceMatches,
|
||||
type NotionOAuthInput,
|
||||
type NotionOAuthReceipt,
|
||||
} from "./external-notion-oauth-contract.js";
|
||||
import type { ExternalObservation } from "./external-support.js";
|
||||
const workspaceId = "11111111-1111-4111-8111-111111111111",
|
||||
pageId = "22222222-2222-4222-8222-222222222222";
|
||||
const storage = (domain: string) => ({
|
||||
cookies: [
|
||||
{
|
||||
name: "session",
|
||||
value: "synthetic-private-session",
|
||||
domain,
|
||||
path: "/",
|
||||
secure: true,
|
||||
},
|
||||
],
|
||||
origins: [],
|
||||
});
|
||||
const input: NotionOAuthInput = {
|
||||
operation: "authorize",
|
||||
publicOrigin: "https://owned.example.test",
|
||||
companyId: "company",
|
||||
prefix: "ACORN",
|
||||
humanId: "person",
|
||||
agentId: "ceo",
|
||||
intentId: "intent",
|
||||
issueId: "source",
|
||||
workspaceId,
|
||||
workspaceName: "Synthetic Launch",
|
||||
providerEmail: "synthetic@example.test",
|
||||
publicSession: storage("owned.example.test"),
|
||||
notionSession: storage(".notion.so"),
|
||||
};
|
||||
function receipt(): NotionOAuthReceipt {
|
||||
return {
|
||||
connectionId: "connection",
|
||||
humanId: "person",
|
||||
intentId: "intent",
|
||||
personalDraft: true,
|
||||
credentialAbsentAtHandoff: true,
|
||||
providerHandoff: true,
|
||||
resumedSameDraft: false,
|
||||
draftSurvivedReload: false,
|
||||
consentCompleted: true,
|
||||
callbackReturned: true,
|
||||
workspaceMatched: true,
|
||||
providerHumanMatched: true,
|
||||
processIsolation: true,
|
||||
testRunnerAttached: false,
|
||||
tracingEnabled: false,
|
||||
videoEnabled: false,
|
||||
};
|
||||
}
|
||||
function valid(id: string): ExternalObservation {
|
||||
return {
|
||||
scenarioId: id,
|
||||
companyId: "company",
|
||||
leadId: "ceo",
|
||||
origin: "http://127.0.0.1:3100",
|
||||
state: {
|
||||
intentId: "intent",
|
||||
initialRunId: "initial",
|
||||
humanId: "person",
|
||||
pendingBeforeHuman: true,
|
||||
cardVisible: true,
|
||||
pendingWithoutCredential: true,
|
||||
oauthReceipt: receipt(),
|
||||
connectionId: "connection",
|
||||
samePersonalActive: true,
|
||||
currentHumanPreserved: true,
|
||||
originalSourcePreserved: true,
|
||||
sourceIssueId: "source",
|
||||
readEntryIds: ["read"],
|
||||
decisions: [
|
||||
{
|
||||
runId: "continuation",
|
||||
decisions: [
|
||||
{
|
||||
invocation: {
|
||||
runId: "continuation",
|
||||
agentId: "ceo",
|
||||
issueId: "source",
|
||||
connectionId: "connection",
|
||||
catalogEntryId: "read",
|
||||
status: "succeeded",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
pages: [
|
||||
{
|
||||
id: pageId,
|
||||
editedAt: "2026-10-26T08:00:00.000Z",
|
||||
fact: {
|
||||
subjectAliases: ["launch", "release"],
|
||||
valueAliases: ["November 8", "8 November"],
|
||||
},
|
||||
},
|
||||
],
|
||||
pagesUnchanged: true,
|
||||
},
|
||||
reply: "As of Oct 26, 2026, the release is now set for 8 November.",
|
||||
phaseReplies: [
|
||||
{
|
||||
runId: "initial",
|
||||
body: "Please connect your personal Notion with the secure setup card; I cannot read it until you authorize.",
|
||||
},
|
||||
{
|
||||
runId: "continuation",
|
||||
commentId: "comment",
|
||||
body: "As of Oct 26, 2026, the release is now set for 8 November.",
|
||||
},
|
||||
],
|
||||
connections: [{ id: "connection" }],
|
||||
interactions: [
|
||||
{
|
||||
id: "intent",
|
||||
kind: "connection_intent",
|
||||
sourceRunId: "initial",
|
||||
addresseeUserId: "person",
|
||||
resolvedByUserId: "person",
|
||||
result: { outcome: "connected", connectionId: "connection" },
|
||||
payload: { serviceSlug: "notion" },
|
||||
},
|
||||
],
|
||||
profiles: [],
|
||||
secrets: [],
|
||||
endpoints: [],
|
||||
activity: [],
|
||||
links: [`https://www.notion.so/Launch-${pageId}?source=chat`],
|
||||
runs: [
|
||||
{ id: "initial", status: "succeeded" },
|
||||
{ id: "continuation", status: "succeeded" },
|
||||
],
|
||||
humanQueries: [],
|
||||
} as unknown as ExternalObservation;
|
||||
}
|
||||
describe("named Notion human setup controls", () => {
|
||||
it("accepts actual personal callback plus a new source-bound body read", () => {
|
||||
for (const id of [
|
||||
"connection-setup.notion-missing",
|
||||
"connection-setup.return-from-login",
|
||||
])
|
||||
expect(gradeNotionSetup(valid(id)).passed).toBe(true);
|
||||
});
|
||||
it("accepts actual leave/reload/resume of the same credential-free draft", () => {
|
||||
const o = valid("connection-setup.save-and-resume");
|
||||
o.state.savedDraft = {
|
||||
...receipt(),
|
||||
intentId: null,
|
||||
consentCompleted: false,
|
||||
callbackReturned: false,
|
||||
workspaceMatched: false,
|
||||
providerHumanMatched: false,
|
||||
draftSurvivedReload: true,
|
||||
};
|
||||
o.state.savedBeforeCanonical = true;
|
||||
(o.state.oauthReceipt as NotionOAuthReceipt).resumedSameDraft = true;
|
||||
expect(gradeNotionSetup(o).passed).toBe(true);
|
||||
});
|
||||
const wrong: Array<[string, (o: ExternalObservation) => void]> = [
|
||||
[
|
||||
"wrong addressed human",
|
||||
(o) => {
|
||||
o.state.humanId = "other";
|
||||
},
|
||||
],
|
||||
[
|
||||
"premature completed summary",
|
||||
(o) => {
|
||||
o.phaseReplies[0]!.body =
|
||||
"I've already reviewed Notion. The launch is November 8.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"provider refusal alone",
|
||||
(o) => {
|
||||
o.state.oauthReceipt = undefined;
|
||||
o.state.decisions = [];
|
||||
o.phaseReplies[1]!.body = "Notion is unavailable; I could not read it.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"wrong granted workspace",
|
||||
(o) => {
|
||||
(o.state.oauthReceipt as NotionOAuthReceipt).workspaceMatched = false;
|
||||
},
|
||||
],
|
||||
[
|
||||
"another callback user",
|
||||
(o) => {
|
||||
(o.state.oauthReceipt as NotionOAuthReceipt).humanId = "other";
|
||||
},
|
||||
],
|
||||
[
|
||||
"duplicate connection",
|
||||
(o) => {
|
||||
o.connections.push({ id: "duplicate" });
|
||||
},
|
||||
],
|
||||
[
|
||||
"oracle-only read",
|
||||
(o) => {
|
||||
o.state.decisions = [];
|
||||
},
|
||||
],
|
||||
[
|
||||
"read on wrong source",
|
||||
(o) => {
|
||||
(o.state.decisions as any[])[0].decisions[0].invocation.issueId =
|
||||
"unrelated";
|
||||
},
|
||||
],
|
||||
[
|
||||
"stale or title-only summary",
|
||||
(o) => {
|
||||
o.phaseReplies[1]!.body = "The launch page title looks promising.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"wrong update date",
|
||||
(o) => {
|
||||
o.phaseReplies[1]!.body = "On Oct 25, 2026 the launch is November 8.";
|
||||
},
|
||||
],
|
||||
[
|
||||
"wrong page identity",
|
||||
(o) => {
|
||||
o.links = ["https://www.notion.so/33333333333343338333333333333333"];
|
||||
},
|
||||
],
|
||||
];
|
||||
for (const [name, mutate] of wrong)
|
||||
it(`rejects ${name}`, () => {
|
||||
const o = valid("connection-setup.notion-missing");
|
||||
mutate(o);
|
||||
expect(gradeNotionSetup(o).passed).toBe(false);
|
||||
});
|
||||
it("rejects a fabricated or replaced resume draft", () => {
|
||||
const o = valid("connection-setup.save-and-resume");
|
||||
o.state.savedDraft = { ...receipt(), connectionId: "other" };
|
||||
o.state.savedBeforeCanonical = true;
|
||||
expect(gradeNotionSetup(o).passed).toBe(false);
|
||||
});
|
||||
it("keeps exact concrete callbacks and phase bounds", () => {
|
||||
expect(notionSetupExternalCases.map((c) => c.scenarioId)).toHaveLength(3);
|
||||
for (const c of notionSetupExternalCases) {
|
||||
expect(
|
||||
[c.setup, c.act, c.observe, c.verifyResult, c.cleanup].every(
|
||||
(f) => typeof f === "function",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(c.expectedRunCount).toBe(2);
|
||||
expect(c.expectedHumanMessageCount).toBe(1);
|
||||
expect(c.coverageGaps).toBeUndefined();
|
||||
}
|
||||
});
|
||||
it("accepts only isolated selected official provider/public sessions", () => {
|
||||
expect(validNotionOAuthInput(input)).toBe(true);
|
||||
for (const change of [
|
||||
{ publicOrigin: "http://owned.example.test" },
|
||||
{ publicSession: storage("127.0.0.1") },
|
||||
{ notionSession: storage("notion.so.attacker.test") },
|
||||
{ workspaceName: "Synthetic\nLaunch" },
|
||||
{ providerEmail: "not-an-email" },
|
||||
])
|
||||
expect(validNotionOAuthInput({ ...input, ...change })).toBe(false);
|
||||
});
|
||||
it("requires actual same-user closed callback receipt", () => {
|
||||
expect(validNotionOAuthReceipt(receipt(), input)).toBe(true);
|
||||
for (const change of [
|
||||
{ callbackReturned: false },
|
||||
{ processIsolation: false },
|
||||
{ humanId: "other" },
|
||||
{ intentId: "wrong" },
|
||||
{ providerHumanMatched: false },
|
||||
{ workspaceMatched: false },
|
||||
{ extra: "private" },
|
||||
])
|
||||
expect(
|
||||
validNotionOAuthReceipt(
|
||||
{ ...receipt(), ...change } as NotionOAuthReceipt,
|
||||
input,
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
it("matches only actual workspace authority, never a coincidental nested ID", () => {
|
||||
expect(
|
||||
notionSelfWorkspaceMatches(
|
||||
{
|
||||
result: {
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: JSON.stringify({
|
||||
type: "bot",
|
||||
bot: { workspace_id: workspaceId },
|
||||
}),
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
workspaceId,
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
notionSelfWorkspaceMatches(
|
||||
{ workspace: { id: workspaceId.replaceAll("-", "") } },
|
||||
workspaceId,
|
||||
),
|
||||
).toBe(true);
|
||||
for (const result of [
|
||||
{ unrelated: { id: workspaceId } },
|
||||
{ workspace: { id: pageId } },
|
||||
{ type: "bot", bot: {} },
|
||||
{ content: [{ type: "text", text: "Notion connected" }] },
|
||||
])
|
||||
expect(notionSelfWorkspaceMatches(result, workspaceId)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,442 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js";
|
||||
import type {
|
||||
ToolConnection,
|
||||
ConnectionGrantsResponse,
|
||||
ToolRunDecisionLookup,
|
||||
ToolProfileEffectiveSummary,
|
||||
} from "../../../packages/shared/src/types/tool-access.js";
|
||||
import { pollUntil } from "../api.js";
|
||||
import { externalRequirements } from "./external-fixtures.js";
|
||||
import { privateNotionOAuth } from "./external-notion-oauth.js";
|
||||
import type { NotionOAuthReceipt } from "./external-notion-oauth-contract.js";
|
||||
import {
|
||||
readNotionFixturePages,
|
||||
type NotionPageOracle,
|
||||
} from "./external-notion-provider.js";
|
||||
import {
|
||||
notionDateMentioned,
|
||||
notionFactSpan,
|
||||
notionPageIdentityMatches,
|
||||
} from "./external-notion-semantics.js";
|
||||
import {
|
||||
checks,
|
||||
definition,
|
||||
externalObservation,
|
||||
observeExternal,
|
||||
root,
|
||||
verifyRenderedReply,
|
||||
} from "./external-support.js";
|
||||
|
||||
export function gradeNotionSetup(value: unknown) {
|
||||
const c = checks(),
|
||||
o = externalObservation(value);
|
||||
c.add(
|
||||
"actual-personal-notion-setup-evidence",
|
||||
!!o,
|
||||
"Actual named-provider setup, pending request and provider-native read evidence are required.",
|
||||
);
|
||||
if (!o) return c.result();
|
||||
const card = o.interactions.find(
|
||||
(row) => row.id === o.state.intentId,
|
||||
) as unknown as ConnectionIntentInteraction | undefined;
|
||||
const receipt = o.state.oauthReceipt as NotionOAuthReceipt | undefined;
|
||||
const initial = o.phaseReplies
|
||||
.filter((row) => row.runId === o.state.initialRunId)
|
||||
.map((row) => row.body)
|
||||
.join("\n");
|
||||
const continuation = o.phaseReplies
|
||||
.filter((row) => row.runId !== o.state.initialRunId)
|
||||
.map((row) => row.body)
|
||||
.join("\n");
|
||||
const pages = o.state.pages as NotionPageOracle[] | undefined,
|
||||
recent = pages?.[0],
|
||||
older = pages?.[1];
|
||||
c.add(
|
||||
"single-addressed-pending-request",
|
||||
!!card &&
|
||||
o.interactions.filter((row) => row.kind === "connection_intent")
|
||||
.length === 1 &&
|
||||
card.sourceRunId === o.state.initialRunId &&
|
||||
card.addresseeUserId === o.state.humanId &&
|
||||
o.state.pendingBeforeHuman === true &&
|
||||
o.state.cardVisible === true &&
|
||||
o.state.pendingWithoutCredential === true,
|
||||
"The original model's addressed request remains pending without credentials; no hidden replacement or premature resolution.",
|
||||
);
|
||||
c.add(
|
||||
"no-premature-unseen-summary",
|
||||
!!recent &&
|
||||
!notionFactSpan(initial, recent.fact) &&
|
||||
(!older || !notionFactSpan(initial, older.fact)) &&
|
||||
!/\b(?:I(?: have|'ve|’ve)? (?:already )?(?:read|reviewed|summarized)|here(?: is|'s|’s) (?:the |your )?(?:latest|launch summary))/i.test(
|
||||
initial,
|
||||
),
|
||||
"The setup reply cannot claim research or substitute unseen page facts before human authorization.",
|
||||
);
|
||||
c.add(
|
||||
"actual-human-provider-consent",
|
||||
!!receipt &&
|
||||
receipt.humanId === o.state.humanId &&
|
||||
receipt.intentId === card?.id &&
|
||||
receipt.personalDraft === true &&
|
||||
receipt.credentialAbsentAtHandoff === true &&
|
||||
receipt.providerHandoff === true &&
|
||||
receipt.consentCompleted === true &&
|
||||
receipt.callbackReturned === true &&
|
||||
receipt.workspaceMatched === true &&
|
||||
receipt.providerHumanMatched === true &&
|
||||
receipt.processIsolation === true &&
|
||||
receipt.testRunnerAttached === false &&
|
||||
receipt.tracingEnabled === false &&
|
||||
receipt.videoEnabled === false,
|
||||
"The addressed fixture person actually completes full-page named Notion consent and returns; private get-self independently proves the selected workspace.",
|
||||
);
|
||||
c.add(
|
||||
"same-usable-personal-connection",
|
||||
!!card &&
|
||||
card.result?.outcome === "connected" &&
|
||||
card.result.connectionId === receipt?.connectionId &&
|
||||
card.resolvedByUserId === o.state.humanId &&
|
||||
!card.resolvedByAgentId &&
|
||||
o.state.connectionId === receipt?.connectionId &&
|
||||
o.connections.length === 1 &&
|
||||
o.state.samePersonalActive === true &&
|
||||
o.state.currentHumanPreserved === true &&
|
||||
o.state.originalSourcePreserved === true,
|
||||
"Only the original personal draft becomes usable; the actual callback resolves the same request/user/conversation with one account.",
|
||||
);
|
||||
const decisions = o.state.decisions as ToolRunDecisionLookup[] | undefined;
|
||||
c.add(
|
||||
"new-canonical-continuation-read",
|
||||
!!decisions?.some(
|
||||
(row) =>
|
||||
row.runId !== o.state.initialRunId &&
|
||||
o.runs.some(
|
||||
(run) => run.id === row.runId && run.status === "succeeded",
|
||||
) &&
|
||||
row.decisions.some(
|
||||
(d) =>
|
||||
d.invocation.runId === row.runId &&
|
||||
d.invocation.agentId === o.leadId &&
|
||||
d.invocation.issueId === o.state.sourceIssueId &&
|
||||
d.invocation.connectionId === o.state.connectionId &&
|
||||
d.invocation.status === "succeeded" &&
|
||||
(o.state.readEntryIds as string[] | undefined)?.includes(
|
||||
d.invocation.catalogEntryId ?? "",
|
||||
),
|
||||
),
|
||||
),
|
||||
"A new native continuation actually reads through the newly authorized Notion connection; the independent get-self/oracle read is insufficient.",
|
||||
);
|
||||
c.add(
|
||||
"current-body-date-and-source",
|
||||
!!recent &&
|
||||
!!notionFactSpan(continuation, recent.fact) &&
|
||||
notionDateMentioned(continuation, recent.editedAt) &&
|
||||
o.links.some((link) => notionPageIdentityMatches(link, recent.id)) &&
|
||||
(!older ||
|
||||
!notionFactSpan(continuation, older.fact) ||
|
||||
/older|background|previous|earlier/i.test(continuation)) &&
|
||||
o.state.pagesUnchanged === true,
|
||||
"The continuation summarizes the actual recent body fact with its real date/source, distinguishing older background.",
|
||||
);
|
||||
if (o.scenarioId === "connection-setup.save-and-resume") {
|
||||
const saved = o.state.savedDraft as NotionOAuthReceipt | undefined;
|
||||
c.add(
|
||||
"real-human-saved-uncredentialed-draft",
|
||||
!!saved &&
|
||||
saved.personalDraft === true &&
|
||||
saved.credentialAbsentAtHandoff === true &&
|
||||
saved.providerHandoff === true &&
|
||||
saved.draftSurvivedReload === true &&
|
||||
!saved.consentCompleted &&
|
||||
!saved.callbackReturned &&
|
||||
saved.intentId === null &&
|
||||
saved.connectionId === receipt?.connectionId &&
|
||||
receipt?.resumedSameDraft === true &&
|
||||
o.state.savedBeforeCanonical === true,
|
||||
"Before the canonical query, actual Continue to Notion saves personal choices; the human leaves before credentials, reloads, and later authorizes that same durable draft.",
|
||||
);
|
||||
}
|
||||
return c.result();
|
||||
}
|
||||
const sourceFiles = [
|
||||
"chat-catalog-domains/external-notion-setup.ts",
|
||||
"chat-catalog-domains/external-notion-oauth.ts",
|
||||
"chat-catalog-domains/external-notion-oauth-worker.ts",
|
||||
"chat-catalog-domains/external-notion-oauth-contract.ts",
|
||||
"chat-catalog-domains/external-notion-provider.ts",
|
||||
"chat-catalog-domains/external-notion-semantics.ts",
|
||||
"chat-catalog-domains/external-public-signin.ts",
|
||||
"chat-catalog-domains/external-public-signin-worker.ts",
|
||||
"chat-catalog-domains/external-public-signin-contract.ts",
|
||||
"public-mcp-flow.ts",
|
||||
"chat-public-origin.ts",
|
||||
"chat-catalog-domains/external-fixtures.ts",
|
||||
"chat-catalog-domains/external-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-scenarios/connection-setup.yaml",
|
||||
"../../ui/src/features/connections/ConnectionSetupFlow.tsx",
|
||||
"../../ui/src/features/connections/ConnectionIntentInteractionBody.tsx",
|
||||
"../../server/src/routes/tool-access.ts",
|
||||
"../../server/src/services/tool-access/oauth/direct-callback.ts",
|
||||
"../../scripts/smoke/notion-generic-live.mjs",
|
||||
];
|
||||
export const notionSetupExternalCases = [
|
||||
"connection-setup.notion-missing",
|
||||
"connection-setup.return-from-login",
|
||||
"connection-setup.save-and-resume",
|
||||
].map((id) =>
|
||||
definition(
|
||||
id,
|
||||
async (ctx) => {
|
||||
const connections = (
|
||||
await ctx.input.api.get<{ connections: ToolConnection[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
)
|
||||
).connections;
|
||||
if (connections.length)
|
||||
throw new Error(
|
||||
"No-connection Notion setup requires a fresh disposable company",
|
||||
);
|
||||
const pages = await readNotionFixturePages(ctx);
|
||||
const humanId = (
|
||||
await ctx.input.api.get<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
)
|
||||
).user.id;
|
||||
const state: Record<string, unknown> = {
|
||||
pages,
|
||||
humanId,
|
||||
baselineConnectionIds: [],
|
||||
savedBeforeCanonical: false,
|
||||
};
|
||||
if (id === "connection-setup.save-and-resume") {
|
||||
try {
|
||||
state.savedDraft = await privateNotionOAuth(ctx, {
|
||||
operation: "save-draft",
|
||||
});
|
||||
state.savedBeforeCanonical = true;
|
||||
} catch (error) {
|
||||
// Every connection in this newly owned disposable company was created by
|
||||
// this one finite UI operation. Do not leave an unreturned private draft.
|
||||
for (const row of (
|
||||
await ctx.input.api.get<{ connections: ToolConnection[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
)
|
||||
).connections)
|
||||
await ctx.input.api.delete(`/api/tool-connections/${row.id}`, {
|
||||
allowNotFound: true,
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
return state;
|
||||
},
|
||||
gradeNotionSetup,
|
||||
{
|
||||
authentication: "human-session",
|
||||
externalInput: externalRequirements[id],
|
||||
expectedRunCount: 2,
|
||||
minimumRunCount: 2,
|
||||
expectedHumanMessageCount: 1,
|
||||
turnTimeoutMs: 120000,
|
||||
attemptTimeoutMs: 900000,
|
||||
sourceFiles,
|
||||
async act(ctx, state) {
|
||||
const rows = await pollUntil({
|
||||
label: "actual addressed pending Notion request",
|
||||
deadlineAt: Date.now() + 150000,
|
||||
load: async () => {
|
||||
await ctx.refreshIssue();
|
||||
return ctx.issue()
|
||||
? ctx.input.api.get<ConnectionIntentInteraction[]>(
|
||||
`/api/issues/${ctx.issue()!.id}/interactions`,
|
||||
)
|
||||
: [];
|
||||
},
|
||||
accept: (rows) =>
|
||||
rows.some(
|
||||
(row) =>
|
||||
row.kind === "connection_intent" &&
|
||||
row.status === "pending" &&
|
||||
row.payload.serviceSlug === "notion",
|
||||
),
|
||||
});
|
||||
const pending = rows.filter(
|
||||
(row) =>
|
||||
row.kind === "connection_intent" &&
|
||||
row.status === "pending" &&
|
||||
row.payload.serviceSlug === "notion",
|
||||
);
|
||||
if (
|
||||
pending.length !== 1 ||
|
||||
pending[0]!.addresseeUserId !== state.humanId ||
|
||||
pending[0]!.payload.requestingAgentId !== ctx.input.fixtures.agent.id
|
||||
)
|
||||
throw new Error(
|
||||
"Notion request is not uniquely addressed to the actual fixture person/requesting CEO",
|
||||
);
|
||||
const card = pending[0]!;
|
||||
state.intentId = card.id;
|
||||
state.initialRunId = card.sourceRunId;
|
||||
state.sourceIssueId = ctx.issue()!.id;
|
||||
state.pendingBeforeHuman = true;
|
||||
await expect(
|
||||
ctx.input.page
|
||||
.getByRole("button", {
|
||||
name: /^(Connect|Connect \/ Use existing)$/,
|
||||
exact: true,
|
||||
})
|
||||
.first(),
|
||||
).toBeVisible();
|
||||
state.cardVisible = true;
|
||||
const before = (
|
||||
await ctx.input.api.get<{ connections: ToolConnection[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
)
|
||||
).connections;
|
||||
const saved = state.savedDraft as NotionOAuthReceipt | undefined;
|
||||
state.pendingWithoutCredential =
|
||||
before.length === (saved ? 1 : 0) &&
|
||||
before.every(
|
||||
(row) =>
|
||||
row.id === saved?.connectionId &&
|
||||
row.status === "draft" &&
|
||||
row.credentialPolicy === "per_user" &&
|
||||
!row.credentialSecretRefs.length &&
|
||||
!row.credentialRefs?.length,
|
||||
);
|
||||
const current = await observeExternal(ctx, state);
|
||||
const initial = current.phaseReplies
|
||||
.filter((row) => row.runId === card.sourceRunId)
|
||||
.map((row) => row.body)
|
||||
.join("\n");
|
||||
const pages = state.pages as NotionPageOracle[];
|
||||
if (
|
||||
!state.pendingWithoutCredential ||
|
||||
!initial.trim() ||
|
||||
pages.some((page) => notionFactSpan(initial, page.fact))
|
||||
)
|
||||
throw new Error(
|
||||
"Notion request was prematurely satisfied or already exposed unseen fixture facts",
|
||||
);
|
||||
// The visible addressed card's supported full-page setup host preserves the
|
||||
// exact same intent. No API connection or completion substitutes for UI.
|
||||
state.oauthReceipt = await privateNotionOAuth(ctx, {
|
||||
operation: "authorize",
|
||||
intentId: card.id,
|
||||
...(saved ? { draftId: saved.connectionId } : {}),
|
||||
});
|
||||
state.connectionId = (
|
||||
state.oauthReceipt as NotionOAuthReceipt
|
||||
).connectionId;
|
||||
await ctx.input.page.reload({ waitUntil: "domcontentloaded" });
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const o = await observeExternal(ctx, state),
|
||||
receipt = state.oauthReceipt as NotionOAuthReceipt | undefined;
|
||||
if (receipt) {
|
||||
const connection = await ctx.input.api.get<ToolConnection>(
|
||||
`/api/tool-connections/${receipt.connectionId}`,
|
||||
);
|
||||
const grants = await ctx.input.api.get<ConnectionGrantsResponse>(
|
||||
`/api/tool-connections/${receipt.connectionId}/grants`,
|
||||
);
|
||||
state.samePersonalActive =
|
||||
connection.companyId === ctx.input.fixtures.company.id &&
|
||||
connection.status === "active" &&
|
||||
connection.healthStatus === "ok" &&
|
||||
connection.credentialPolicy === "per_user" &&
|
||||
grants.grants.filter(
|
||||
(g) =>
|
||||
g.kind === "user" &&
|
||||
g.subjectUserId === state.humanId &&
|
||||
g.status === "active",
|
||||
).length === 1 &&
|
||||
!grants.grants.some(
|
||||
(g) => g.kind === "organization" && g.status === "active",
|
||||
);
|
||||
const effective =
|
||||
await ctx.input.api.get<ToolProfileEffectiveSummary>(
|
||||
`${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`,
|
||||
);
|
||||
const catalog = await ctx.input.api.get<{
|
||||
catalog: Array<{ id: string; isReadOnly: boolean }>;
|
||||
}>(`/api/tool-connections/${receipt.connectionId}/catalog`);
|
||||
state.readEntryIds = catalog.catalog
|
||||
.filter(
|
||||
(e) =>
|
||||
e.isReadOnly &&
|
||||
effective.allowedTools.some((t) => t.id === e.id),
|
||||
)
|
||||
.map((e) => e.id);
|
||||
}
|
||||
state.decisions = await Promise.all(
|
||||
(await ctx.allRuns()).map((run) =>
|
||||
ctx.input.api.get<ToolRunDecisionLookup>(
|
||||
`${root(ctx)}/tools/runs/${run.id}/decisions`,
|
||||
),
|
||||
),
|
||||
);
|
||||
state.currentHumanPreserved =
|
||||
(
|
||||
await ctx.input.api.get<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
)
|
||||
).user.id === state.humanId;
|
||||
state.originalSourcePreserved = ctx.issue()?.id === state.sourceIssueId;
|
||||
const current = await readNotionFixturePages(ctx),
|
||||
before = state.pages as NotionPageOracle[];
|
||||
state.pagesUnchanged =
|
||||
current.length === before.length &&
|
||||
current.every(
|
||||
(p, n) =>
|
||||
p.id === before[n]?.id &&
|
||||
p.editedAt === before[n]?.editedAt &&
|
||||
p.marker === before[n]?.marker,
|
||||
);
|
||||
return { ...o, state };
|
||||
},
|
||||
async verifyResult(ctx, state, value) {
|
||||
await verifyRenderedReply(ctx, value, String(state.initialRunId));
|
||||
const o = externalObservation(value)!,
|
||||
page = (state.pages as NotionPageOracle[])[0]!;
|
||||
const href = o.links.find((link) =>
|
||||
notionPageIdentityMatches(link, page.id),
|
||||
);
|
||||
if (!href)
|
||||
throw new Error(
|
||||
"Actual current Notion source link absent from resumed summary",
|
||||
);
|
||||
await expect(
|
||||
ctx.input.page
|
||||
.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`)
|
||||
.first(),
|
||||
).toBeVisible();
|
||||
await ctx.input.evidence("chat-catalog-notion-human-setup.json", {
|
||||
scenarioId: ctx.scenario.id,
|
||||
intentId: state.intentId,
|
||||
sourceIssueId: state.sourceIssueId,
|
||||
initialRunId: state.initialRunId,
|
||||
connectionId: state.connectionId,
|
||||
oauth: state.oauthReceipt,
|
||||
draft: state.savedDraft,
|
||||
currentHumanPreserved: state.currentHumanPreserved,
|
||||
originalSourcePreserved: state.originalSourcePreserved,
|
||||
});
|
||||
},
|
||||
async cleanup(ctx) {
|
||||
for (const row of (
|
||||
await ctx.input.api.get<{ connections: ToolConnection[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
)
|
||||
).connections)
|
||||
await ctx.input.api.delete(`/api/tool-connections/${row.id}`, {
|
||||
allowNotFound: true,
|
||||
});
|
||||
},
|
||||
},
|
||||
),
|
||||
);
|
||||
Reference in new issue
Block a user