mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-06 05:56:19 +01:00
test(e2e): bind Notion choice explanation and actual setup selection
This commit is contained in:
Dotta committed
1 parent
32f35650f0
commit
6cfdd2f6a1
2 files changed
+654
-60
No files matched your search
@@ -1,25 +1,240 @@
|
||||
import { describe,expect,it } from "vitest";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { getConnectableAppDefinition } from "../../../packages/shared/src/app-definitions.js";
|
||||
import { gradeProviderChoice,notionSetupChoices,truthfulNotionChoices } from "./external-provider-choice.js";
|
||||
const reason="Vercel Connect setup is disabled on this Paperclip instance.";
|
||||
function valid(){return {scenarioId:"connection-setup.choose-provider",companyId:"company",leadId:"ceo",reply:"Use Notion sign-in for the easiest connection. Vercel Connect is disabled here; ask the administrator to enable it before choosing that option.",phaseReplies:[],runs:[],connections:[],profiles:[],secrets:[],endpoints:[],interactions:[],activity:[],links:[],humanQueries:[],state:{catalogOfficialOAuth:true,unavailableAdvertised:true,unavailableReason:reason,modelDirectedNotionSetup:true,selectedNotionVisible:true,signInControlVisible:true,originalConversationRetained:true,originalQueryRetained:true}};}
|
||||
describe("current Notion catalog setup choices",()=>{
|
||||
it("derives offered and unavailable choices from real catalog shape without any account",()=>{
|
||||
const gallery={apps:[getConnectableAppDefinition("notion")!],credentialSources:{vercelConnect:{available:false,reason}}};
|
||||
expect(notionSetupChoices(gallery)).toMatchObject({methodKey:"mcp-oauth",serverUrl:"https://mcp.notion.com/mcp",unavailableReason:reason});
|
||||
expect(()=>notionSetupChoices({...gallery,credentialSources:{vercelConnect:{available:true,reason:null}}})).toThrow(/Setup blocker/);
|
||||
expect(()=>notionSetupChoices({...gallery,apps:[]})).toThrow(/Setup blocker/);
|
||||
});
|
||||
it("accepts current user-facing choices and the selected real setup path",()=>expect(gradeProviderChoice(valid()).passed).toBe(true));
|
||||
it("rejects missing prerequisites, wrong setup, lost task and premature connection",()=>{
|
||||
for(const mutate of [
|
||||
(o:ReturnType<typeof valid>)=>{o.reply="Notion sign-in and Vercel Connect both work immediately.";},
|
||||
(o:ReturnType<typeof valid>)=>{o.reply="I've connected Notion. Vercel Connect is disabled; ask an administrator.";},
|
||||
(o:ReturnType<typeof valid>)=>{o.state.modelDirectedNotionSetup=false;},
|
||||
(o:ReturnType<typeof valid>)=>{o.state.originalConversationRetained=false;},
|
||||
(o:ReturnType<typeof valid>)=>{o.state.signInControlVisible=false;},
|
||||
]){const o=valid();mutate(o);expect(gradeProviderChoice(o).passed).toBe(false);}
|
||||
expect(truthfulNotionChoices("Sign in to Notion. Vercel Connect needs workload OIDC configured first.","Vercel Connect needs workload OIDC or PAPERCLIP_VERCEL_CONNECT_ACCESS_TOKEN.")).toBe(true);
|
||||
expect(gradeProviderChoice(undefined).passed).toBe(false);
|
||||
});
|
||||
import {
|
||||
gradeProviderChoice,
|
||||
notionSetupChoices,
|
||||
truthfulNotionChoices,
|
||||
publicChoiceMessages,
|
||||
intendedNotionCard,
|
||||
supportedNotionLink,
|
||||
} from "./external-provider-choice.js";
|
||||
const reason = "Vercel Connect setup is disabled on this Paperclip instance.";
|
||||
function valid() {
|
||||
return {
|
||||
scenarioId: "connection-setup.choose-provider",
|
||||
companyId: "company",
|
||||
leadId: "ceo",
|
||||
reply:
|
||||
"Use Notion sign-in for the easiest connection. Vercel Connect is disabled here; ask the administrator to enable it before choosing that option.",
|
||||
phaseReplies: [],
|
||||
runs: [{ id: "source-run", agentId: "ceo", status: "succeeded" }],
|
||||
connections: [],
|
||||
profiles: [],
|
||||
secrets: [],
|
||||
endpoints: [],
|
||||
interactions: [],
|
||||
activity: [],
|
||||
links: [],
|
||||
humanQueries: [],
|
||||
state: {
|
||||
advertisedChoices: { service: "notion" },
|
||||
catalogChoicesIndependentlyMatched: true,
|
||||
choiceExplanation: {
|
||||
runId: "source-run",
|
||||
eventId: "message",
|
||||
body: "Use Notion sign-in. Vercel Connect is disabled; ask an administrator to enable it.",
|
||||
},
|
||||
choiceExplanationVisible: true,
|
||||
selectionLane: "inline-card",
|
||||
selectionAttributionMatched: true,
|
||||
catalogOfficialOAuth: true,
|
||||
unavailableAdvertised: true,
|
||||
unavailableReason: reason,
|
||||
modelDirectedNotionSetup: true,
|
||||
selectedNotionVisible: true,
|
||||
signInControlVisible: true,
|
||||
originalConversationRetained: true,
|
||||
originalQueryRetained: true,
|
||||
},
|
||||
};
|
||||
}
|
||||
describe("current Notion catalog setup choices", () => {
|
||||
it("derives offered and unavailable choices from real catalog shape without any account", () => {
|
||||
const gallery = {
|
||||
apps: [getConnectableAppDefinition("notion")!],
|
||||
credentialSources: { vercelConnect: { available: false, reason } },
|
||||
};
|
||||
expect(notionSetupChoices(gallery)).toMatchObject({
|
||||
methodKey: "mcp-oauth",
|
||||
serverUrl: "https://mcp.notion.com/mcp",
|
||||
unavailableReason: reason,
|
||||
});
|
||||
expect(() =>
|
||||
notionSetupChoices({
|
||||
...gallery,
|
||||
credentialSources: { vercelConnect: { available: true, reason: null } },
|
||||
}),
|
||||
).toThrow(/Setup blocker/);
|
||||
expect(() => notionSetupChoices({ ...gallery, apps: [] })).toThrow(
|
||||
/Setup blocker/,
|
||||
);
|
||||
});
|
||||
it("accepts current user-facing choices and the selected real setup path", () =>
|
||||
expect(gradeProviderChoice(valid()).passed).toBe(true));
|
||||
it("rejects missing prerequisites, wrong setup, lost task and premature connection", () => {
|
||||
for (const mutate of [
|
||||
(o: ReturnType<typeof valid>) => {
|
||||
o.state.choiceExplanation.body =
|
||||
"Notion sign-in and Vercel Connect both work immediately.";
|
||||
},
|
||||
(o: ReturnType<typeof valid>) => {
|
||||
o.state.choiceExplanation.body =
|
||||
"I've connected Notion. Vercel Connect is disabled; ask an administrator.";
|
||||
},
|
||||
(o: ReturnType<typeof valid>) => {
|
||||
o.state.modelDirectedNotionSetup = false;
|
||||
},
|
||||
(o: ReturnType<typeof valid>) => {
|
||||
o.state.originalConversationRetained = false;
|
||||
},
|
||||
(o: ReturnType<typeof valid>) => {
|
||||
o.state.signInControlVisible = false;
|
||||
},
|
||||
]) {
|
||||
const o = valid();
|
||||
mutate(o);
|
||||
expect(gradeProviderChoice(o).passed).toBe(false);
|
||||
}
|
||||
expect(
|
||||
truthfulNotionChoices(
|
||||
"Sign in to Notion. Vercel Connect needs workload OIDC configured first.",
|
||||
"Vercel Connect needs workload OIDC or PAPERCLIP_VERCEL_CONNECT_ACCESS_TOKEN.",
|
||||
),
|
||||
).toBe(true);
|
||||
expect(gradeProviderChoice(undefined).passed).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js";
|
||||
describe("actual source attribution and supported choice lanes", () => {
|
||||
it("accepts a complete explanation with either actual selection lane; bare original card remains nonpassing", () => {
|
||||
for (const lane of ["inline-card", "method-link"]) {
|
||||
const o = valid();
|
||||
o.state.selectionLane = lane;
|
||||
expect(gradeProviderChoice(o).passed).toBe(true);
|
||||
}
|
||||
const original = valid();
|
||||
original.reply = "";
|
||||
original.state.choiceExplanation.body =
|
||||
"I’ll check the available Notion connection flow and give you the shortest setup path.";
|
||||
expect(gradeProviderChoice(original).passed).toBe(false);
|
||||
});
|
||||
it("rejects lost catalog, invisible explanation, wrong source run and unbound selection", () => {
|
||||
for (const change of [
|
||||
{ catalogChoicesIndependentlyMatched: false },
|
||||
{ choiceExplanationVisible: false },
|
||||
{ selectionAttributionMatched: false },
|
||||
{
|
||||
choiceExplanation: {
|
||||
body: valid().state.choiceExplanation.body,
|
||||
runId: "foreign",
|
||||
eventId: "message",
|
||||
},
|
||||
},
|
||||
]) {
|
||||
const o = valid();
|
||||
Object.assign(o.state, change);
|
||||
expect(gradeProviderChoice(o).passed).toBe(false);
|
||||
}
|
||||
});
|
||||
it("accepts only completed public messages from the exact source company/agent/run", () => {
|
||||
const e = {
|
||||
seq: 1,
|
||||
eventType: "item.completed",
|
||||
runId: "run",
|
||||
companyId: "company",
|
||||
agentId: "ceo",
|
||||
payload: {
|
||||
prpEvent: {
|
||||
runId: "run",
|
||||
payload: {
|
||||
item: {
|
||||
type: "agentMessage",
|
||||
phase: "commentary",
|
||||
status: "completed",
|
||||
id: "msg",
|
||||
text: "Public setup explanation",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
const target = { runId: "run", companyId: "company", agentId: "ceo" };
|
||||
expect(publicChoiceMessages([e], target)).toEqual([
|
||||
{ runId: "run", eventId: "msg", body: "Public setup explanation" },
|
||||
]);
|
||||
for (const delta of [
|
||||
{ runId: "foreign" },
|
||||
{ agentId: "other" },
|
||||
{ companyId: "foreign" },
|
||||
{ eventType: "item.started" },
|
||||
])
|
||||
expect(publicChoiceMessages([{ ...e, ...delta }], target)).toEqual([]);
|
||||
const hidden = structuredClone(e);
|
||||
hidden.payload.prpEvent.payload.item.phase = "analysis";
|
||||
expect(publicChoiceMessages([hidden], target)).toEqual([]);
|
||||
});
|
||||
it("binds the inline request to the actual person, task, native run and Notion service", () => {
|
||||
const c = {
|
||||
id: "intent",
|
||||
kind: "connection_intent",
|
||||
status: "pending",
|
||||
companyId: "company",
|
||||
issueId: "issue",
|
||||
createdByAgentId: "ceo",
|
||||
addresseeUserId: "human",
|
||||
sourceRunId: "run",
|
||||
payload: { serviceSlug: "notion" },
|
||||
} as ConnectionIntentInteraction,
|
||||
target = {
|
||||
companyId: "company",
|
||||
issueId: "issue",
|
||||
agentId: "ceo",
|
||||
humanId: "human",
|
||||
runIds: ["run"],
|
||||
};
|
||||
expect(intendedNotionCard([c], target)?.id).toBe("intent");
|
||||
for (const delta of [
|
||||
{ companyId: "foreign" },
|
||||
{ issueId: "other" },
|
||||
{ createdByAgentId: "other" },
|
||||
{ addresseeUserId: "other" },
|
||||
{ sourceRunId: "other" },
|
||||
{ status: "answered" },
|
||||
{ payload: { serviceSlug: "slack" } },
|
||||
])
|
||||
expect(
|
||||
intendedNotionCard(
|
||||
[{ ...c, ...delta } as ConnectionIntentInteraction],
|
||||
target,
|
||||
),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
intendedNotionCard([c, { ...c, id: "duplicate" }], target),
|
||||
).toBeUndefined();
|
||||
});
|
||||
it("allows actual same-company selected method links and rejects wrong method/company/provider", () => {
|
||||
expect(
|
||||
supportedNotionLink(
|
||||
"/ACORN/apps/connect?source=notion&method=mcp-oauth",
|
||||
"http://fixture.invalid",
|
||||
"ACORN",
|
||||
"mcp-oauth",
|
||||
),
|
||||
).toBe(true);
|
||||
for (const link of [
|
||||
"https://other.invalid/ACORN/apps/connect?source=notion",
|
||||
"/OTHER/apps/connect?source=notion",
|
||||
"/ACORN/apps/connect?source=slack",
|
||||
"/ACORN/apps/connect?source=notion&method=unrelated",
|
||||
])
|
||||
expect(
|
||||
supportedNotionLink(
|
||||
link,
|
||||
"http://fixture.invalid",
|
||||
"ACORN",
|
||||
"mcp-oauth",
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,46 +1,425 @@
|
||||
import { expect } from "@playwright/test";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import type { AppDefinition } from "../../../packages/shared/src/types/app-definition.js";
|
||||
import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js";
|
||||
import { getAvailableConnectionMethods } from "../../../packages/shared/src/app-definitions.js";
|
||||
import { definition, root, checks, externalObservation, observeExternal, verifyRenderedReply } from "./external-support.js";
|
||||
import {
|
||||
definition,
|
||||
root,
|
||||
checks,
|
||||
externalObservation,
|
||||
observeExternal,
|
||||
} from "./external-support.js";
|
||||
import { waitForReply } from "./identity-support.js";
|
||||
|
||||
interface Gallery { apps: AppDefinition[]; credentialSources: { vercelConnect: { available: boolean; reason: string | null } } }
|
||||
import { collectRunEvents } from "../run-observations.js";
|
||||
import type { ChatCatalogContext } from "../chat-catalog-contract.js";
|
||||
interface Gallery {
|
||||
apps: AppDefinition[];
|
||||
credentialSources: {
|
||||
vercelConnect: { available: boolean; reason: string | null };
|
||||
};
|
||||
}
|
||||
export function notionSetupChoices(gallery: Gallery) {
|
||||
const notion=gallery.apps.find(app=>app.slug==="notion");
|
||||
const methods=notion ? getAvailableConnectionMethods(notion) : [];
|
||||
const oauth=methods.find(method=>method.auth==="oauth"&&method.transport==="mcp_remote"&&method.defaults?.serverUrl==="https://mcp.notion.com/mcp");
|
||||
const managed=gallery.credentialSources.vercelConnect;
|
||||
if(!oauth||!oauth.credentialSources?.vercelConnect||managed.available||!managed.reason)throw new Error("Setup blocker: the actual Notion catalog must offer official sign-in and an explicitly unavailable Vercel Connect option with its prerequisite");
|
||||
return {methodKey:oauth.key,serverUrl:oauth.defaults!.serverUrl!,unavailableReason:managed.reason};
|
||||
const notion = gallery.apps.find((app) => app.slug === "notion"),
|
||||
methods = notion ? getAvailableConnectionMethods(notion) : [];
|
||||
const oauth = methods.find(
|
||||
(method) =>
|
||||
method.auth === "oauth" &&
|
||||
method.transport === "mcp_remote" &&
|
||||
method.defaults?.serverUrl === "https://mcp.notion.com/mcp",
|
||||
);
|
||||
const managed = gallery.credentialSources.vercelConnect;
|
||||
if (
|
||||
!oauth?.credentialSources?.vercelConnect?.services.includes("notion") ||
|
||||
managed.available ||
|
||||
!managed.reason
|
||||
)
|
||||
throw new Error(
|
||||
"Setup blocker: actual reviewed Notion catalog must offer official sign-in and supported unavailable Vercel Connect with its prerequisite",
|
||||
);
|
||||
return {
|
||||
methodKey: oauth.key,
|
||||
serverUrl: oauth.defaults!.serverUrl!,
|
||||
unavailableReason: managed.reason,
|
||||
advertisedChoices: {
|
||||
service: "notion",
|
||||
methodKey: oauth.key,
|
||||
auth: oauth.auth,
|
||||
transport: oauth.transport,
|
||||
whenToUse: oauth.whenToUse,
|
||||
serverUrl: oauth.defaults!.serverUrl!,
|
||||
alternative: {
|
||||
source: "vercel_connect",
|
||||
available: managed.available,
|
||||
reason: managed.reason,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
export function truthfulNotionChoices(reply:string,reason:string){
|
||||
const offered=/notion/i.test(reply)&&/sign.?in|log.?in|authoriz|connect.*account/i.test(reply);
|
||||
const prerequisite=/vercel\s+connect/i.test(reply)&&(/disabled/i.test(reason)?/disabled|enable|administrator|admin/i.test(reply):/oidc|workload|token|configur/i.test(reply));
|
||||
return offered&&prerequisite&&!/\b(?:I(?:['’]ve| have)?|we(?:['’]ve| have)?)\s+(?:already\s+)?connected\b|\bnotion\s+is\s+(?:now\s+)?connected\b/i.test(reply);
|
||||
export function truthfulNotionChoices(reply: string, reason: string) {
|
||||
const offered =
|
||||
/notion/i.test(reply) &&
|
||||
/sign.?in|log.?in|authoriz|connect.*account/i.test(reply);
|
||||
const prerequisite =
|
||||
/vercel\s+connect/i.test(reply) &&
|
||||
(/disabled/i.test(reason)
|
||||
? /disabled|enable|administrator|admin/i.test(reply)
|
||||
: /oidc|workload|token|configur/i.test(reply));
|
||||
return (
|
||||
offered &&
|
||||
prerequisite &&
|
||||
!/\b(?:I(?:['’]ve| have)?|we(?:['’]ve| have)?)\s+(?:already\s+)?connected\b|\bnotion\s+is\s+(?:now\s+)?connected\b/i.test(
|
||||
reply,
|
||||
)
|
||||
);
|
||||
}
|
||||
export function gradeProviderChoice(value:unknown){
|
||||
const c=checks(),o=externalObservation(value);c.add("complete-catalog-choice-evidence",!!o,"Actual public catalog methods, the advertised unavailable prerequisite and the directed setup UI are required.");if(!o)return c.result();
|
||||
c.add("real-catalog-and-truthful-choices",o.state.catalogOfficialOAuth===true&&o.state.unavailableAdvertised===true&&typeof o.state.unavailableReason==="string"&&truthfulNotionChoices(o.reply,o.state.unavailableReason),"The reply explains the actually offered sign-in choice and the prerequisite for unavailable Vercel Connect, without claiming setup is complete.");
|
||||
c.add("directed-official-setup-path",o.state.modelDirectedNotionSetup===true&&o.state.selectedNotionVisible===true&&o.state.signInControlVisible===true,"The person follows the actual model's Notion setup link into the supported selected-provider flow; the oracle does not substitute a different protocol.");
|
||||
c.add("original-intent-retained-without-phantom-account",o.state.originalConversationRetained===true&&o.state.originalQueryRetained===true&&o.connections.length===0,"Entering setup retains the original task and exact human intent, and does not make an unfinished choice a usable connection.");return c.result();
|
||||
const record = (v: unknown): Record<string, unknown> =>
|
||||
v && typeof v === "object" && !Array.isArray(v)
|
||||
? (v as Record<string, unknown>)
|
||||
: {};
|
||||
interface MessageEvent {
|
||||
seq?: number;
|
||||
runId?: string;
|
||||
companyId?: string;
|
||||
agentId?: string;
|
||||
eventType?: string;
|
||||
payload?: unknown;
|
||||
}
|
||||
export const providerChoiceExternalCases=[definition("connection-setup.choose-provider",async ctx=>{
|
||||
const gallery=await ctx.input.api.get<Gallery>(`${root(ctx)}/tools/gallery`),choices=notionSetupChoices(gallery);
|
||||
const before=await ctx.input.api.get<{connections:unknown[]}>(`${root(ctx)}/tools/connections`);if(before.connections.length)throw new Error("Catalog choice requires a disposable company without an existing Notion account");
|
||||
return {...choices,catalogOfficialOAuth:true,unavailableAdvertised:true};
|
||||
},gradeProviderChoice,{
|
||||
authentication:"human-session",
|
||||
sourceFiles:["chat-catalog-domains/external-provider-choice.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","../../packages/shared/src/app-definitions.ts","../../packages/shared/src/app-definitions/notion.json","chat-scenarios/connection-setup.yaml"],
|
||||
async act(ctx,state){
|
||||
await waitForReply(ctx);const o=await observeExternal(ctx,state);if(!truthfulNotionChoices(o.reply,String(state.unavailableReason)))throw new Error("Actual reply does not explain the advertised Notion choices and unavailable prerequisite");
|
||||
const href=o.links.find(link=>{try{const url=new URL(link,o.origin);return url.origin===o.origin&&(url.pathname===`/${ctx.input.fixtures.company.issuePrefix}/apps/connect`&&[url.searchParams.get("source"),url.searchParams.get("appKey")].includes("notion"));}catch{return false;}});
|
||||
if(!href)throw new Error("Actual reply must direct the person to the selected supported Notion setup surface");state.modelDirectedNotionSetup=true;
|
||||
const source=ctx.input.page.url(),issueId=ctx.issue()!.id;await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click();
|
||||
await expect(ctx.input.page.getByRole("main")).toContainText(/Notion/);state.selectedNotionVisible=true;
|
||||
await expect(ctx.input.page.getByRole("button",{name:/Continue to (?:Notion|sign in)|Sign in to Notion/}).first()).toBeVisible();state.signInControlVisible=true;
|
||||
// This scenario selects a method; no provider authorization is supplied or invented.
|
||||
await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"});await ctx.refreshIssue();state.originalConversationRetained=ctx.issue()?.id===issueId;
|
||||
const comments=await ctx.input.api.get<Array<{body:string;authorUserId?:string}>>(`/api/issues/${issueId}/comments?order=asc`);state.originalQueryRetained=comments.some(comment=>!!comment.authorUserId&&comment.body===ctx.scenario.query);
|
||||
export function publicChoiceMessages(
|
||||
events: MessageEvent[],
|
||||
expected: { companyId: string; agentId: string; runId: string },
|
||||
) {
|
||||
return events.flatMap((event) => {
|
||||
const prp = record(record(event.payload).prpEvent),
|
||||
payload = record(prp.payload),
|
||||
item = record(payload.item);
|
||||
return event.companyId === expected.companyId &&
|
||||
event.agentId === expected.agentId &&
|
||||
event.runId === expected.runId &&
|
||||
prp.runId === expected.runId &&
|
||||
event.eventType === "item.completed" &&
|
||||
item.type === "agentMessage" &&
|
||||
item.status === "completed" &&
|
||||
["commentary", "final_answer"].includes(String(item.phase)) &&
|
||||
typeof item.id === "string" &&
|
||||
typeof item.text === "string" &&
|
||||
item.text.length <= 12000
|
||||
? [{ runId: expected.runId, eventId: item.id, body: item.text }]
|
||||
: [];
|
||||
});
|
||||
}
|
||||
export function intendedNotionCard(
|
||||
cards: ConnectionIntentInteraction[],
|
||||
expected: {
|
||||
companyId: string;
|
||||
issueId: string;
|
||||
agentId: string;
|
||||
humanId: string;
|
||||
runIds: string[];
|
||||
},
|
||||
async verifyResult(ctx,_state,value){await verifyRenderedReply(ctx,value);},
|
||||
})];
|
||||
) {
|
||||
const matches = cards.filter(
|
||||
(c) =>
|
||||
c.kind === "connection_intent" &&
|
||||
c.status === "pending" &&
|
||||
c.companyId === expected.companyId &&
|
||||
c.issueId === expected.issueId &&
|
||||
c.createdByAgentId === expected.agentId &&
|
||||
c.addresseeUserId === expected.humanId &&
|
||||
c.payload.serviceSlug === "notion" &&
|
||||
!!c.sourceRunId &&
|
||||
expected.runIds.includes(c.sourceRunId),
|
||||
);
|
||||
return matches.length === 1 ? matches[0] : undefined;
|
||||
}
|
||||
export function supportedNotionLink(
|
||||
link: string,
|
||||
origin: string,
|
||||
prefix: string,
|
||||
method: string,
|
||||
) {
|
||||
try {
|
||||
const u = new URL(link, origin);
|
||||
return (
|
||||
u.origin === origin &&
|
||||
u.pathname === `/${prefix}/apps/connect` &&
|
||||
[u.searchParams.get("source"), u.searchParams.get("appKey")].includes(
|
||||
"notion",
|
||||
) &&
|
||||
(!u.searchParams.has("method") || u.searchParams.get("method") === method)
|
||||
);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
const visibleText = (s: string) =>
|
||||
s
|
||||
.replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
|
||||
.replace(/[*`_]/g, "")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
async function observeChoiceExplanation(
|
||||
ctx: ChatCatalogContext,
|
||||
state: Record<string, unknown>,
|
||||
) {
|
||||
const runs = (await ctx.allRuns()).filter(
|
||||
(r) =>
|
||||
r.agentId === ctx.input.fixtures.agent.id && r.status === "succeeded",
|
||||
),
|
||||
messages = [];
|
||||
for (const run of runs) {
|
||||
const events = await collectRunEvents<MessageEvent>((afterSeq, limit) =>
|
||||
ctx.input.api.getRunObservation(
|
||||
`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`,
|
||||
),
|
||||
);
|
||||
messages.push(
|
||||
...publicChoiceMessages(events, {
|
||||
companyId: ctx.input.fixtures.company.id,
|
||||
agentId: ctx.input.fixtures.agent.id,
|
||||
runId: run.id,
|
||||
}),
|
||||
);
|
||||
}
|
||||
const selected = messages.find(
|
||||
(m) =>
|
||||
truthfulNotionChoices(m.body, String(state.unavailableReason)) &&
|
||||
m.body.trim(),
|
||||
);
|
||||
if (selected) {
|
||||
await expect(
|
||||
ctx.input.page
|
||||
.getByText(visibleText(selected.body), { exact: true })
|
||||
.first(),
|
||||
).toBeVisible();
|
||||
state.choiceExplanation = selected;
|
||||
state.choiceExplanationVisible = true;
|
||||
}
|
||||
}
|
||||
export function gradeProviderChoice(value: unknown) {
|
||||
const c = checks(),
|
||||
o = externalObservation(value);
|
||||
c.add(
|
||||
"complete-catalog-choice-evidence",
|
||||
!!o,
|
||||
"Actual public catalog choices, prerequisite and attributed human setup selection are required.",
|
||||
);
|
||||
if (!o) return c.result();
|
||||
const explanation = record(o.state.choiceExplanation),
|
||||
body = typeof explanation.body === "string" ? explanation.body : o.reply;
|
||||
c.add(
|
||||
"real-catalog-and-truthful-choices",
|
||||
o.state.catalogChoicesIndependentlyMatched === true &&
|
||||
!!o.state.advertisedChoices &&
|
||||
typeof o.state.unavailableReason === "string" &&
|
||||
truthfulNotionChoices(body, o.state.unavailableReason) &&
|
||||
o.state.choiceExplanationVisible === true &&
|
||||
o.runs.some(
|
||||
(r) =>
|
||||
r.id === explanation.runId &&
|
||||
r.agentId === o.leadId &&
|
||||
r.status === "succeeded",
|
||||
),
|
||||
"A real visible model message from the source native run explains the actual reviewed choices and unavailable prerequisite; a bare setup card cannot replace that answer.",
|
||||
);
|
||||
c.add(
|
||||
"directed-official-setup-path",
|
||||
o.state.modelDirectedNotionSetup === true &&
|
||||
["inline-card", "method-link"].includes(String(o.state.selectionLane)) &&
|
||||
o.state.selectionAttributionMatched === true &&
|
||||
o.state.selectedNotionVisible === true &&
|
||||
o.state.signInControlVisible === true,
|
||||
"The person selects the attributed real inline request or supported actual method link, then sees the selected Notion setup without protocol substitution.",
|
||||
);
|
||||
c.add(
|
||||
"original-intent-retained-without-phantom-account",
|
||||
o.state.originalConversationRetained === true &&
|
||||
o.state.originalQueryRetained === true &&
|
||||
o.connections.length === 0,
|
||||
"The original task and exact query remain intact; no authorization or usable account is invented.",
|
||||
);
|
||||
return c.result();
|
||||
}
|
||||
export const providerChoiceExternalCases = [
|
||||
definition(
|
||||
"connection-setup.choose-provider",
|
||||
async (ctx) => {
|
||||
const choices = notionSetupChoices(
|
||||
await ctx.input.api.get<Gallery>(`${root(ctx)}/tools/gallery`),
|
||||
);
|
||||
const before = await ctx.input.api.get<{ connections: unknown[] }>(
|
||||
`${root(ctx)}/tools/connections`,
|
||||
);
|
||||
if (before.connections.length)
|
||||
throw new Error(
|
||||
"Catalog choice requires no existing account in the disposable company",
|
||||
);
|
||||
return {
|
||||
...choices,
|
||||
humanId: (
|
||||
await ctx.input.api.get<{ user: { id: string } }>(
|
||||
"/api/auth/get-session",
|
||||
)
|
||||
).user.id,
|
||||
};
|
||||
},
|
||||
gradeProviderChoice,
|
||||
{
|
||||
authentication: "human-session",
|
||||
sourceFiles: [
|
||||
"chat-catalog-domains/external-provider-choice.ts",
|
||||
"run-observations.ts",
|
||||
"chat-catalog-domains/external-support.ts",
|
||||
"chat-catalog-domains/internal-support.ts",
|
||||
"chat-catalog-domains/identity-support.ts",
|
||||
"../../packages/shared/src/app-definitions.ts",
|
||||
"../../packages/shared/src/app-definitions/notion.json",
|
||||
"../../ui/src/features/connections/ConnectionIntentInteractionBody.tsx",
|
||||
"../../ui/src/features/connections/ConnectionSetupFlow.tsx",
|
||||
"chat-scenarios/connection-setup.yaml",
|
||||
],
|
||||
async verifySetup(ctx, state) {
|
||||
const actual = notionSetupChoices(
|
||||
await ctx.input.api.get<Gallery>(`${root(ctx)}/tools/gallery`),
|
||||
),
|
||||
c = checks();
|
||||
c.add(
|
||||
"actual-reviewed-advertised-choices",
|
||||
isDeepStrictEqual(actual.advertisedChoices, state.advertisedChoices),
|
||||
"Independent public gallery retains reviewed Notion method identity and actual unsupported credential-source prerequisite.",
|
||||
);
|
||||
return c.result();
|
||||
},
|
||||
async act(ctx, state) {
|
||||
await waitForReply(ctx);
|
||||
await observeChoiceExplanation(ctx, state);
|
||||
if (!state.choiceExplanationVisible)
|
||||
throw new Error(
|
||||
"Actual visible source-run answer does not explain advertised Notion choices and unavailable prerequisite",
|
||||
);
|
||||
const o = await observeExternal(ctx, state),
|
||||
source = ctx.input.page.url(),
|
||||
issueId = ctx.issue()!.id,
|
||||
runs = await ctx.allRuns();
|
||||
const cards = await ctx.input.api.get<ConnectionIntentInteraction[]>(
|
||||
`/api/issues/${issueId}/interactions`,
|
||||
),
|
||||
card = intendedNotionCard(cards, {
|
||||
companyId: o.companyId,
|
||||
issueId,
|
||||
agentId: o.leadId,
|
||||
humanId: String(state.humanId),
|
||||
runIds: runs
|
||||
.filter((r) => r.status === "succeeded")
|
||||
.map((r) => r.id),
|
||||
});
|
||||
if (card) {
|
||||
state.selectionLane = "inline-card";
|
||||
state.selectedIntentId = card.id;
|
||||
await ctx.input.page
|
||||
.locator(`[id=${JSON.stringify(`interaction-${card.id}`)}]`)
|
||||
.getByRole("button", { name: "Connect", exact: true })
|
||||
.click();
|
||||
await expect(ctx.input.page.getByRole("dialog")).toContainText(
|
||||
"Notion",
|
||||
);
|
||||
} else {
|
||||
const href = o.links.find((link) =>
|
||||
supportedNotionLink(
|
||||
link,
|
||||
o.origin,
|
||||
String(ctx.input.fixtures.company.issuePrefix),
|
||||
String(state.methodKey),
|
||||
),
|
||||
);
|
||||
if (!href)
|
||||
throw new Error(
|
||||
"No attributed inline request or actual supported Notion method link",
|
||||
);
|
||||
state.selectionLane = "method-link";
|
||||
state.selectedMethodHref = href;
|
||||
await ctx.input.page
|
||||
.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`)
|
||||
.first()
|
||||
.click();
|
||||
await expect(ctx.input.page.getByRole("main")).toContainText(
|
||||
"Notion",
|
||||
);
|
||||
}
|
||||
state.modelDirectedNotionSetup = true;
|
||||
state.selectedNotionVisible = true;
|
||||
const surface =
|
||||
state.selectionLane === "inline-card"
|
||||
? ctx.input.page.getByRole("dialog")
|
||||
: ctx.input.page.getByRole("main");
|
||||
await expect(
|
||||
surface
|
||||
.getByRole("button", {
|
||||
name: /Continue to (?:Notion|sign in)|Sign in to Notion/,
|
||||
})
|
||||
.first(),
|
||||
).toBeVisible();
|
||||
state.signInControlVisible = true;
|
||||
if (state.selectionLane === "inline-card")
|
||||
await surface
|
||||
.getByRole("button", { name: "Close", exact: true })
|
||||
.click();
|
||||
else
|
||||
await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
|
||||
await ctx.refreshIssue();
|
||||
state.originalConversationRetained = ctx.issue()?.id === issueId;
|
||||
const comments = await ctx.input.api.get<
|
||||
Array<{ body: string; authorUserId?: string }>
|
||||
>(`/api/issues/${issueId}/comments?order=asc`);
|
||||
state.originalQueryRetained = comments.some(
|
||||
(comment) =>
|
||||
comment.authorUserId === state.humanId &&
|
||||
comment.body === ctx.scenario.query,
|
||||
);
|
||||
},
|
||||
async observe(ctx, state) {
|
||||
const actual = notionSetupChoices(
|
||||
await ctx.input.api.get<Gallery>(`${root(ctx)}/tools/gallery`),
|
||||
);
|
||||
state.catalogChoicesIndependentlyMatched = isDeepStrictEqual(
|
||||
actual.advertisedChoices,
|
||||
state.advertisedChoices,
|
||||
);
|
||||
const o = await observeExternal(ctx, state);
|
||||
state.selectionAttributionMatched =
|
||||
state.selectionLane === "inline-card"
|
||||
? intendedNotionCard(
|
||||
o.interactions as unknown as ConnectionIntentInteraction[],
|
||||
{
|
||||
companyId: o.companyId,
|
||||
issueId: ctx.issue()!.id,
|
||||
agentId: o.leadId,
|
||||
humanId: String(state.humanId),
|
||||
runIds: o.runs
|
||||
.filter((r) => r.status === "succeeded")
|
||||
.map((r) => r.id),
|
||||
},
|
||||
)?.id === state.selectedIntentId
|
||||
: typeof state.selectedMethodHref === "string" &&
|
||||
o.links.includes(state.selectedMethodHref) &&
|
||||
supportedNotionLink(
|
||||
state.selectedMethodHref,
|
||||
o.origin,
|
||||
String(ctx.input.fixtures.company.issuePrefix),
|
||||
String(state.methodKey),
|
||||
);
|
||||
return { ...o, state };
|
||||
},
|
||||
async verifyResult(ctx, state) {
|
||||
const explanation = record(state.choiceExplanation);
|
||||
if (typeof explanation.body !== "string")
|
||||
throw new Error("Missing actual source-run choice explanation");
|
||||
await expect(
|
||||
ctx.input.page
|
||||
.getByText(visibleText(explanation.body), { exact: true })
|
||||
.first(),
|
||||
).toBeVisible();
|
||||
},
|
||||
},
|
||||
),
|
||||
];
|
||||
Reference in new issue
Block a user