diff --git a/tests/runner-e2e/chat-catalog-domains/external-provider-choice.test.ts b/tests/runner-e2e/chat-catalog-domains/external-provider-choice.test.ts index c62e1615c9f..016489d2a36 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-provider-choice.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-provider-choice.test.ts @@ -1,25 +1,240 @@ -import { describe,expect,it } from "vitest"; +import { describe, expect, it } from "vitest"; import { getConnectableAppDefinition } from "../../../packages/shared/src/app-definitions.js"; -import { gradeProviderChoice,notionSetupChoices,truthfulNotionChoices } from "./external-provider-choice.js"; -const reason="Vercel Connect setup is disabled on this Paperclip instance."; -function valid(){return {scenarioId:"connection-setup.choose-provider",companyId:"company",leadId:"ceo",reply:"Use Notion sign-in for the easiest connection. Vercel Connect is disabled here; ask the administrator to enable it before choosing that option.",phaseReplies:[],runs:[],connections:[],profiles:[],secrets:[],endpoints:[],interactions:[],activity:[],links:[],humanQueries:[],state:{catalogOfficialOAuth:true,unavailableAdvertised:true,unavailableReason:reason,modelDirectedNotionSetup:true,selectedNotionVisible:true,signInControlVisible:true,originalConversationRetained:true,originalQueryRetained:true}};} -describe("current Notion catalog setup choices",()=>{ - it("derives offered and unavailable choices from real catalog shape without any account",()=>{ - const gallery={apps:[getConnectableAppDefinition("notion")!],credentialSources:{vercelConnect:{available:false,reason}}}; - expect(notionSetupChoices(gallery)).toMatchObject({methodKey:"mcp-oauth",serverUrl:"https://mcp.notion.com/mcp",unavailableReason:reason}); - expect(()=>notionSetupChoices({...gallery,credentialSources:{vercelConnect:{available:true,reason:null}}})).toThrow(/Setup blocker/); - expect(()=>notionSetupChoices({...gallery,apps:[]})).toThrow(/Setup blocker/); - }); - it("accepts current user-facing choices and the selected real setup path",()=>expect(gradeProviderChoice(valid()).passed).toBe(true)); - it("rejects missing prerequisites, wrong setup, lost task and premature connection",()=>{ - for(const mutate of [ - (o:ReturnType)=>{o.reply="Notion sign-in and Vercel Connect both work immediately.";}, - (o:ReturnType)=>{o.reply="I've connected Notion. Vercel Connect is disabled; ask an administrator.";}, - (o:ReturnType)=>{o.state.modelDirectedNotionSetup=false;}, - (o:ReturnType)=>{o.state.originalConversationRetained=false;}, - (o:ReturnType)=>{o.state.signInControlVisible=false;}, - ]){const o=valid();mutate(o);expect(gradeProviderChoice(o).passed).toBe(false);} - expect(truthfulNotionChoices("Sign in to Notion. Vercel Connect needs workload OIDC configured first.","Vercel Connect needs workload OIDC or PAPERCLIP_VERCEL_CONNECT_ACCESS_TOKEN.")).toBe(true); - expect(gradeProviderChoice(undefined).passed).toBe(false); - }); +import { + gradeProviderChoice, + notionSetupChoices, + truthfulNotionChoices, + publicChoiceMessages, + intendedNotionCard, + supportedNotionLink, +} from "./external-provider-choice.js"; +const reason = "Vercel Connect setup is disabled on this Paperclip instance."; +function valid() { + return { + scenarioId: "connection-setup.choose-provider", + companyId: "company", + leadId: "ceo", + reply: + "Use Notion sign-in for the easiest connection. Vercel Connect is disabled here; ask the administrator to enable it before choosing that option.", + phaseReplies: [], + runs: [{ id: "source-run", agentId: "ceo", status: "succeeded" }], + connections: [], + profiles: [], + secrets: [], + endpoints: [], + interactions: [], + activity: [], + links: [], + humanQueries: [], + state: { + advertisedChoices: { service: "notion" }, + catalogChoicesIndependentlyMatched: true, + choiceExplanation: { + runId: "source-run", + eventId: "message", + body: "Use Notion sign-in. Vercel Connect is disabled; ask an administrator to enable it.", + }, + choiceExplanationVisible: true, + selectionLane: "inline-card", + selectionAttributionMatched: true, + catalogOfficialOAuth: true, + unavailableAdvertised: true, + unavailableReason: reason, + modelDirectedNotionSetup: true, + selectedNotionVisible: true, + signInControlVisible: true, + originalConversationRetained: true, + originalQueryRetained: true, + }, + }; +} +describe("current Notion catalog setup choices", () => { + it("derives offered and unavailable choices from real catalog shape without any account", () => { + const gallery = { + apps: [getConnectableAppDefinition("notion")!], + credentialSources: { vercelConnect: { available: false, reason } }, + }; + expect(notionSetupChoices(gallery)).toMatchObject({ + methodKey: "mcp-oauth", + serverUrl: "https://mcp.notion.com/mcp", + unavailableReason: reason, + }); + expect(() => + notionSetupChoices({ + ...gallery, + credentialSources: { vercelConnect: { available: true, reason: null } }, + }), + ).toThrow(/Setup blocker/); + expect(() => notionSetupChoices({ ...gallery, apps: [] })).toThrow( + /Setup blocker/, + ); + }); + it("accepts current user-facing choices and the selected real setup path", () => + expect(gradeProviderChoice(valid()).passed).toBe(true)); + it("rejects missing prerequisites, wrong setup, lost task and premature connection", () => { + for (const mutate of [ + (o: ReturnType) => { + o.state.choiceExplanation.body = + "Notion sign-in and Vercel Connect both work immediately."; + }, + (o: ReturnType) => { + o.state.choiceExplanation.body = + "I've connected Notion. Vercel Connect is disabled; ask an administrator."; + }, + (o: ReturnType) => { + o.state.modelDirectedNotionSetup = false; + }, + (o: ReturnType) => { + o.state.originalConversationRetained = false; + }, + (o: ReturnType) => { + o.state.signInControlVisible = false; + }, + ]) { + const o = valid(); + mutate(o); + expect(gradeProviderChoice(o).passed).toBe(false); + } + expect( + truthfulNotionChoices( + "Sign in to Notion. Vercel Connect needs workload OIDC configured first.", + "Vercel Connect needs workload OIDC or PAPERCLIP_VERCEL_CONNECT_ACCESS_TOKEN.", + ), + ).toBe(true); + expect(gradeProviderChoice(undefined).passed).toBe(false); + }); +}); + +import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js"; +describe("actual source attribution and supported choice lanes", () => { + it("accepts a complete explanation with either actual selection lane; bare original card remains nonpassing", () => { + for (const lane of ["inline-card", "method-link"]) { + const o = valid(); + o.state.selectionLane = lane; + expect(gradeProviderChoice(o).passed).toBe(true); + } + const original = valid(); + original.reply = ""; + original.state.choiceExplanation.body = + "I’ll check the available Notion connection flow and give you the shortest setup path."; + expect(gradeProviderChoice(original).passed).toBe(false); + }); + it("rejects lost catalog, invisible explanation, wrong source run and unbound selection", () => { + for (const change of [ + { catalogChoicesIndependentlyMatched: false }, + { choiceExplanationVisible: false }, + { selectionAttributionMatched: false }, + { + choiceExplanation: { + body: valid().state.choiceExplanation.body, + runId: "foreign", + eventId: "message", + }, + }, + ]) { + const o = valid(); + Object.assign(o.state, change); + expect(gradeProviderChoice(o).passed).toBe(false); + } + }); + it("accepts only completed public messages from the exact source company/agent/run", () => { + const e = { + seq: 1, + eventType: "item.completed", + runId: "run", + companyId: "company", + agentId: "ceo", + payload: { + prpEvent: { + runId: "run", + payload: { + item: { + type: "agentMessage", + phase: "commentary", + status: "completed", + id: "msg", + text: "Public setup explanation", + }, + }, + }, + }, + }; + const target = { runId: "run", companyId: "company", agentId: "ceo" }; + expect(publicChoiceMessages([e], target)).toEqual([ + { runId: "run", eventId: "msg", body: "Public setup explanation" }, + ]); + for (const delta of [ + { runId: "foreign" }, + { agentId: "other" }, + { companyId: "foreign" }, + { eventType: "item.started" }, + ]) + expect(publicChoiceMessages([{ ...e, ...delta }], target)).toEqual([]); + const hidden = structuredClone(e); + hidden.payload.prpEvent.payload.item.phase = "analysis"; + expect(publicChoiceMessages([hidden], target)).toEqual([]); + }); + it("binds the inline request to the actual person, task, native run and Notion service", () => { + const c = { + id: "intent", + kind: "connection_intent", + status: "pending", + companyId: "company", + issueId: "issue", + createdByAgentId: "ceo", + addresseeUserId: "human", + sourceRunId: "run", + payload: { serviceSlug: "notion" }, + } as ConnectionIntentInteraction, + target = { + companyId: "company", + issueId: "issue", + agentId: "ceo", + humanId: "human", + runIds: ["run"], + }; + expect(intendedNotionCard([c], target)?.id).toBe("intent"); + for (const delta of [ + { companyId: "foreign" }, + { issueId: "other" }, + { createdByAgentId: "other" }, + { addresseeUserId: "other" }, + { sourceRunId: "other" }, + { status: "answered" }, + { payload: { serviceSlug: "slack" } }, + ]) + expect( + intendedNotionCard( + [{ ...c, ...delta } as ConnectionIntentInteraction], + target, + ), + ).toBeUndefined(); + expect( + intendedNotionCard([c, { ...c, id: "duplicate" }], target), + ).toBeUndefined(); + }); + it("allows actual same-company selected method links and rejects wrong method/company/provider", () => { + expect( + supportedNotionLink( + "/ACORN/apps/connect?source=notion&method=mcp-oauth", + "http://fixture.invalid", + "ACORN", + "mcp-oauth", + ), + ).toBe(true); + for (const link of [ + "https://other.invalid/ACORN/apps/connect?source=notion", + "/OTHER/apps/connect?source=notion", + "/ACORN/apps/connect?source=slack", + "/ACORN/apps/connect?source=notion&method=unrelated", + ]) + expect( + supportedNotionLink( + link, + "http://fixture.invalid", + "ACORN", + "mcp-oauth", + ), + ).toBe(false); + }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/external-provider-choice.ts b/tests/runner-e2e/chat-catalog-domains/external-provider-choice.ts index 335edeb174e..1a0cb0833b8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-provider-choice.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-provider-choice.ts @@ -1,46 +1,425 @@ import { expect } from "@playwright/test"; +import { isDeepStrictEqual } from "node:util"; import type { AppDefinition } from "../../../packages/shared/src/types/app-definition.js"; +import type { ConnectionIntentInteraction } from "../../../packages/shared/src/types/issue.js"; import { getAvailableConnectionMethods } from "../../../packages/shared/src/app-definitions.js"; -import { definition, root, checks, externalObservation, observeExternal, verifyRenderedReply } from "./external-support.js"; +import { + definition, + root, + checks, + externalObservation, + observeExternal, +} from "./external-support.js"; import { waitForReply } from "./identity-support.js"; - -interface Gallery { apps: AppDefinition[]; credentialSources: { vercelConnect: { available: boolean; reason: string | null } } } +import { collectRunEvents } from "../run-observations.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +interface Gallery { + apps: AppDefinition[]; + credentialSources: { + vercelConnect: { available: boolean; reason: string | null }; + }; +} export function notionSetupChoices(gallery: Gallery) { - const notion=gallery.apps.find(app=>app.slug==="notion"); - const methods=notion ? getAvailableConnectionMethods(notion) : []; - const oauth=methods.find(method=>method.auth==="oauth"&&method.transport==="mcp_remote"&&method.defaults?.serverUrl==="https://mcp.notion.com/mcp"); - const managed=gallery.credentialSources.vercelConnect; - if(!oauth||!oauth.credentialSources?.vercelConnect||managed.available||!managed.reason)throw new Error("Setup blocker: the actual Notion catalog must offer official sign-in and an explicitly unavailable Vercel Connect option with its prerequisite"); - return {methodKey:oauth.key,serverUrl:oauth.defaults!.serverUrl!,unavailableReason:managed.reason}; + const notion = gallery.apps.find((app) => app.slug === "notion"), + methods = notion ? getAvailableConnectionMethods(notion) : []; + const oauth = methods.find( + (method) => + method.auth === "oauth" && + method.transport === "mcp_remote" && + method.defaults?.serverUrl === "https://mcp.notion.com/mcp", + ); + const managed = gallery.credentialSources.vercelConnect; + if ( + !oauth?.credentialSources?.vercelConnect?.services.includes("notion") || + managed.available || + !managed.reason + ) + throw new Error( + "Setup blocker: actual reviewed Notion catalog must offer official sign-in and supported unavailable Vercel Connect with its prerequisite", + ); + return { + methodKey: oauth.key, + serverUrl: oauth.defaults!.serverUrl!, + unavailableReason: managed.reason, + advertisedChoices: { + service: "notion", + methodKey: oauth.key, + auth: oauth.auth, + transport: oauth.transport, + whenToUse: oauth.whenToUse, + serverUrl: oauth.defaults!.serverUrl!, + alternative: { + source: "vercel_connect", + available: managed.available, + reason: managed.reason, + }, + }, + }; } -export function truthfulNotionChoices(reply:string,reason:string){ - const offered=/notion/i.test(reply)&&/sign.?in|log.?in|authoriz|connect.*account/i.test(reply); - const prerequisite=/vercel\s+connect/i.test(reply)&&(/disabled/i.test(reason)?/disabled|enable|administrator|admin/i.test(reply):/oidc|workload|token|configur/i.test(reply)); - return offered&&prerequisite&&!/\b(?:I(?:['’]ve| have)?|we(?:['’]ve| have)?)\s+(?:already\s+)?connected\b|\bnotion\s+is\s+(?:now\s+)?connected\b/i.test(reply); +export function truthfulNotionChoices(reply: string, reason: string) { + const offered = + /notion/i.test(reply) && + /sign.?in|log.?in|authoriz|connect.*account/i.test(reply); + const prerequisite = + /vercel\s+connect/i.test(reply) && + (/disabled/i.test(reason) + ? /disabled|enable|administrator|admin/i.test(reply) + : /oidc|workload|token|configur/i.test(reply)); + return ( + offered && + prerequisite && + !/\b(?:I(?:['’]ve| have)?|we(?:['’]ve| have)?)\s+(?:already\s+)?connected\b|\bnotion\s+is\s+(?:now\s+)?connected\b/i.test( + reply, + ) + ); } -export function gradeProviderChoice(value:unknown){ - const c=checks(),o=externalObservation(value);c.add("complete-catalog-choice-evidence",!!o,"Actual public catalog methods, the advertised unavailable prerequisite and the directed setup UI are required.");if(!o)return c.result(); - c.add("real-catalog-and-truthful-choices",o.state.catalogOfficialOAuth===true&&o.state.unavailableAdvertised===true&&typeof o.state.unavailableReason==="string"&&truthfulNotionChoices(o.reply,o.state.unavailableReason),"The reply explains the actually offered sign-in choice and the prerequisite for unavailable Vercel Connect, without claiming setup is complete."); - c.add("directed-official-setup-path",o.state.modelDirectedNotionSetup===true&&o.state.selectedNotionVisible===true&&o.state.signInControlVisible===true,"The person follows the actual model's Notion setup link into the supported selected-provider flow; the oracle does not substitute a different protocol."); - c.add("original-intent-retained-without-phantom-account",o.state.originalConversationRetained===true&&o.state.originalQueryRetained===true&&o.connections.length===0,"Entering setup retains the original task and exact human intent, and does not make an unfinished choice a usable connection.");return c.result(); +const record = (v: unknown): Record => + v && typeof v === "object" && !Array.isArray(v) + ? (v as Record) + : {}; +interface MessageEvent { + seq?: number; + runId?: string; + companyId?: string; + agentId?: string; + eventType?: string; + payload?: unknown; } -export const providerChoiceExternalCases=[definition("connection-setup.choose-provider",async ctx=>{ - const gallery=await ctx.input.api.get(`${root(ctx)}/tools/gallery`),choices=notionSetupChoices(gallery); - const before=await ctx.input.api.get<{connections:unknown[]}>(`${root(ctx)}/tools/connections`);if(before.connections.length)throw new Error("Catalog choice requires a disposable company without an existing Notion account"); - return {...choices,catalogOfficialOAuth:true,unavailableAdvertised:true}; -},gradeProviderChoice,{ - authentication:"human-session", - sourceFiles:["chat-catalog-domains/external-provider-choice.ts","chat-catalog-domains/external-support.ts","chat-catalog-domains/internal-support.ts","chat-catalog-domains/identity-support.ts","../../packages/shared/src/app-definitions.ts","../../packages/shared/src/app-definitions/notion.json","chat-scenarios/connection-setup.yaml"], - async act(ctx,state){ - await waitForReply(ctx);const o=await observeExternal(ctx,state);if(!truthfulNotionChoices(o.reply,String(state.unavailableReason)))throw new Error("Actual reply does not explain the advertised Notion choices and unavailable prerequisite"); - const href=o.links.find(link=>{try{const url=new URL(link,o.origin);return url.origin===o.origin&&(url.pathname===`/${ctx.input.fixtures.company.issuePrefix}/apps/connect`&&[url.searchParams.get("source"),url.searchParams.get("appKey")].includes("notion"));}catch{return false;}}); - if(!href)throw new Error("Actual reply must direct the person to the selected supported Notion setup surface");state.modelDirectedNotionSetup=true; - const source=ctx.input.page.url(),issueId=ctx.issue()!.id;await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); - await expect(ctx.input.page.getByRole("main")).toContainText(/Notion/);state.selectedNotionVisible=true; - await expect(ctx.input.page.getByRole("button",{name:/Continue to (?:Notion|sign in)|Sign in to Notion/}).first()).toBeVisible();state.signInControlVisible=true; - // This scenario selects a method; no provider authorization is supplied or invented. - await ctx.input.page.goto(source,{waitUntil:"domcontentloaded"});await ctx.refreshIssue();state.originalConversationRetained=ctx.issue()?.id===issueId; - const comments=await ctx.input.api.get>(`/api/issues/${issueId}/comments?order=asc`);state.originalQueryRetained=comments.some(comment=>!!comment.authorUserId&&comment.body===ctx.scenario.query); +export function publicChoiceMessages( + events: MessageEvent[], + expected: { companyId: string; agentId: string; runId: string }, +) { + return events.flatMap((event) => { + const prp = record(record(event.payload).prpEvent), + payload = record(prp.payload), + item = record(payload.item); + return event.companyId === expected.companyId && + event.agentId === expected.agentId && + event.runId === expected.runId && + prp.runId === expected.runId && + event.eventType === "item.completed" && + item.type === "agentMessage" && + item.status === "completed" && + ["commentary", "final_answer"].includes(String(item.phase)) && + typeof item.id === "string" && + typeof item.text === "string" && + item.text.length <= 12000 + ? [{ runId: expected.runId, eventId: item.id, body: item.text }] + : []; + }); +} +export function intendedNotionCard( + cards: ConnectionIntentInteraction[], + expected: { + companyId: string; + issueId: string; + agentId: string; + humanId: string; + runIds: string[]; }, - async verifyResult(ctx,_state,value){await verifyRenderedReply(ctx,value);}, -})]; +) { + const matches = cards.filter( + (c) => + c.kind === "connection_intent" && + c.status === "pending" && + c.companyId === expected.companyId && + c.issueId === expected.issueId && + c.createdByAgentId === expected.agentId && + c.addresseeUserId === expected.humanId && + c.payload.serviceSlug === "notion" && + !!c.sourceRunId && + expected.runIds.includes(c.sourceRunId), + ); + return matches.length === 1 ? matches[0] : undefined; +} +export function supportedNotionLink( + link: string, + origin: string, + prefix: string, + method: string, +) { + try { + const u = new URL(link, origin); + return ( + u.origin === origin && + u.pathname === `/${prefix}/apps/connect` && + [u.searchParams.get("source"), u.searchParams.get("appKey")].includes( + "notion", + ) && + (!u.searchParams.has("method") || u.searchParams.get("method") === method) + ); + } catch { + return false; + } +} +const visibleText = (s: string) => + s + .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1") + .replace(/[*`_]/g, "") + .replace(/\s+/g, " ") + .trim(); +async function observeChoiceExplanation( + ctx: ChatCatalogContext, + state: Record, +) { + const runs = (await ctx.allRuns()).filter( + (r) => + r.agentId === ctx.input.fixtures.agent.id && r.status === "succeeded", + ), + messages = []; + for (const run of runs) { + const events = await collectRunEvents((afterSeq, limit) => + ctx.input.api.getRunObservation( + `/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`, + ), + ); + messages.push( + ...publicChoiceMessages(events, { + companyId: ctx.input.fixtures.company.id, + agentId: ctx.input.fixtures.agent.id, + runId: run.id, + }), + ); + } + const selected = messages.find( + (m) => + truthfulNotionChoices(m.body, String(state.unavailableReason)) && + m.body.trim(), + ); + if (selected) { + await expect( + ctx.input.page + .getByText(visibleText(selected.body), { exact: true }) + .first(), + ).toBeVisible(); + state.choiceExplanation = selected; + state.choiceExplanationVisible = true; + } +} +export function gradeProviderChoice(value: unknown) { + const c = checks(), + o = externalObservation(value); + c.add( + "complete-catalog-choice-evidence", + !!o, + "Actual public catalog choices, prerequisite and attributed human setup selection are required.", + ); + if (!o) return c.result(); + const explanation = record(o.state.choiceExplanation), + body = typeof explanation.body === "string" ? explanation.body : o.reply; + c.add( + "real-catalog-and-truthful-choices", + o.state.catalogChoicesIndependentlyMatched === true && + !!o.state.advertisedChoices && + typeof o.state.unavailableReason === "string" && + truthfulNotionChoices(body, o.state.unavailableReason) && + o.state.choiceExplanationVisible === true && + o.runs.some( + (r) => + r.id === explanation.runId && + r.agentId === o.leadId && + r.status === "succeeded", + ), + "A real visible model message from the source native run explains the actual reviewed choices and unavailable prerequisite; a bare setup card cannot replace that answer.", + ); + c.add( + "directed-official-setup-path", + o.state.modelDirectedNotionSetup === true && + ["inline-card", "method-link"].includes(String(o.state.selectionLane)) && + o.state.selectionAttributionMatched === true && + o.state.selectedNotionVisible === true && + o.state.signInControlVisible === true, + "The person selects the attributed real inline request or supported actual method link, then sees the selected Notion setup without protocol substitution.", + ); + c.add( + "original-intent-retained-without-phantom-account", + o.state.originalConversationRetained === true && + o.state.originalQueryRetained === true && + o.connections.length === 0, + "The original task and exact query remain intact; no authorization or usable account is invented.", + ); + return c.result(); +} +export const providerChoiceExternalCases = [ + definition( + "connection-setup.choose-provider", + async (ctx) => { + const choices = notionSetupChoices( + await ctx.input.api.get(`${root(ctx)}/tools/gallery`), + ); + const before = await ctx.input.api.get<{ connections: unknown[] }>( + `${root(ctx)}/tools/connections`, + ); + if (before.connections.length) + throw new Error( + "Catalog choice requires no existing account in the disposable company", + ); + return { + ...choices, + humanId: ( + await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ) + ).user.id, + }; + }, + gradeProviderChoice, + { + authentication: "human-session", + sourceFiles: [ + "chat-catalog-domains/external-provider-choice.ts", + "run-observations.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "../../packages/shared/src/app-definitions.ts", + "../../packages/shared/src/app-definitions/notion.json", + "../../ui/src/features/connections/ConnectionIntentInteractionBody.tsx", + "../../ui/src/features/connections/ConnectionSetupFlow.tsx", + "chat-scenarios/connection-setup.yaml", + ], + async verifySetup(ctx, state) { + const actual = notionSetupChoices( + await ctx.input.api.get(`${root(ctx)}/tools/gallery`), + ), + c = checks(); + c.add( + "actual-reviewed-advertised-choices", + isDeepStrictEqual(actual.advertisedChoices, state.advertisedChoices), + "Independent public gallery retains reviewed Notion method identity and actual unsupported credential-source prerequisite.", + ); + return c.result(); + }, + async act(ctx, state) { + await waitForReply(ctx); + await observeChoiceExplanation(ctx, state); + if (!state.choiceExplanationVisible) + throw new Error( + "Actual visible source-run answer does not explain advertised Notion choices and unavailable prerequisite", + ); + const o = await observeExternal(ctx, state), + source = ctx.input.page.url(), + issueId = ctx.issue()!.id, + runs = await ctx.allRuns(); + const cards = await ctx.input.api.get( + `/api/issues/${issueId}/interactions`, + ), + card = intendedNotionCard(cards, { + companyId: o.companyId, + issueId, + agentId: o.leadId, + humanId: String(state.humanId), + runIds: runs + .filter((r) => r.status === "succeeded") + .map((r) => r.id), + }); + if (card) { + state.selectionLane = "inline-card"; + state.selectedIntentId = card.id; + await ctx.input.page + .locator(`[id=${JSON.stringify(`interaction-${card.id}`)}]`) + .getByRole("button", { name: "Connect", exact: true }) + .click(); + await expect(ctx.input.page.getByRole("dialog")).toContainText( + "Notion", + ); + } else { + const href = o.links.find((link) => + supportedNotionLink( + link, + o.origin, + String(ctx.input.fixtures.company.issuePrefix), + String(state.methodKey), + ), + ); + if (!href) + throw new Error( + "No attributed inline request or actual supported Notion method link", + ); + state.selectionLane = "method-link"; + state.selectedMethodHref = href; + await ctx.input.page + .locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`) + .first() + .click(); + await expect(ctx.input.page.getByRole("main")).toContainText( + "Notion", + ); + } + state.modelDirectedNotionSetup = true; + state.selectedNotionVisible = true; + const surface = + state.selectionLane === "inline-card" + ? ctx.input.page.getByRole("dialog") + : ctx.input.page.getByRole("main"); + await expect( + surface + .getByRole("button", { + name: /Continue to (?:Notion|sign in)|Sign in to Notion/, + }) + .first(), + ).toBeVisible(); + state.signInControlVisible = true; + if (state.selectionLane === "inline-card") + await surface + .getByRole("button", { name: "Close", exact: true }) + .click(); + else + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await ctx.refreshIssue(); + state.originalConversationRetained = ctx.issue()?.id === issueId; + const comments = await ctx.input.api.get< + Array<{ body: string; authorUserId?: string }> + >(`/api/issues/${issueId}/comments?order=asc`); + state.originalQueryRetained = comments.some( + (comment) => + comment.authorUserId === state.humanId && + comment.body === ctx.scenario.query, + ); + }, + async observe(ctx, state) { + const actual = notionSetupChoices( + await ctx.input.api.get(`${root(ctx)}/tools/gallery`), + ); + state.catalogChoicesIndependentlyMatched = isDeepStrictEqual( + actual.advertisedChoices, + state.advertisedChoices, + ); + const o = await observeExternal(ctx, state); + state.selectionAttributionMatched = + state.selectionLane === "inline-card" + ? intendedNotionCard( + o.interactions as unknown as ConnectionIntentInteraction[], + { + companyId: o.companyId, + issueId: ctx.issue()!.id, + agentId: o.leadId, + humanId: String(state.humanId), + runIds: o.runs + .filter((r) => r.status === "succeeded") + .map((r) => r.id), + }, + )?.id === state.selectedIntentId + : typeof state.selectedMethodHref === "string" && + o.links.includes(state.selectedMethodHref) && + supportedNotionLink( + state.selectedMethodHref, + o.origin, + String(ctx.input.fixtures.company.issuePrefix), + String(state.methodKey), + ); + return { ...o, state }; + }, + async verifyResult(ctx, state) { + const explanation = record(state.choiceExplanation); + if (typeof explanation.body !== "string") + throw new Error("Missing actual source-run choice explanation"); + await expect( + ctx.input.page + .getByText(visibleText(explanation.body), { exact: true }) + .first(), + ).toBeVisible(); + }, + }, + ), +];