From 40af4fe402faf096306f95d897e7c5e853c536eb Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 20:28:44 -0500 Subject: [PATCH] test(evals): verify viewer-limited assistant consent --- .../external-assistant-connections.ts | 6 +- .../external-assistant-consent.ts | 99 +++++++++++++++++++ .../external-definitions.ts | 1 + .../chat-catalog-domains/external.test.ts | 11 ++- tests/runner-e2e/public-mcp-client.ts | 10 +- 5 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts diff --git a/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts b/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts index 21c39d9ca85..89588209a33 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts @@ -7,7 +7,7 @@ import { beginConnection, connect, mcp, origin, type Tokens } from "../public-mc const requests = new WeakMap>>(); const grants = new WeakMap(); -async function prepare(ctx: ChatCatalogContext) { +export async function prepareAssistantSetup(ctx: ChatCatalogContext) { if (origin !== ctx.input.api.baseURL) throw new Error("Public MCP fixture origin differs from the disposable server"); if (!ctx.input.privateSecrets) throw new Error("Public MCP requires the attempt-private redaction sink"); await ctx.input.api.patch("/api/instance/settings/experimental", { enablePublicMcp: true }); @@ -38,7 +38,7 @@ export function gradeAssistantSetup(value: unknown) { return c.result(); } export const assistantConnectionExternalCases = [definition("assistant-connections.connect-external-assistant", async ctx => { - const setup = await prepare(ctx); + const setup = await prepareAssistantSetup(ctx); const response = await fetch(setup.serverUrl, { method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: { companyId: ctx.input.fixtures.company.id } }) }); const unauthenticatedMcpStatus = response.status; await response.body?.cancel(); @@ -64,7 +64,7 @@ export const assistantConnectionExternalCases = [definition("assistant-connectio async observe(ctx, state) { state.assistantConnections = await ctx.input.api.get("/api/mcp/connections"); return observeExternal(ctx, state); }, async cleanup(ctx) { for (const grant of await ctx.input.api.get("/api/mcp/connections")) await ctx.input.api.delete(`/api/mcp/connections/${grant.id}`); grants.delete(ctx); }, }), definition("assistant-connections.decline-consent", async ctx => { - await prepare(ctx); const source = ctx.input.page.url(); + await prepareAssistantSetup(ctx); const source = ctx.input.page.url(); const other = await connect(ctx.input.page.context(), { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, ctx.input.page, ctx.input.privateSecrets!); grants.set(ctx, other); const before = await ctx.input.api.get("/api/mcp/connections"); const request = await beginConnection(true); requests.set(ctx, request); ctx.input.privateSecrets!.push(request.verifier, request.state); diff --git a/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts b/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts new file mode 100644 index 00000000000..6eed3b60cb4 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts @@ -0,0 +1,99 @@ +import { randomBytes, randomUUID } from "node:crypto"; +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { McpConnection } from "../../../packages/shared/src/public-mcp.js"; +import { connect, mcp, type ToolResult } from "../public-mcp-client.js"; +import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js"; +import { prepareAssistantSetup } from "./external-assistant-connections.js"; +import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { waitForReply } from "./identity-support.js"; +const people = new WeakMap>>(); +function writeDenied(result: ToolResult) { return result.isError === true && result.content.some(row => row.type === "text" && /paperclip:write permission is required/.test(row.text ?? "")); } +export function gradeViewerConsent(value: unknown) { + const c = checks(), o = externalObservation(value); + c.add("complete-viewer-consent-evidence", !!o, "Actual viewer membership, requested/granted scopes, visible consent and real capability outcomes are required."); if (!o) return c.result(); + const connections = o.state.viewerConnections as McpConnection[] | undefined; + c.add("requested-write-is-not-granted-write", o.state.requestedWrite === true && o.state.readOnlyRoleVisible === true && o.state.writeCheckboxDisabled === true + && o.state.writeCheckboxUnchecked === true && o.state.companyVisible === true && o.state.viewerEscalationStatus === 403 + && connections?.length === 1 && connections[0]?.companyId === o.companyId && connections[0]?.scopes.includes("paperclip:read") + && !connections[0]?.scopes.includes("paperclip:write") && o.state.tokenScopesReadOnly === true, + "The actual signed-in viewer sees the intended company and requested write scope but cannot accept writes, including via the public consent API."); + c.add("only-explicit-human-grant", o.state.viewerGrantId === connections?.[0]?.id && o.state.humanConsented === true + && o.activity.some(row => row.action === "mcp.connection_authorized" && row.entityId === o.state.viewerGrantId && row.actorType === "user" && row.actorId === o.state.viewerId), + "Only the invited viewer's actual browser consent authorizes this grant, with independent actor attribution."); + c.add("actual-allowed-task-read", o.state.readTaskId === o.state.taskId && o.state.readTaskCompanyId === o.companyId && o.state.readMarkerMatched === true, + "The resulting external client reads the intended real company task and unpredictable fixture content."); + c.add("create-and-update-denied", o.state.createDeniedForScope === true && o.state.updateDeniedForScope === true && o.state.taskPreserved === true + && o.state.forbiddenTaskCount === 0, "Valid create and update attempts fail specifically for write authority and leave durable tasks unchanged."); + c.add("no-agent-self-consent-claim", !/\bI (?:have )?(?:approved|authorized|connected)\b|\b(?:is|now|already) connected\b/i.test(o.reply), + "The initial reply does not claim the person's later browser authorization has already occurred."); + return c.result(); +} +export const assistantConsentExternalCases = [definition("assistant-connections.consent-read-only", async ctx => { + await prepareAssistantSetup(ctx); const browser = ctx.input.page.context().browser(), secrets = ctx.input.privateSecrets; + if (!browser || !secrets) throw new Error("Viewer consent requires an isolated signed-in person and private redactor"); + const viewer = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets, name: "Viewer Casey" }); people.set(ctx, viewer); + try { + await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, viewer, secrets, "viewer"); + const marker = `Viewer allowed fixture ${randomBytes(12).toString("hex")}`; + const task = await ctx.input.api.post<{ id: string; title: string; description: string }>(`${root(ctx)}/issues`, { title: "Viewer consent sample task", description: marker, status: "backlog" }); + return { viewerId: viewer.id, taskId: task.id, taskBefore: task, marker, forbiddenTitle: `Forbidden viewer task ${ctx.input.nonce}` }; + } catch (error) { people.delete(ctx); await viewer.dispose(); throw error; } +}, gradeViewerConsent, { + authentication: "human-session", + sourceFiles: ["chat-catalog-domains/external-assistant-consent.ts", "chat-catalog-domains/external-assistant-connections.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"], + async act(ctx, state) { + await waitForReply(ctx); const viewer = people.get(ctx); if (!viewer || !ctx.input.privateSecrets) throw new Error("Viewer consent fixture missing"); + const page = await viewer.context.newPage(); + try { + const connection = await connect(viewer.context, { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, page, ctx.input.privateSecrets, false, { + requestedWrite: true, + async inspectConsent(consent) { + await expect(consent.getByRole("radio", { name: ctx.input.fixtures.company.name, exact: true })).toBeChecked(); state.companyVisible = true; + await expect(consent.getByText("Your role in this organization is read-only.", { exact: true })).toBeVisible(); state.readOnlyRoleVisible = true; + const writes = consent.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true }); + await expect(writes).toBeDisabled(); await expect(writes).not.toBeChecked(); state.writeCheckboxDisabled = true; state.writeCheckboxUnchecked = true; + const requestId = new URL(consent.url()).pathname.split("/").at(-1)!; + const requestResponse = await viewer.request(`/api/mcp/requests/${encodeURIComponent(requestId)}`); + if (!requestResponse.ok) throw new Error("Viewer cannot inspect the actual pending authorization request"); + const request = await requestResponse.json() as { requestedWrite: boolean }; state.requestedWrite = request.requestedWrite; + const escalation = await viewer.request(`/api/mcp/requests/${encodeURIComponent(requestId)}/consent`, "POST", { decision: "approve", companyId: ctx.input.fixtures.company.id, allowWrites: true }); + state.viewerEscalationStatus = escalation.status; await escalation.body?.cancel(); + }, + }); + state.humanConsented = true; + state.tokenScopesReadOnly = connection.tokens.scope.split(/\s+/).includes("paperclip:read") && !connection.tokens.scope.split(/\s+/).includes("paperclip:write"); + const client = await mcp(connection.tokens); + try { + const identity = await client.call("paperclip_connection"); state.viewerGrantId = identity.structuredContent?.connectionId; + const read = await client.call("paperclip_read_task", { companyId: ctx.input.fixtures.company.id, taskId: state.taskId }); + if (read.isError) throw new Error("Viewer read capability was unavailable after consent"); + const task = read.structuredContent?.task as { id?: string; companyId?: string; description?: string } | undefined; + state.readTaskId = task?.id; state.readTaskCompanyId = task?.companyId; state.readMarkerMatched = task?.description === state.marker; + state.createDeniedForScope = writeDenied(await client.call("paperclip_create_task", { companyId: ctx.input.fixtures.company.id, requestId: randomUUID(), title: state.forbiddenTitle, description: "Negative viewer create probe", assigneeAgentId: ctx.input.fixtures.agent.id })); + state.updateDeniedForScope = writeDenied(await client.call("paperclip_update_task", { companyId: ctx.input.fixtures.company.id, taskId: state.taskId, requestId: randomUUID(), changes: { title: "Forbidden viewer mutation" } })); + } finally { await client.close(); } + await expect(page.getByRole("heading", { name: "Assistant connections", exact: true })).toBeVisible(); + } finally { await page.close(); } + }, + async observe(ctx, state) { + const viewer = people.get(ctx); if (!viewer) throw new Error("Viewer identity fixture missing"); + const response = await viewer.request("/api/mcp/connections"); if (!response.ok) throw new Error("Viewer grant listing failed"); state.viewerConnections = await response.json(); + const task = await ctx.input.api.get<{ id: string; title: string; description: string }>(`/api/issues/${state.taskId}`); const before = state.taskBefore as typeof task; + state.taskPreserved = task.id === before.id && task.title === before.title && task.description === before.description; + const o = await observeExternal(ctx, state); state.forbiddenTaskCount = o.tasks.filter(row => row.title === state.forbiddenTitle).length; return { ...o, state }; + }, + async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); }, + async cleanup(ctx, state) { + const viewer = people.get(ctx); people.delete(ctx); + try { + if (viewer) { + const listing = await viewer.request("/api/mcp/connections"); if (!listing.ok) throw new Error("Viewer cleanup cannot inspect owned grants"); + for (const grant of await listing.json() as McpConnection[]) { + if (grant.companyId !== ctx.input.fixtures.company.id) throw new Error("Disposable viewer unexpectedly owns another company grant"); + const response = await viewer.request(`/api/mcp/connections/${grant.id}`, "DELETE"); if (!response.ok && response.status !== 404) throw new Error("Viewer grant cleanup failed"); await response.body?.cancel(); + } + } + } finally { await viewer?.dispose(); } + }, +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts index 38408590128..a02348c56b8 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-definitions.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-definitions.ts @@ -13,3 +13,4 @@ export { assistantDeviceExternalCases } from "./external-assistant-device.js"; export { gatewayClientExternalCases } from "./external-gateway-client.js"; export { pluginInstallExternalCases } from "./external-plugin-install.js"; export { mcpFailureExternalCases } from "./external-mcp-failure.js"; +export { assistantConsentExternalCases } from "./external-assistant-consent.js"; diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 7585f2ee1dd..716ffe604cb 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -4,6 +4,7 @@ import { chatChannelExternalCases } from "./external-chat-channels.js"; import { gatewayClientExternalCases } from "./external-gateway-client.js"; import { pluginInstallExternalCases } from "./external-plugin-install.js"; import { mcpFailureExternalCases } from "./external-mcp-failure.js"; +import { assistantConsentExternalCases } from "./external-assistant-consent.js"; import { assistantDeviceExternalCases } from "./external-assistant-device.js"; import { pluginExternalCases } from "./external-plugins.js"; import { appCatalogExternalCases } from "./external-app-catalog.js"; @@ -15,7 +16,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js"; import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js"; import type { ExternalObservation } from "./external-support.js"; -const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases]; +const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases]; function base(scenarioId: string): ExternalObservation { return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" }, phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], @@ -23,7 +24,12 @@ function base(scenarioId: string): ExternalObservation { } function valid(id: string) { const o = base(id); - if (id === "connection-setup.unreachable-mcp") { + if (id === "assistant-connections.consent-read-only") { + Object.assign(o.state, { requestedWrite: true, readOnlyRoleVisible: true, writeCheckboxDisabled: true, writeCheckboxUnchecked: true, companyVisible: true, viewerEscalationStatus: 403, + viewerConnections: [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "offline_access"] }], tokenScopesReadOnly: true, viewerGrantId: "grant", humanConsented: true, viewerId: "viewer", + readTaskId: "task", taskId: "task", readTaskCompanyId: "company", readMarkerMatched: true, createDeniedForScope: true, updateDeniedForScope: true, taskPreserved: true, forbiddenTaskCount: 0 }); + o.activity = [{ id: "authorization", action: "mcp.connection_authorized", entityId: "grant", actorType: "user", actorId: "viewer" }]; + } else if (id === "connection-setup.unreachable-mcp") { Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true }); o.reply = "Open the supported MCP server connection setup and check its address."; } else if (id === "plugins.install-plugin") { @@ -94,6 +100,7 @@ function valid(id: string) { return o; } const wrong: Record void> = { + "assistant-connections.consent-read-only": o => { o.state.viewerConnections = [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "paperclip:write"] }]; }, "connection-setup.unreachable-mcp": o => { o.connections = [{ id: "phantom", status: "active" }]; }, "plugins.install-plugin": o => { o.state.pluginAfter = { id: "plugin", pluginKey: "synthetic.preview", status: "error" }; }, "gateways-profiles.connect-client": o => { o.state.providerWriteCalls = 1; }, diff --git a/tests/runner-e2e/public-mcp-client.ts b/tests/runner-e2e/public-mcp-client.ts index 4f81ac37877..0da6ed3a9e9 100644 --- a/tests/runner-e2e/public-mcp-client.ts +++ b/tests/runner-e2e/public-mcp-client.ts @@ -43,7 +43,8 @@ export async function beginConnection(write = true, redirect = origin + "/eval-c return { clientId, verifier, state, redirect, params, consentUrl, requestId: new URL(consentUrl).pathname.split("/").at(-1)! }; } -export async function connect(context: BrowserContext, team: Team, write = true, page?: Page, secrets: string[] = [], configure = false) { +export async function connect(context: BrowserContext, team: Team, write = true, page?: Page, secrets: string[] = [], configure = false, + options: { requestedWrite?: boolean; inspectConsent?: (page: Page) => Promise } = {}) { // A real assistant callback has its own origin. A same-origin fake callback // would be intercepted by Paperclip's service worker and render the board. const callbackServer = page ? createServer((_req, res) => { res.writeHead(200, { "Content-Type": "text/html", "Cache-Control": "no-store", "Referrer-Policy": "no-referrer" }); res.end("

Assistant connected

"); }) : undefined; @@ -51,13 +52,18 @@ export async function connect(context: BrowserContext, team: Team, write = true, const address = callbackServer?.address(); const redirect = address && typeof address !== "string" ? `http://127.0.0.1:${address.port}/callback` : undefined; try { - const request = await beginConnection(write, redirect, configure); + const request = await beginConnection(options.requestedWrite ?? write, redirect, configure); let redirectUrl: string; if (page) { await page.goto(request.consentUrl); await expect(page.getByRole("heading", { name: /^Connect .+ to Paperclip$/ })).toBeVisible(); await page.getByRole("radio", { name: team.name, exact: true }).check(); if (write || configure) await page.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true }).check(); + if (!write && !configure && options.requestedWrite) { + const writes = page.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true }); + if (await writes.isEnabled()) await writes.uncheck(); + } + await options.inspectConsent?.(page); await page.getByRole("button", { name: "Connect organization", exact: true }).click(); await expect(page.getByRole("heading", { name: "Assistant connected" })).toBeVisible(); redirectUrl = page.url();