test(evals): verify viewer-limited assistant consent

tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts

@@ -7,7 +7,7 @@ import { beginConnection, connect, mcp, origin, type Tokens } from "../public-mc
 
 const requests = new WeakMap<ChatCatalogContext, Awaited<ReturnType<typeof beginConnection>>>();
 const grants = new WeakMap<ChatCatalogContext, { tokens: Tokens }>();
-async function prepare(ctx: ChatCatalogContext) {
+export async function prepareAssistantSetup(ctx: ChatCatalogContext) {
   if (origin !== ctx.input.api.baseURL) throw new Error("Public MCP fixture origin differs from the disposable server");
   if (!ctx.input.privateSecrets) throw new Error("Public MCP requires the attempt-private redaction sink");
   await ctx.input.api.patch("/api/instance/settings/experimental", { enablePublicMcp: true });
@@ -38,7 +38,7 @@ export function gradeAssistantSetup(value: unknown) {
   return c.result();
 }
 export const assistantConnectionExternalCases = [definition("assistant-connections.connect-external-assistant", async ctx => {
-  const setup = await prepare(ctx);
+  const setup = await prepareAssistantSetup(ctx);
   const response = await fetch(setup.serverUrl, { method: "POST", headers: { "content-type": "application/json" },
     body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: { companyId: ctx.input.fixtures.company.id } }) });
   const unauthenticatedMcpStatus = response.status; await response.body?.cancel();
@@ -64,7 +64,7 @@ export const assistantConnectionExternalCases = [definition("assistant-connectio
   async observe(ctx, state) { state.assistantConnections = await ctx.input.api.get<McpConnection[]>("/api/mcp/connections"); return observeExternal(ctx, state); },
   async cleanup(ctx) { for (const grant of await ctx.input.api.get<McpConnection[]>("/api/mcp/connections")) await ctx.input.api.delete(`/api/mcp/connections/${grant.id}`); grants.delete(ctx); },
 }), definition("assistant-connections.decline-consent", async ctx => {
-  await prepare(ctx); const source = ctx.input.page.url();
+  await prepareAssistantSetup(ctx); const source = ctx.input.page.url();
   const other = await connect(ctx.input.page.context(), { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, ctx.input.page, ctx.input.privateSecrets!); grants.set(ctx, other);
   const before = await ctx.input.api.get<McpConnection[]>("/api/mcp/connections");
   const request = await beginConnection(true); requests.set(ctx, request); ctx.input.privateSecrets!.push(request.verifier, request.state);

tests/runner-e2e/chat-catalog-domains/external-assistant-consent.ts

@@ -0,0 +1,99 @@
+import { randomBytes, randomUUID } from "node:crypto";
+import { expect } from "@playwright/test";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { McpConnection } from "../../../packages/shared/src/public-mcp.js";
+import { connect, mcp, type ToolResult } from "../public-mcp-client.js";
+import { createAuthenticatedPerson, enrollAuthenticatedPerson } from "./identity-auth.js";
+import { prepareAssistantSetup } from "./external-assistant-connections.js";
+import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
+import { waitForReply } from "./identity-support.js";
+const people = new WeakMap<ChatCatalogContext, Awaited<ReturnType<typeof createAuthenticatedPerson>>>();
+function writeDenied(result: ToolResult) { return result.isError === true && result.content.some(row => row.type === "text" && /paperclip:write permission is required/.test(row.text ?? "")); }
+export function gradeViewerConsent(value: unknown) {
+  const c = checks(), o = externalObservation(value);
+  c.add("complete-viewer-consent-evidence", !!o, "Actual viewer membership, requested/granted scopes, visible consent and real capability outcomes are required."); if (!o) return c.result();
+  const connections = o.state.viewerConnections as McpConnection[] | undefined;
+  c.add("requested-write-is-not-granted-write", o.state.requestedWrite === true && o.state.readOnlyRoleVisible === true && o.state.writeCheckboxDisabled === true
+    && o.state.writeCheckboxUnchecked === true && o.state.companyVisible === true && o.state.viewerEscalationStatus === 403
+    && connections?.length === 1 && connections[0]?.companyId === o.companyId && connections[0]?.scopes.includes("paperclip:read")
+    && !connections[0]?.scopes.includes("paperclip:write") && o.state.tokenScopesReadOnly === true,
+    "The actual signed-in viewer sees the intended company and requested write scope but cannot accept writes, including via the public consent API.");
+  c.add("only-explicit-human-grant", o.state.viewerGrantId === connections?.[0]?.id && o.state.humanConsented === true
+    && o.activity.some(row => row.action === "mcp.connection_authorized" && row.entityId === o.state.viewerGrantId && row.actorType === "user" && row.actorId === o.state.viewerId),
+    "Only the invited viewer's actual browser consent authorizes this grant, with independent actor attribution.");
+  c.add("actual-allowed-task-read", o.state.readTaskId === o.state.taskId && o.state.readTaskCompanyId === o.companyId && o.state.readMarkerMatched === true,
+    "The resulting external client reads the intended real company task and unpredictable fixture content.");
+  c.add("create-and-update-denied", o.state.createDeniedForScope === true && o.state.updateDeniedForScope === true && o.state.taskPreserved === true
+    && o.state.forbiddenTaskCount === 0, "Valid create and update attempts fail specifically for write authority and leave durable tasks unchanged.");
+  c.add("no-agent-self-consent-claim", !/\bI (?:have )?(?:approved|authorized|connected)\b|\b(?:is|now|already) connected\b/i.test(o.reply),
+    "The initial reply does not claim the person's later browser authorization has already occurred.");
+  return c.result();
+}
+export const assistantConsentExternalCases = [definition("assistant-connections.consent-read-only", async ctx => {
+  await prepareAssistantSetup(ctx); const browser = ctx.input.page.context().browser(), secrets = ctx.input.privateSecrets;
+  if (!browser || !secrets) throw new Error("Viewer consent requires an isolated signed-in person and private redactor");
+  const viewer = await createAuthenticatedPerson({ api: ctx.input.api, browser, secrets, name: "Viewer Casey" }); people.set(ctx, viewer);
+  try {
+    await enrollAuthenticatedPerson(ctx.input.api, ctx.input.fixtures.company.id, viewer, secrets, "viewer");
+    const marker = `Viewer allowed fixture ${randomBytes(12).toString("hex")}`;
+    const task = await ctx.input.api.post<{ id: string; title: string; description: string }>(`${root(ctx)}/issues`, { title: "Viewer consent sample task", description: marker, status: "backlog" });
+    return { viewerId: viewer.id, taskId: task.id, taskBefore: task, marker, forbiddenTitle: `Forbidden viewer task ${ctx.input.nonce}` };
+  } catch (error) { people.delete(ctx); await viewer.dispose(); throw error; }
+}, gradeViewerConsent, {
+  authentication: "human-session",
+  sourceFiles: ["chat-catalog-domains/external-assistant-consent.ts", "chat-catalog-domains/external-assistant-connections.ts", "chat-catalog-domains/identity-auth.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); const viewer = people.get(ctx); if (!viewer || !ctx.input.privateSecrets) throw new Error("Viewer consent fixture missing");
+    const page = await viewer.context.newPage();
+    try {
+      const connection = await connect(viewer.context, { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, page, ctx.input.privateSecrets, false, {
+        requestedWrite: true,
+        async inspectConsent(consent) {
+          await expect(consent.getByRole("radio", { name: ctx.input.fixtures.company.name, exact: true })).toBeChecked(); state.companyVisible = true;
+          await expect(consent.getByText("Your role in this organization is read-only.", { exact: true })).toBeVisible(); state.readOnlyRoleVisible = true;
+          const writes = consent.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true });
+          await expect(writes).toBeDisabled(); await expect(writes).not.toBeChecked(); state.writeCheckboxDisabled = true; state.writeCheckboxUnchecked = true;
+          const requestId = new URL(consent.url()).pathname.split("/").at(-1)!;
+          const requestResponse = await viewer.request(`/api/mcp/requests/${encodeURIComponent(requestId)}`);
+          if (!requestResponse.ok) throw new Error("Viewer cannot inspect the actual pending authorization request");
+          const request = await requestResponse.json() as { requestedWrite: boolean }; state.requestedWrite = request.requestedWrite;
+          const escalation = await viewer.request(`/api/mcp/requests/${encodeURIComponent(requestId)}/consent`, "POST", { decision: "approve", companyId: ctx.input.fixtures.company.id, allowWrites: true });
+          state.viewerEscalationStatus = escalation.status; await escalation.body?.cancel();
+        },
+      });
+      state.humanConsented = true;
+      state.tokenScopesReadOnly = connection.tokens.scope.split(/\s+/).includes("paperclip:read") && !connection.tokens.scope.split(/\s+/).includes("paperclip:write");
+      const client = await mcp(connection.tokens);
+      try {
+        const identity = await client.call("paperclip_connection"); state.viewerGrantId = identity.structuredContent?.connectionId;
+        const read = await client.call("paperclip_read_task", { companyId: ctx.input.fixtures.company.id, taskId: state.taskId });
+        if (read.isError) throw new Error("Viewer read capability was unavailable after consent");
+        const task = read.structuredContent?.task as { id?: string; companyId?: string; description?: string } | undefined;
+        state.readTaskId = task?.id; state.readTaskCompanyId = task?.companyId; state.readMarkerMatched = task?.description === state.marker;
+        state.createDeniedForScope = writeDenied(await client.call("paperclip_create_task", { companyId: ctx.input.fixtures.company.id, requestId: randomUUID(), title: state.forbiddenTitle, description: "Negative viewer create probe", assigneeAgentId: ctx.input.fixtures.agent.id }));
+        state.updateDeniedForScope = writeDenied(await client.call("paperclip_update_task", { companyId: ctx.input.fixtures.company.id, taskId: state.taskId, requestId: randomUUID(), changes: { title: "Forbidden viewer mutation" } }));
+      } finally { await client.close(); }
+      await expect(page.getByRole("heading", { name: "Assistant connections", exact: true })).toBeVisible();
+    } finally { await page.close(); }
+  },
+  async observe(ctx, state) {
+    const viewer = people.get(ctx); if (!viewer) throw new Error("Viewer identity fixture missing");
+    const response = await viewer.request("/api/mcp/connections"); if (!response.ok) throw new Error("Viewer grant listing failed"); state.viewerConnections = await response.json();
+    const task = await ctx.input.api.get<{ id: string; title: string; description: string }>(`/api/issues/${state.taskId}`); const before = state.taskBefore as typeof task;
+    state.taskPreserved = task.id === before.id && task.title === before.title && task.description === before.description;
+    const o = await observeExternal(ctx, state); state.forbiddenTaskCount = o.tasks.filter(row => row.title === state.forbiddenTitle).length; return { ...o, state };
+  },
+  async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
+  async cleanup(ctx, state) {
+    const viewer = people.get(ctx); people.delete(ctx);
+    try {
+      if (viewer) {
+        const listing = await viewer.request("/api/mcp/connections"); if (!listing.ok) throw new Error("Viewer cleanup cannot inspect owned grants");
+        for (const grant of await listing.json() as McpConnection[]) {
+          if (grant.companyId !== ctx.input.fixtures.company.id) throw new Error("Disposable viewer unexpectedly owns another company grant");
+          const response = await viewer.request(`/api/mcp/connections/${grant.id}`, "DELETE"); if (!response.ok && response.status !== 404) throw new Error("Viewer grant cleanup failed"); await response.body?.cancel();
+        }
+      }
+    } finally { await viewer?.dispose(); }
+  },
+})];

tests/runner-e2e/chat-catalog-domains/external-definitions.ts

@@ -13,3 +13,4 @@ export { assistantDeviceExternalCases } from "./external-assistant-device.js";
 export { gatewayClientExternalCases } from "./external-gateway-client.js";
 export { pluginInstallExternalCases } from "./external-plugin-install.js";
 export { mcpFailureExternalCases } from "./external-mcp-failure.js";
+export { assistantConsentExternalCases } from "./external-assistant-consent.js";

tests/runner-e2e/chat-catalog-domains/external.test.ts

@@ -4,6 +4,7 @@ import { chatChannelExternalCases } from "./external-chat-channels.js";
 import { gatewayClientExternalCases } from "./external-gateway-client.js";
 import { pluginInstallExternalCases } from "./external-plugin-install.js";
 import { mcpFailureExternalCases } from "./external-mcp-failure.js";
+import { assistantConsentExternalCases } from "./external-assistant-consent.js";
 import { assistantDeviceExternalCases } from "./external-assistant-device.js";
 import { pluginExternalCases } from "./external-plugins.js";
 import { appCatalogExternalCases } from "./external-app-catalog.js";
@@ -15,7 +16,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js";
 import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js";
 import type { ExternalObservation } from "./external-support.js";
 
-const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases];
+const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases];
 function base(scenarioId: string): ExternalObservation {
   return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" },
     phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [],
@@ -23,7 +24,12 @@ function base(scenarioId: string): ExternalObservation {
 }
 function valid(id: string) {
   const o = base(id);
-  if (id === "connection-setup.unreachable-mcp") {
+  if (id === "assistant-connections.consent-read-only") {
+    Object.assign(o.state, { requestedWrite: true, readOnlyRoleVisible: true, writeCheckboxDisabled: true, writeCheckboxUnchecked: true, companyVisible: true, viewerEscalationStatus: 403,
+      viewerConnections: [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "offline_access"] }], tokenScopesReadOnly: true, viewerGrantId: "grant", humanConsented: true, viewerId: "viewer",
+      readTaskId: "task", taskId: "task", readTaskCompanyId: "company", readMarkerMatched: true, createDeniedForScope: true, updateDeniedForScope: true, taskPreserved: true, forbiddenTaskCount: 0 });
+    o.activity = [{ id: "authorization", action: "mcp.connection_authorized", entityId: "grant", actorType: "user", actorId: "viewer" }];
+  } else if (id === "connection-setup.unreachable-mcp") {
     Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true });
     o.reply = "Open the supported MCP server connection setup and check its address.";
   } else if (id === "plugins.install-plugin") {
@@ -94,6 +100,7 @@ function valid(id: string) {
   return o;
 }
 const wrong: Record<string, (o: ExternalObservation) => void> = {
+  "assistant-connections.consent-read-only": o => { o.state.viewerConnections = [{ id: "grant", companyId: "company", scopes: ["paperclip:read", "paperclip:write"] }]; },
   "connection-setup.unreachable-mcp": o => { o.connections = [{ id: "phantom", status: "active" }]; },
   "plugins.install-plugin": o => { o.state.pluginAfter = { id: "plugin", pluginKey: "synthetic.preview", status: "error" }; },
   "gateways-profiles.connect-client": o => { o.state.providerWriteCalls = 1; },

tests/runner-e2e/public-mcp-client.ts

@@ -43,7 +43,8 @@ export async function beginConnection(write = true, redirect = origin + "/eval-c
   return { clientId, verifier, state, redirect, params, consentUrl, requestId: new URL(consentUrl).pathname.split("/").at(-1)! };
 }
 
-export async function connect(context: BrowserContext, team: Team, write = true, page?: Page, secrets: string[] = [], configure = false) {
+export async function connect(context: BrowserContext, team: Team, write = true, page?: Page, secrets: string[] = [], configure = false,
+  options: { requestedWrite?: boolean; inspectConsent?: (page: Page) => Promise<void> } = {}) {
   // A real assistant callback has its own origin. A same-origin fake callback
   // would be intercepted by Paperclip's service worker and render the board.
   const callbackServer = page ? createServer((_req, res) => { res.writeHead(200, { "Content-Type": "text/html", "Cache-Control": "no-store", "Referrer-Policy": "no-referrer" }); res.end("<h1>Assistant connected</h1>"); }) : undefined;
@@ -51,13 +52,18 @@ export async function connect(context: BrowserContext, team: Team, write = true,
   const address = callbackServer?.address();
   const redirect = address && typeof address !== "string" ? `http://127.0.0.1:${address.port}/callback` : undefined;
   try {
-  const request = await beginConnection(write, redirect, configure);
+  const request = await beginConnection(options.requestedWrite ?? write, redirect, configure);
   let redirectUrl: string;
   if (page) {
     await page.goto(request.consentUrl);
     await expect(page.getByRole("heading", { name: /^Connect .+ to Paperclip$/ })).toBeVisible();
     await page.getByRole("radio", { name: team.name, exact: true }).check();
     if (write || configure) await page.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true }).check();
+    if (!write && !configure && options.requestedWrite) {
+      const writes = page.getByRole("checkbox", { name: "Write all of your Paperclip data", exact: true });
+      if (await writes.isEnabled()) await writes.uncheck();
+    }
+    await options.inspectConsent?.(page);
     await page.getByRole("button", { name: "Connect organization", exact: true }).click();
     await expect(page.getByRole("heading", { name: "Assistant connected" })).toBeVisible();
     redirectUrl = page.url();