diff --git a/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts b/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts index 89588209a33..c9fe9145073 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-assistant-connections.ts @@ -1,93 +1,196 @@ +import { + chatgptSetupCase, + gradeChatgptSetup, +} from "./external-chatgpt-setup.js"; import { expect } from "@playwright/test"; -import type { McpConnection, McpConnectionSetup } from "../../../packages/shared/src/public-mcp.js"; +import type { + McpConnection, + McpConnectionSetup, +} from "../../../packages/shared/src/public-mcp.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; -import { definition, externalObservation, observeExternal, root, checks } from "./external-support.js"; +import { + definition, + externalObservation, + observeExternal, + root, + checks, +} from "./external-support.js"; import { waitForReply } from "./identity-support.js"; -import { beginConnection, connect, mcp, origin, type Tokens } from "../public-mcp-client.js"; +import { + beginConnection, + connect, + mcp, + origin, + type Tokens, +} from "../public-mcp-client.js"; -const requests = new WeakMap>>(); +const requests = new WeakMap< + ChatCatalogContext, + Awaited> +>(); const grants = new WeakMap(); export async function prepareAssistantSetup(ctx: ChatCatalogContext) { - if (origin !== ctx.input.api.baseURL) throw new Error("Public MCP fixture origin differs from the disposable server"); - if (!ctx.input.privateSecrets) throw new Error("Public MCP requires the attempt-private redaction sink"); - await ctx.input.api.patch("/api/instance/settings/experimental", { enablePublicMcp: true }); + if (origin !== ctx.input.api.baseURL) + throw new Error( + "Public MCP fixture origin differs from the disposable server", + ); + if (!ctx.input.privateSecrets) + throw new Error("Public MCP requires the attempt-private redaction sink"); + await ctx.input.api.patch("/api/instance/settings/experimental", { + enablePublicMcp: true, + }); const setup = await ctx.input.api.get("/api/mcp/setup"); - if (!setup.enabled || new URL(setup.serverUrl).origin !== origin) throw new Error("Setup blocker: authenticated public MCP issuer/resource is not configured for this disposable server"); + if (!setup.enabled || new URL(setup.serverUrl).origin !== origin) + throw new Error( + "Setup blocker: authenticated public MCP issuer/resource is not configured for this disposable server", + ); return setup; } export function gradeAssistantSetup(value: unknown) { - const c = checks(); const o = externalObservation(value); - c.add("complete-assistant-evidence", !!o, "Current setup metadata, actual signed-in consent and grant observations are required."); if (!o) return c.result(); + const c = checks(); + const o = externalObservation(value); + c.add( + "complete-assistant-evidence", + !!o, + "Current setup metadata, actual signed-in consent and grant observations are required.", + ); + if (!o) return c.result(); if (o.scenarioId === "assistant-connections.connect-external-assistant") { - const connections = o.state.assistantConnections as McpConnection[] | undefined; - c.add("actual-human-scoped-grant", Array.isArray(connections) && connections.length === 1 && connections[0]?.companyId === o.companyId - && connections[0]?.scopes.includes("paperclip:read") && !connections[0]?.scopes.includes("paperclip:write") - && o.state.humanConsentCompleted === true, "The person completes real OAuth browser consent for the intended company and read scope."); - c.add("actual-client-usable", o.state.clientCompanyId === o.companyId && o.state.clientReadScope === true && o.state.directRestStatus === 401, - "The real client calls the public MCP resource; its token is not a general REST board bearer."); - c.add("no-unauthenticated-company-hint-access", o.state.unauthenticatedMcpStatus === 401, "A company hint is not a credential or membership grant."); - c.add("supported-setup-link", o.links.some(href => { try { const url = new URL(href, o.origin); return url.origin === o.origin && url.pathname === o.state.resultPath; } catch { return false; } }), - "The reply links the actual company setup surface."); + return gradeChatgptSetup(value); } else if (o.scenarioId === "assistant-connections.decline-consent") { - c.add("actual-human-denied-request", o.state.consentVisible === true && o.state.callbackError === "access_denied" && o.state.callbackStateMatched === true - && o.state.newGrantCount === 0, "The real consent Cancel action denies this request and returns its original OAuth state."); - c.add("unrelated-grant-preserved", o.state.otherGrantBefore === o.state.otherGrantAfter && o.state.otherGrantStillUsable === true, - "Declining one authorization preserves the unrelated real grant and its access."); - c.add("truthful-decline", /disconnect|declin|cancel|won.t|will not|not connect|don.t/i.test(o.reply), "Chat acknowledges the declined connection without claiming success."); - } else c.add("known-assistant-scenario", false, "A case-specific assistant oracle is required."); + c.add( + "actual-human-denied-request", + o.state.consentVisible === true && + o.state.callbackError === "access_denied" && + o.state.callbackStateMatched === true && + o.state.newGrantCount === 0, + "The real consent Cancel action denies this request and returns its original OAuth state.", + ); + c.add( + "unrelated-grant-preserved", + o.state.otherGrantBefore === o.state.otherGrantAfter && + o.state.otherGrantStillUsable === true, + "Declining one authorization preserves the unrelated real grant and its access.", + ); + c.add( + "truthful-decline", + /disconnect|declin|cancel|won.t|will not|not connect|don.t/i.test( + o.reply, + ), + "Chat acknowledges the declined connection without claiming success.", + ); + } else + c.add( + "known-assistant-scenario", + false, + "A case-specific assistant oracle is required.", + ); return c.result(); } -export const assistantConnectionExternalCases = [definition("assistant-connections.connect-external-assistant", async ctx => { - const setup = await prepareAssistantSetup(ctx); - const response = await fetch(setup.serverUrl, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/list", params: { companyId: ctx.input.fixtures.company.id } }) }); - const unauthenticatedMcpStatus = response.status; await response.body?.cancel(); - return { setupServerUrl: setup.serverUrl, unauthenticatedMcpStatus, resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/assistant-connection`, resultLabel: "Assistant Connection (MCP)" }; -}, gradeAssistantSetup, { - authentication: "human-session", - coverageGaps: ["named-chatgpt-human-setup"], - sourceFiles: ["chat-catalog-domains/external-assistant-connections.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"], - async act(ctx, state) { - await waitForReply(ctx); const source = ctx.input.page.url(); - const link = ctx.input.page.locator(`[id^="comment-"] a[href$="/apps/assistant-connection"]`).first(); await expect(link).toBeVisible(); await link.click(); - await expect(ctx.input.page.getByRole("main")).toContainText(String(state.setupServerUrl)); - const connection = await connect(ctx.input.page.context(), { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, ctx.input.page, ctx.input.privateSecrets!); grants.set(ctx, connection); - state.humanConsentCompleted = true; - const client = await mcp(connection.tokens); - try { const result = await client.call("paperclip_connection"); state.clientCompanyId = result.structuredContent?.companyId; - state.clientReadScope = (result.structuredContent?.scopes as string[] | undefined)?.includes("paperclip:read") === true; - } finally { await client.close(); } - const response = await fetch(`${origin}${root(ctx)}/issues`, { headers: { Authorization: `Bearer ${connection.tokens.access_token}` } }); - state.directRestStatus = response.status; await response.body?.cancel(); - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); - }, - async observe(ctx, state) { state.assistantConnections = await ctx.input.api.get("/api/mcp/connections"); return observeExternal(ctx, state); }, - async cleanup(ctx) { for (const grant of await ctx.input.api.get("/api/mcp/connections")) await ctx.input.api.delete(`/api/mcp/connections/${grant.id}`); grants.delete(ctx); }, -}), definition("assistant-connections.decline-consent", async ctx => { - await prepareAssistantSetup(ctx); const source = ctx.input.page.url(); - const other = await connect(ctx.input.page.context(), { ...ctx.input.fixtures.company, issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL" }, false, ctx.input.page, ctx.input.privateSecrets!); grants.set(ctx, other); - const before = await ctx.input.api.get("/api/mcp/connections"); - const request = await beginConnection(true); requests.set(ctx, request); ctx.input.privateSecrets!.push(request.verifier, request.state); - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); - return { otherGrantId: before[0]?.id, otherGrantBefore: JSON.stringify(before[0]), requestId: request.requestId }; -}, gradeAssistantSetup, { - authentication: "human-session", - sourceFiles: ["chat-catalog-domains/external-assistant-connections.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "public-mcp-client.ts", "chat-scenarios/assistant-connections.yaml"], - async act(ctx, state) { - await waitForReply(ctx); const request = requests.get(ctx); if (!request) throw new Error("Pending authorization fixture missing"); - const source = ctx.input.page.url(); await ctx.input.page.goto(request.consentUrl); - await expect(ctx.input.page.getByRole("heading", { name: /^Connect .+ to Paperclip$/ })).toBeVisible(); - await ctx.input.page.getByRole("radio", { name: ctx.input.fixtures.company.name, exact: true }).check(); - state.consentVisible = true; - await ctx.input.page.getByRole("button", { name: "Cancel", exact: true }).click(); - await ctx.input.page.waitForURL(url => url.searchParams.has("error")); - const callback = new URL(ctx.input.page.url()); state.callbackError = callback.searchParams.get("error"); state.callbackStateMatched = callback.searchParams.get("state") === request.state; - const rows = await ctx.input.api.get("/api/mcp/connections"); state.newGrantCount = rows.filter(row => row.id !== state.otherGrantId).length; - state.otherGrantAfter = JSON.stringify(rows.find(row => row.id === state.otherGrantId)); - const client = await mcp(grants.get(ctx)!.tokens); - try { state.otherGrantStillUsable = (await client.call("paperclip_connection")).structuredContent?.companyId === ctx.input.fixtures.company.id; } finally { await client.close(); } - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); - }, - async verifyResult(ctx) { await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); }, - async cleanup(ctx) { for (const grant of await ctx.input.api.get("/api/mcp/connections")) await ctx.input.api.delete(`/api/mcp/connections/${grant.id}`); requests.delete(ctx); grants.delete(ctx); }, -})]; +export const assistantConnectionExternalCases = [ + chatgptSetupCase, + definition( + "assistant-connections.decline-consent", + async (ctx) => { + await prepareAssistantSetup(ctx); + const source = ctx.input.page.url(); + const other = await connect( + ctx.input.page.context(), + { + ...ctx.input.fixtures.company, + issuePrefix: ctx.input.fixtures.company.issuePrefix ?? "EVAL", + }, + false, + ctx.input.page, + ctx.input.privateSecrets!, + ); + grants.set(ctx, other); + const before = await ctx.input.api.get( + "/api/mcp/connections", + ); + const request = await beginConnection(true); + requests.set(ctx, request); + ctx.input.privateSecrets!.push(request.verifier, request.state); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + return { + otherGrantId: before[0]?.id, + otherGrantBefore: JSON.stringify(before[0]), + requestId: request.requestId, + }; + }, + gradeAssistantSetup, + { + authentication: "human-session", + sourceFiles: [ + "chat-catalog-domains/external-assistant-connections.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "public-mcp-client.ts", + "chat-scenarios/assistant-connections.yaml", + ], + async act(ctx, state) { + await waitForReply(ctx); + const request = requests.get(ctx); + if (!request) throw new Error("Pending authorization fixture missing"); + const source = ctx.input.page.url(); + await ctx.input.page.goto(request.consentUrl); + await expect( + ctx.input.page.getByRole("heading", { + name: /^Connect .+ to Paperclip$/, + }), + ).toBeVisible(); + await ctx.input.page + .getByRole("radio", { + name: ctx.input.fixtures.company.name, + exact: true, + }) + .check(); + state.consentVisible = true; + await ctx.input.page + .getByRole("button", { name: "Cancel", exact: true }) + .click(); + await ctx.input.page.waitForURL((url) => url.searchParams.has("error")); + const callback = new URL(ctx.input.page.url()); + state.callbackError = callback.searchParams.get("error"); + state.callbackStateMatched = + callback.searchParams.get("state") === request.state; + const rows = await ctx.input.api.get( + "/api/mcp/connections", + ); + state.newGrantCount = rows.filter( + (row) => row.id !== state.otherGrantId, + ).length; + state.otherGrantAfter = JSON.stringify( + rows.find((row) => row.id === state.otherGrantId), + ); + const client = await mcp(grants.get(ctx)!.tokens); + try { + state.otherGrantStillUsable = + (await client.call("paperclip_connection")).structuredContent + ?.companyId === ctx.input.fixtures.company.id; + } finally { + await client.close(); + } + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await expect( + ctx.input.page.getByTestId("task-chat-composer-input"), + ).toBeVisible(); + }, + async verifyResult(ctx) { + await ctx.input.page.reload({ waitUntil: "domcontentloaded" }); + await expect( + ctx.input.page.getByTestId("task-chat-composer-input"), + ).toBeVisible(); + }, + async cleanup(ctx) { + for (const grant of await ctx.input.api.get( + "/api/mcp/connections", + )) + await ctx.input.api.delete(`/api/mcp/connections/${grant.id}`); + requests.delete(ctx); + grants.delete(ctx); + }, + }, + ), +]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-chatgpt-contract.ts b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-contract.ts new file mode 100644 index 00000000000..cbef539464c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-contract.ts @@ -0,0 +1,226 @@ +import type { PublicSignInStorage } from "./external-public-signin-contract.js"; +import type { McpConnectionRequest } from "../../../packages/shared/src/public-mcp.js"; +export interface ChatgptInput { + operation: "availability" | "setup"; + executionId: string; + attemptId: string; + receiptPath: string; + backendOrigin: string; + publicOrigin: string; + companyId: string; + companyName: string; + prefix: string; + humanId: string; + operatorRef: string; + authorizationRef: string; + publicSession: PublicSignInStorage; + control?: { + id: string; + identifier: string; + title: string; + ownerFact: string; + }; + executablePath?: string; + channel?: string; +} +export interface ChatgptReceipt { + operation: ChatgptInput["operation"]; + executionId: string; + attemptId: string; + companyId: string; + humanId: string; + processIsolation: true; + testRunnerAttached: false; + tracingEnabled: false; + videoEnabled: false; + operatorAvailable: true; + grantId?: string; + requestId?: string; + commentId?: string; + clientRequestId?: string; + actualChatgptUi?: true; + realHumanConsent?: true; + actualProviderCallback?: true; + boundMcpComment?: true; + zeroNativeAdmissions?: true; + requestedScopes?: string[]; + acceptedScopes?: string[]; + configureDisclosed?: boolean; +} +const uuid = (s: unknown) => + typeof s === "string" && + /^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$/i.test(s); +export function namedChatgptRequest( + r: McpConnectionRequest, + companyId: string, +) { + return ( + !!r && + /^(?:ChatGPT|OpenAI|OpenAI ChatGPT)$/i.test(r.clientName) && + r.redirectOrigin === "https://chatgpt.com" && + (!r.clientOrigin || r.clientOrigin === "https://chatgpt.com") && + !r.agentConnection && + !r.requiresSignIn && + r.requestedWrite === true && + (!r.requestedCompanyId || r.requestedCompanyId === companyId) && + r.companies.some((c) => c.id === companyId && c.canWrite === true) + ); +} +export function validChatgptInput(i: ChatgptInput) { + try { + const p = new URL(i.publicOrigin), + b = new URL(i.backendOrigin), + ref = (s: string) => /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,119}$/.test(s); + return ( + ["availability", "setup"].includes(i.operation) && + i.executionId.endsWith( + ".local.assistant-connections.connect-external-assistant", + ) && + /^[a-f0-9]{32}$/.test(i.attemptId) && + i.receiptPath.startsWith("/") && + p.protocol === "https:" && + p.origin === i.publicOrigin && + !p.username && + !p.password && + b.protocol === "http:" && + b.hostname === "127.0.0.1" && + b.origin === i.backendOrigin && + uuid(i.companyId) && + !!i.humanId && + i.humanId.length <= 200 && + !!i.companyName && + /^[A-Z][A-Z0-9]*$/.test(i.prefix) && + ref(i.operatorRef) && + ref(i.authorizationRef) && + !!i.publicSession && + Array.isArray(i.publicSession.cookies) && + i.publicSession.cookies.length > 0 && + i.publicSession.cookies.every( + (c) => c.secure && c.domain.replace(/^\./, "") === p.hostname, + ) && + i.publicSession.origins.every((o) => o.origin === p.origin) && + (i.operation === "availability" + ? !i.control + : !!i.control && + uuid(i.control.id) && + /^[A-Z][A-Z0-9]*-\d+$/.test(i.control.identifier) && + i.control.title === "Assistant setup control" && + /^[A-Za-z]+ [A-Za-z]+$/.test(i.control.ownerFact)) + ); + } catch { + return false; + } +} +export function validChatgptReceipt(r: ChatgptReceipt, i: ChatgptInput) { + const allowed = [ + "operation", + "executionId", + "attemptId", + "companyId", + "humanId", + "processIsolation", + "testRunnerAttached", + "tracingEnabled", + "videoEnabled", + "operatorAvailable", + "grantId", + "requestId", + "commentId", + "clientRequestId", + "actualChatgptUi", + "realHumanConsent", + "actualProviderCallback", + "boundMcpComment", + "zeroNativeAdmissions", + "requestedScopes", + "acceptedScopes", + "configureDisclosed", + ]; + return ( + !!r && + Object.keys(r).every((k) => allowed.includes(k)) && + r.operation === i.operation && + r.executionId === i.executionId && + r.attemptId === i.attemptId && + r.companyId === i.companyId && + r.humanId === i.humanId && + r.processIsolation === true && + r.testRunnerAttached === false && + r.tracingEnabled === false && + r.videoEnabled === false && + r.operatorAvailable === true && + (i.operation === "availability" + ? !r.grantId && !r.commentId && !r.realHumanConsent + : uuid(r.grantId) && + uuid(r.commentId) && + typeof r.requestId === "string" && + !!r.requestId && + typeof r.clientRequestId === "string" && + !!r.clientRequestId && + r.actualChatgptUi === true && + r.realHumanConsent === true && + r.actualProviderCallback === true && + r.boundMcpComment === true && + r.zeroNativeAdmissions === true && + exactAcceptedChatgptScopes( + r.requestedScopes, + r.acceptedScopes, + r.configureDisclosed, + )) + ); +} +export function setupCheckComment(body: string, owner: string) { + return ( + !/[@<>]/.test(body) && + body + .trim() + .replace(/[.!]$/, " ") + .replace(/\s+/g, " ") + .trim() + .toLowerCase() === + `Setup check: the launch owner is ${owner}`.toLowerCase() + ); +} +export function createdChatgptAppId( + body: unknown, + name: string, + requestText: string, + serverURL: string, +) { + if (!requestText.includes(name) || !requestText.includes(serverURL)) return; + const row = body as { + id?: unknown; + app?: { id?: unknown }; + connector?: { id?: unknown }; + } | null; + const id = row?.id ?? row?.app?.id ?? row?.connector?.id; + return typeof id === "string" && /^[a-zA-Z0-9_-]{8,100}$/.test(id) + ? id + : undefined; +} + +/** Saved authority must equal the actual request and explicit human acceptance. + * Configure is allowed only if its meaning was visibly disclosed on real consent. */ +export function exactAcceptedChatgptScopes( + requested: unknown, + accepted: unknown, + configureDisclosed: unknown, +) { + if (!Array.isArray(requested) || !Array.isArray(accepted)) return false; + const allowed = [ + "paperclip:read", + "paperclip:write", + "paperclip:configure", + "offline_access", + ]; + return ( + requested.includes("paperclip:read") && + requested.includes("paperclip:write") && + requested.every((s) => typeof s === "string" && allowed.includes(s)) && + new Set(requested).size === requested.length && + new Set(accepted).size === accepted.length && + requested.length === accepted.length && + requested.every((s) => accepted.includes(s)) && + (!requested.includes("paperclip:configure") || configureDisclosed === true) + ); +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-chatgpt-setup.test.ts b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-setup.test.ts new file mode 100644 index 00000000000..eb0c8d1280c --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-setup.test.ts @@ -0,0 +1,403 @@ +import { describe, expect, it } from "vitest"; +import { + gradeChatgptSetup, + truthfulChatgptHandoff, + chatgptSetupCase, +} from "./external-chatgpt-setup.js"; +import { + namedChatgptRequest, + exactAcceptedChatgptScopes, + validChatgptInput, + validChatgptReceipt, + createdChatgptAppId, + setupCheckComment, + type ChatgptInput, + type ChatgptReceipt, +} from "./external-chatgpt-contract.js"; +import type { ExternalObservation } from "./external-support.js"; +import type { McpConnectionRequest } from "../../../packages/shared/src/public-mcp.js"; +const company = "11111111-1111-4111-8111-111111111111", + grant = "22222222-2222-4222-8222-222222222222", + comment = "33333333-3333-4333-8333-333333333333"; +const scopes = ["paperclip:read", "paperclip:write", "offline_access"]; +function input(): ChatgptInput { + return { + operation: "setup", + executionId: + "catalog.local.assistant-connections.connect-external-assistant", + attemptId: "a".repeat(32), + receiptPath: "/private/receipt.json", + backendOrigin: "http://127.0.0.1:3100", + publicOrigin: "https://fixture.example.test", + companyId: company, + companyName: "Synthetic Company", + prefix: "ACORN", + humanId: "person", + operatorRef: "qa-operator", + authorizationRef: "authorized-chatgpt", + publicSession: { + cookies: [ + { + name: "session", + value: "synthetic-test-only", + domain: "fixture.example.test", + path: "/", + secure: true, + }, + ], + origins: [], + }, + control: { + id: comment, + identifier: "ACORN-2", + title: "Assistant setup control", + ownerFact: "Mira Patel", + }, + }; +} +function receipt(): ChatgptReceipt { + return { + operation: "setup", + executionId: input().executionId, + attemptId: input().attemptId, + companyId: company, + humanId: "person", + processIsolation: true, + testRunnerAttached: false, + tracingEnabled: false, + videoEnabled: false, + operatorAvailable: true, + grantId: grant, + requestId: "request", + commentId: comment, + clientRequestId: "tool-request", + actualChatgptUi: true, + realHumanConsent: true, + actualProviderCallback: true, + boundMcpComment: true, + zeroNativeAdmissions: true, + requestedScopes: [...scopes], + acceptedScopes: [...scopes], + configureDisclosed: true, + }; +} +function valid(): ExternalObservation { + return { + scenarioId: chatgptSetupCase.scenarioId, + origin: "http://fixture.invalid", + companyId: company, + leadId: "ceo", + reply: "Open the ChatGPT connection setup and complete the human consent.", + links: ["/ACORN/apps/assistant-connection"], + state: { + namedReceipt: receipt(), + humanId: "person", + operatorAvailable: true, + setupMetadataMatched: true, + actualBrowserInstructionsVisible: true, + resultPath: "/ACORN/apps/assistant-connection", + assistantConnections: [ + { + id: grant, + companyId: company, + clientName: "ChatGPT", + scopes: [...scopes], + revokedAt: null, + }, + ], + currentPersonPreserved: true, + controlTaskId: "control", + ownerFact: "Mira Patel", + controlCommentCount: 1, + controlComment: { + id: comment, + companyId: company, + issueId: "control", + authorUserId: "person", + authorAgentId: null, + createdByRunId: null, + clientRequestId: "tool-request", + body: "Setup check: the launch owner is Mira Patel.", + }, + controlTaskUnchanged: true, + noAgentAssignmentOrWatchdog: true, + zeroNativeAdmissions: true, + unauthenticatedMcpStatus: 401, + unauthenticatedHintDidNotGrant: true, + }, + activity: [ + { + id: "authorization", + action: "mcp.connection_authorized", + entityId: grant, + actorType: "user", + actorId: "person", + details: { clientId: "actual-chatgpt-client", scopes: [...scopes] }, + }, + { + id: "call", + action: "mcp.tool_called", + entityId: grant, + actorType: "user", + actorId: "person", + details: { + clientId: "actual-chatgpt-client", + operation: "paperclip_add_comment", + requestId: "tool-request", + }, + }, + ], + phaseReplies: [], + connections: [], + profiles: [], + secrets: [], + endpoints: [], + interactions: [], + tasks: [], + projects: [], + goals: [], + comments: [], + documents: [], + revisions: [], + humanQueries: [], + runs: [], + }; +} +describe("named ChatGPT real human setup independent authorities", () => { + it("accepts actual named grant, grounded client comment, exact audit and unchanged inert task", () => + expect(gradeChatgptSetup(valid()).passed).toBe(true)); + const wrong: Record void> = { + "operator acknowledgement without actual provider UI": (o) => { + (o.state.namedReceipt as ChatgptReceipt).actualChatgptUi = undefined; + }, + "generic MCP client": (o) => { + ( + o.state.assistantConnections as Array<{ clientName: string }> + )[0]!.clientName = "Synthetic MCP"; + }, + "wrong company": (o) => { + (o.state.namedReceipt as ChatgptReceipt).companyId = "other"; + }, + "wrong person": (o) => { + o.state.currentPersonPreserved = false; + }, + "wrong grant client audit": (o) => { + o.activity[1]!.details = { + ...o.activity[1]!.details, + clientId: "surrogate", + }; + }, + "authorized scopes differ from saved grant": (o) => { + o.activity[0]!.details = { + ...o.activity[0]!.details, + scopes: ["paperclip:read"], + }; + }, + "REST surrogate lacks MCP audit": (o) => { + o.activity.pop(); + }, + "unrelated tool request": (o) => { + o.activity[1]!.details = { + ...o.activity[1]!.details, + requestId: "other", + }; + }, + "wrong task": (o) => { + (o.state.controlComment as { issueId: string }).issueId = "other"; + }, + "wrong read fact": (o) => { + (o.state.controlComment as { body: string }).body = + "Setup check: the launch owner is Evan Hayes."; + }, + "extra write": (o) => { + o.activity.push({ ...o.activity[1]!, id: "extra" }); + }, + "native wake": (o) => { + o.state.zeroNativeAdmissions = false; + }, + "task mutation": (o) => { + o.state.controlTaskUnchanged = false; + }, + "company hint authority": (o) => { + o.state.unauthenticatedMcpStatus = 200; + }, + "connected before actual human action": (o) => { + o.reply = "I've already connected ChatGPT. Open setup."; + }, + "unexpected configure": (o) => { + ( + o.state.assistantConnections as Array<{ scopes: string[] }> + )[0]!.scopes.push("paperclip:configure"); + }, + "grant to agent impersonation": (o) => { + ( + o.state.assistantConnections as Array<{ scopes: string[] }> + )[0]!.scopes.push("paperclip:agent"); + }, + }; + for (const [name, mutate] of Object.entries(wrong)) + it(`rejects ${name}`, () => { + const o = valid(); + mutate(o); + expect(gradeChatgptSetup(o).passed).toBe(false); + }); + it("permits requested, visibly disclosed, explicitly accepted configure but rejects hidden or extra authority", () => { + const o = valid(), + r = o.state.namedReceipt as ChatgptReceipt; + r.requestedScopes!.push("paperclip:configure"); + r.acceptedScopes!.push("paperclip:configure"); + ( + o.state.assistantConnections as Array<{ scopes: string[] }> + )[0]!.scopes.push("paperclip:configure"); + o.activity[0]!.details!.scopes = [...scopes, "paperclip:configure"]; + expect(gradeChatgptSetup(o).passed).toBe(true); + r.configureDisclosed = false; + expect(gradeChatgptSetup(o).passed).toBe(false); + }); + it("accepts natural truthful setup direction and rejects affirmative contractions", () => { + for (const reply of [ + "ChatGPT is not yet connected. Open setup.", + "After you have connected ChatGPT, use the app.", + "I cannot connect ChatGPT on your behalf.", + ]) + expect(truthfulChatgptHandoff(reply)).toBe(true); + for (const reply of [ + "I've connected ChatGPT.", + "I’ve already authorized ChatGPT.", + "I have installed ChatGPT.", + "ChatGPT is already connected.", + ]) + expect(truthfulChatgptHandoff(reply)).toBe(false); + }); +}); +describe("finite private named-provider contract", () => { + it("requires secure public-host-only session and bounded exact-case input", () => { + expect(validChatgptInput(input())).toBe(true); + for (const mutate of [ + (i: ChatgptInput) => { + i.executionId = "unrelated"; + }, + (i: ChatgptInput) => { + i.publicSession.cookies[0]!.secure = false; + }, + (i: ChatgptInput) => { + i.publicSession.cookies[0]!.domain = "chatgpt.com"; + }, + (i: ChatgptInput) => { + i.control!.ownerFact = "@agent"; + }, + ]) { + const i = input(); + mutate(i); + expect(validChatgptInput(i)).toBe(false); + } + }); + it("receipt is closed, bound to actual attempt/person, and cannot carry token/session values", () => { + expect(validChatgptReceipt(receipt(), input())).toBe(true); + for (const delta of [ + { attemptId: "b".repeat(32) }, + { humanId: "other" }, + { accessToken: "must-not-be-exported" }, + { actualProviderCallback: false }, + { acceptedScopes: [...scopes, "paperclip:agent"] }, + ]) + expect( + validChatgptReceipt( + { ...receipt(), ...delta } as ChatgptReceipt, + input(), + ), + ).toBe(false); + }); + it("requires actual named official callback/person/company request", () => { + const r = { + id: "request", + clientName: "ChatGPT", + redirectOrigin: "https://chatgpt.com", + clientOrigin: "https://chatgpt.com", + agentConnection: false, + requestedWrite: true, + requestedConfigure: false, + offlineAccess: true, + requiresSignIn: false, + requestedCompanyId: company, + companies: [ + { id: company, name: "Synthetic", logoUrl: null, canWrite: true }, + ], + setupUrl: null, + } satisfies McpConnectionRequest; + expect(namedChatgptRequest(r, company)).toBe(true); + for (const delta of [ + { clientName: "Test MCP" }, + { redirectOrigin: "https://attacker.invalid" }, + { requestedCompanyId: "other" }, + { agentConnection: true }, + { requiresSignIn: true }, + { requestedWrite: false }, + { companies: [] }, + ]) + expect(namedChatgptRequest({ ...r, ...delta }, company)).toBe(false); + }); + it("accepts only exact requested and accepted scopes, with actual configure disclosure", () => { + expect(exactAcceptedChatgptScopes(scopes, scopes, false)).toBe(true); + expect( + exactAcceptedChatgptScopes( + [...scopes, "paperclip:configure"], + [...scopes, "paperclip:configure"], + true, + ), + ).toBe(true); + for (const [r, a, d] of [ + [scopes, [...scopes, "paperclip:configure"], true], + [ + [...scopes, "paperclip:configure"], + [...scopes, "paperclip:configure"], + false, + ], + [scopes, ["paperclip:read"], true], + [[...scopes, "paperclip:agent"], [...scopes, "paperclip:agent"], true], + ]) + expect(exactAcceptedChatgptScopes(r, a, d)).toBe(false); + }); + it("scopes provider cleanup to actual new app ID and submitted actual URL/name", () => { + const text = "New owned app https://fixture.example.test/mcp/paperclip"; + expect( + createdChatgptAppId( + { app: { id: "owned_app_123" } }, + "New owned app", + text, + "https://fixture.example.test/mcp/paperclip", + ), + ).toBe("owned_app_123"); + expect( + createdChatgptAppId( + { id: "owned_app_123" }, + "Unrelated app", + text, + "https://fixture.example.test/mcp/paperclip", + ), + ).toBeUndefined(); + expect( + createdChatgptAppId( + { id: "../escape" }, + "New owned app", + text, + "https://fixture.example.test/mcp/paperclip", + ), + ).toBeUndefined(); + }); + it("grounded neutral comment tolerates punctuation but no mentions/wrong fact or extra action", () => { + expect( + setupCheckComment( + " Setup check: the launch owner is Mira Patel! ", + "Mira Patel", + ), + ).toBe(true); + for (const b of [ + "Setup check: the launch owner is Evan Hayes.", + "Setup check: the launch owner is Mira Patel. Start work.", + "@Mira Patel", + "

Setup check: the launch owner is Mira Patel.

", + ]) + expect(setupCheckComment(b, "Mira Patel")).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-chatgpt-setup.ts b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-setup.ts new file mode 100644 index 00000000000..2ec84fb9e4d --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-setup.ts @@ -0,0 +1,451 @@ +import { expect } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { + McpConnection, + McpConnectionSetup, +} from "../../../packages/shared/src/public-mcp.js"; +import type { + Issue, + IssueComment, +} from "../../../packages/shared/src/types/issue.js"; +import { + definition, + checks, + root, + observeExternal, + externalObservation, +} from "./external-support.js"; +import { runtimePrivateSpec } from "./runtime-private.js"; +import { + chatgptPublicOrigin, + privateChatgptAvailability, + privateChatgptSetup, + cleanupPrivateChatgpt, +} from "./external-chatgpt.js"; +import { + setupCheckComment, + exactAcceptedChatgptScopes, + type ChatgptReceipt, +} from "./external-chatgpt-contract.js"; +import { waitForReply } from "./identity-support.js"; +import { isDeepStrictEqual } from "node:util"; +import type { Entity } from "./internal-support.js"; +function taskIdentity(t: Issue) { + return { + id: t.id, + companyId: t.companyId, + title: t.title, + description: t.description, + status: t.status, + assigneeAgentId: t.assigneeAgentId, + assigneeUserId: t.assigneeUserId, + responsibleUserId: t.responsibleUserId, + projectId: t.projectId, + parentId: t.parentId, + }; +} +async function companyNativeIds(ctx: ChatCatalogContext) { + const rows = await ctx.input.api.getRunObservation>( + `${root(ctx)}/heartbeat-runs?limit=100`, + ); + if (rows.length >= 100) + throw new Error( + "Cannot independently prove bounded complete company run inventory", + ); + return rows.map((r) => r.id).sort(); +} +export function truthfulChatgptHandoff(reply: string) { + const falseClaim = reply.split(/[.!?;\n]+|\bbut\b/i).some((clause) => { + const match = + /\bI(?:['’]ve| have)?\s+(?:already\s+)?(?:connected|installed|authorized)\b|\bChatGPT\s+(?:is|has been)\s+(?:now |already )?(?:connected|installed|authorized)\b/i.exec( + clause, + ); + return ( + !!match && + !/\b(?:once|after|when|if|until|before)\b[^,]{0,80}$/i.test( + clause.slice(0, match.index), + ) + ); + }); + return /ChatGPT/i.test(reply) && !falseClaim; +} +export function gradeChatgptSetup(value: unknown) { + const c = checks(), + o = externalObservation(value); + c.add( + "named-chatgpt-setup-evidence", + !!o, + "Actual selected human ChatGPT UI, real OAuth consent, same-grant MCP use and public state are required.", + ); + if (!o) return c.result(); + const r = o.state.namedReceipt as ChatgptReceipt | undefined, + rows = o.state.assistantConnections as McpConnection[] | undefined, + grant = rows?.find((g) => g.id === r?.grantId), + comment = o.state.controlComment as IssueComment | undefined; + c.add( + "supported-truthful-company-handoff", + o.state.setupMetadataMatched === true && + o.state.actualBrowserInstructionsVisible === true && + truthfulChatgptHandoff(o.reply) && + o.links.some((href) => { + try { + const u = new URL(href, o.origin); + return u.origin === o.origin && u.pathname === o.state.resultPath; + } catch { + return false; + } + }), + "The canonical CEO reply points to actual company assistant instructions and does not claim ChatGPT was installed before human action.", + ); + c.add( + "actual-selected-named-human-ui-consent", + o.state.operatorAvailable === true && + !!r && + r.operation === "setup" && + r.companyId === o.companyId && + r.humanId === o.state.humanId && + r.processIsolation === true && + r.testRunnerAttached === false && + r.tracingEnabled === false && + r.videoEnabled === false && + r.actualChatgptUi === true && + r.realHumanConsent === true && + r.actualProviderCallback === true && + r.boundMcpComment === true && + r.zeroNativeAdmissions === true, + "The selected human really uses ChatGPT's private custom-app UI and submits the actual Paperclip OAuth consent; an operator acknowledgement or generic MCP client alone is insufficient.", + ); + + c.add( + "actual-single-named-current-person-grant", + !!grant && + rows?.filter((g) => g.companyId === o.companyId && !g.revokedAt) + .length === 1 && + grant.companyId === o.companyId && + !grant.revokedAt && + /^(?:ChatGPT|OpenAI|OpenAI ChatGPT)$/i.test(grant.clientName) && + grant.scopes.includes("paperclip:read") && + grant.scopes.includes("paperclip:write") && + exactAcceptedChatgptScopes( + r?.requestedScopes, + grant.scopes, + r?.configureDisclosed, + ) && + exactAcceptedChatgptScopes( + r?.acceptedScopes, + grant.scopes, + r?.configureDisclosed, + ) && + o.state.currentPersonPreserved === true, + "A real single named ChatGPT grant exists for the intended company/current person with the explicitly approved scopes.", + ); + const activity = o.activity as Array; + const authorized = activity.find( + (a) => + a.action === "mcp.connection_authorized" && + a.entityId === grant?.id && + a.actorType === "user" && + a.actorId === o.state.humanId, + ); + const calls = activity.filter( + (a) => a.action === "mcp.tool_called" && a.entityId === grant?.id, + ); + c.add( + "independent-exact-grant-comment-audit", + !!r && + !!authorized && + calls.length === 1 && + calls[0]!.actorType === "user" && + calls[0]!.actorId === o.state.humanId && + typeof authorized.details?.clientId === "string" && + !!authorized.details.clientId && + exactAcceptedChatgptScopes( + authorized.details.scopes, + grant?.scopes, + r.configureDisclosed, + ) && + calls[0]!.details?.clientId === authorized.details?.clientId && + calls[0]!.details?.operation === "paperclip_add_comment" && + calls[0]!.details?.requestId === r.clientRequestId && + !!comment && + comment.id === r.commentId && + comment.issueId === o.state.controlTaskId && + comment.companyId === o.companyId && + comment.authorUserId === o.state.humanId && + !comment.authorAgentId && + !comment.createdByRunId && + comment.clientRequestId === r.clientRequestId && + setupCheckComment(comment.body, String(o.state.ownerFact)) && + o.state.controlCommentCount === 1, + "Independent public audit joins the exact named client/grant/person/requestId to its one actual task-grounded comment; no surrogate REST write or wrong target can pass.", + ); + c.add( + "human-controlled-task-and-no-native-wake", + o.state.controlTaskUnchanged === true && + o.state.noAgentAssignmentOrWatchdog === true && + o.state.zeroNativeAdmissions === true, + "The owned backlog task remains human-controlled, unassigned to agents, without watchdog or mentions; actual full company run IDs prove no native admission during qualification.", + ); + c.add( + "company-hint-is-not-authority", + o.state.unauthenticatedMcpStatus === 401 && + o.state.unauthenticatedHintDidNotGrant === true, + "A real unauthenticated request carrying only a copied company hint receives no grant or company access.", + ); + return c.result(); +} +export const chatgptSetupCase = definition( + "assistant-connections.connect-external-assistant", + async (ctx) => { + await ctx.input.api.patch("/api/instance/settings/experimental", { + enablePublicMcp: true, + }); + const origin = await chatgptPublicOrigin(ctx), + setup = await ctx.input.api.get("/api/mcp/setup"); + if ( + !setup.enabled || + setup.serverUrl !== `${origin.publicOrigin}/mcp/paperclip` + ) + throw new Error( + "Setup blocker: actual named ChatGPT setup does not advertise its validated owned HTTPS resource", + ); + const availability = await privateChatgptAvailability(ctx); + const humanId = ( + await ctx.input.api.get<{ user: { id: string } }>("/api/auth/get-session") + ).user.id; + const before = await ctx.input.api.get( + "/api/mcp/connections", + ), + url = new URL(setup.serverUrl); + url.searchParams.set("companyId", ctx.input.fixtures.company.id); + const response = await fetch(url, { + method: "POST", + redirect: "error", + signal: AbortSignal.timeout(15000), + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + id: 1, + method: "tools/list", + params: { companyId: ctx.input.fixtures.company.id }, + }), + }); + const unauthenticatedMcpStatus = response.status; + await response.body?.cancel(); + const after = await ctx.input.api.get( + "/api/mcp/connections", + ); + if ( + before.some( + (g) => g.companyId === ctx.input.fixtures.company.id && !g.revokedAt, + ) + ) + throw new Error( + "Named ChatGPT setup requires no preexisting grant in the disposable company", + ); + const owners = [ + "Mira Patel", + "Jonah Reed", + "Lena Torres", + "Owen Brooks", + "Nadia Chen", + "Theo Martin", + "Iris Bennett", + "Evan Hayes", + ], + ownerFact = + owners[ + [...ctx.input.nonce].reduce((sum, c) => sum + c.charCodeAt(0), 0) % + owners.length + ]!; + const task = await ctx.input.api.post(`${root(ctx)}/issues`, { + title: "Assistant setup control", + description: `Synthetic setup qualification only. Launch owner: ${ownerFact}. No agent work is requested.`, + status: "backlog", + assigneeAgentId: null, + assigneeUserId: humanId, + responsibleUserId: humanId, + }); + if ( + task.assigneeAgentId || + task.assigneeUserId !== humanId || + task.status !== "backlog" || + task.watchdog + ) + throw new Error( + "Control task is not an inert human-owned backlog resource", + ); + return { + setupMetadataMatched: true, + setupServerUrl: setup.serverUrl, + humanId, + operatorAvailable: availability.operatorAvailable, + unauthenticatedMcpStatus, + unauthenticatedHintDidNotGrant: isDeepStrictEqual( + before.map((g) => g.id).sort(), + after.map((g) => g.id).sort(), + ), + controlTaskId: task.id, + controlIdentifier: task.identifier, + controlBefore: taskIdentity(task), + ownerFact, + resultPath: `/${ctx.input.fixtures.company.issuePrefix}/apps/assistant-connection`, + resultLabel: "Assistant Connection (MCP)", + }; + }, + gradeChatgptSetup, + { + authentication: "human-session", + ...runtimePrivateSpec("assistant-connections.connect-external-assistant"), + expectedRunCount: 1, + expectedHumanMessageCount: 1, + turnTimeoutMs: 120000, + attemptTimeoutMs: 1500000, + sourceFiles: [ + "chat-catalog-domains/external-chatgpt-setup.ts", + "chat-catalog-domains/external-chatgpt.ts", + "chat-catalog-domains/external-chatgpt-worker.ts", + "chat-catalog-domains/external-chatgpt-contract.ts", + "chat-catalog-domains/runtime-private.ts", + "chat-runtime-private-requirements.ts", + "chat-catalog-domains/external-public-signin.ts", + "chat-catalog-domains/external-public-signin-worker.ts", + "chat-catalog-domains/external-public-signin-contract.ts", + "public-mcp-flow.ts", + "chat-public-origin.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "chat-scenarios/assistant-connections.yaml", + "../../ui/src/pages/apps/AssistantConnection.tsx", + "../../ui/src/pages/McpConnect.tsx", + "../../server/src/routes/public-mcp.ts", + "../../server/src/services/public-mcp/capabilities.ts", + "../../server/src/services/public-mcp/oauth.ts", + "../../packages/shared/src/mcp-setup.ts", + ], + async act(ctx, state) { + await waitForReply(ctx); + const o = await observeExternal(ctx, state); + if (!truthfulChatgptHandoff(o.reply)) + throw new Error( + "CEO invented connected ChatGPT state before actual human setup", + ); + const source = ctx.input.page.url(); + const href = o.links.find((href) => { + try { + const u = new URL(href, o.origin); + return u.origin === o.origin && u.pathname === state.resultPath; + } catch { + return false; + } + }); + if (!href) + throw new Error("Actual intended company human setup link is missing"); + await ctx.input.page + .locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`) + .first() + .click(); + await ctx.input.page + .getByText("Set up manually", { exact: true }) + .click(); + await ctx.input.page + .getByRole("tab", { name: "ChatGPT / Claude web", exact: true }) + .click(); + await expect( + ctx.input.page + .getByText(String(state.setupServerUrl), { exact: true }) + .first(), + ).toBeVisible(); + state.actualBrowserInstructionsVisible = true; + state.nativeBeforeQualification = await companyNativeIds(ctx); + state.namedReceipt = await privateChatgptSetup(ctx, { + id: String(state.controlTaskId), + identifier: String(state.controlIdentifier), + title: "Assistant setup control", + ownerFact: String(state.ownerFact), + }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async observe(ctx, state) { + state.assistantConnections = await ctx.input.api.get( + "/api/mcp/connections", + ); + const task = await ctx.input.api.get( + `/api/issues/${state.controlTaskId}`, + ), + comments = await ctx.input.api.get( + `/api/issues/${state.controlTaskId}/comments?order=asc`, + ); + state.controlCommentCount = comments.length; + state.controlComment = comments[0]; + state.controlTaskUnchanged = isDeepStrictEqual( + taskIdentity(task), + state.controlBefore, + ); + state.noAgentAssignmentOrWatchdog = + !task.assigneeAgentId && + task.assigneeUserId === state.humanId && + task.status === "backlog" && + !task.watchdog; + state.zeroNativeAdmissions = isDeepStrictEqual( + await companyNativeIds(ctx), + state.nativeBeforeQualification, + ); + state.currentPersonPreserved = + ( + await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ) + ).user.id === state.humanId; + return observeExternal(ctx, state); + }, + async verifyResult(ctx, state) { + const source = ctx.input.page.url(); + await ctx.input.page.goto(String(state.resultPath)); + await expect( + ctx.input.page.getByRole("heading", { + name: "Assistant Connection (MCP)", + exact: true, + }), + ).toBeVisible(); + const receipt = state.namedReceipt as ChatgptReceipt; + const rows = state.assistantConnections as McpConnection[]; + const grant = rows.find((g) => g.id === receipt.grantId); + if (!grant) + throw new Error( + "Actual named ChatGPT public grant disappeared before result verification", + ); + await expect( + ctx.input.page.getByText(new RegExp(grant.clientName, "i")).first(), + ).toBeVisible(); + await ctx.input.evidence("chat-catalog-named-chatgpt-setup.json", { + grantId: receipt.grantId, + requestId: receipt.requestId, + commentId: receipt.commentId, + companyId: ctx.input.fixtures.company.id, + humanId: state.humanId, + actualNamedUi: true, + actualHumanConsent: true, + zeroNativeAdmissions: state.zeroNativeAdmissions, + }); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + }, + async cleanup(ctx, state) { + try { + await cleanupPrivateChatgpt(ctx); + } finally { + const receipt = state.namedReceipt as ChatgptReceipt | undefined; + if (receipt?.grantId) + await ctx.input.api.delete( + `/api/mcp/connections/${receipt.grantId}`, + { allowNotFound: true }, + ); + if (typeof state.controlTaskId === "string") + await ctx.input.api.delete(`/api/issues/${state.controlTaskId}`, { + allowNotFound: true, + }); + } + }, + }, +); diff --git a/tests/runner-e2e/chat-catalog-domains/external-chatgpt-worker.ts b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-worker.ts new file mode 100644 index 00000000000..809d645571b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-chatgpt-worker.ts @@ -0,0 +1,600 @@ +/** Finite required-human named ChatGPT setup. No generic client registration, + * token exchange, REST comment surrogate, tracing, screenshots or provider state + * export. The browser stays private until its exact cleanup gate is released. */ +import { readFile, writeFile, lstat } from "node:fs/promises"; +import path from "node:path"; +import { + chromium, + test, + type Page, + type BrowserContext, +} from "@playwright/test"; +import { readValidatedPublicOriginReceipt } from "../chat-public-origin.js"; +import type { + McpConnection, + McpConnectionRequest, + McpConnectionSetup, +} from "../../../packages/shared/src/public-mcp.js"; +import { + namedChatgptRequest, + validChatgptInput, + setupCheckComment, + createdChatgptAppId, + exactAcceptedChatgptScopes, + type ChatgptInput, + type ChatgptReceipt, +} from "./external-chatgpt-contract.js"; +const esc = (s: string) => + s.replace( + /[&<>"']/g, + (c) => + ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[ + c + ]!, + ); +async function pane( + page: Page, + title: string, + body: string, + checks: string[], + button: string, +) { + await page.setContent( + `Private ChatGPT setup

${esc(title)}

${body}
${checks.map((text) => `

`).join("")}
`, + ); + await page.bringToFront(); +} +async function confirm(page: Page, timeout: number) { + await page.waitForFunction( + () => document.body.dataset.confirmed === "yes", + undefined, + { timeout }, + ); +} +async function json( + c: BrowserContext, + origin: string, + pathname: string, +): Promise { + const r = await c.request.get(origin + pathname); + if (!r.ok()) throw new Error("Private named-client public read failed"); + return r.json() as Promise; +} +async function main() { + let attached = false; + try { + test.info(); + attached = true; + } catch { + /* standalone */ + } + if (attached) + throw new Error("Private named ChatGPT operation cannot run in Test"); + const filename = process.argv[2]!, + directory = path.dirname(filename), + stat = await lstat(filename); + if ( + !stat.isFile() || + stat.isSymbolicLink() || + stat.uid !== process.getuid?.() || + stat.mode & 0o077 || + stat.size > 2 * 1024 * 1024 + ) + throw new Error("Invalid private named-client file"); + const i = JSON.parse(await readFile(filename, "utf8")) as ChatgptInput; + if (!validChatgptInput(i)) + throw new Error("Invalid finite named-client input"); + const origin = await readValidatedPublicOriginReceipt({ + receiptPath: i.receiptPath, + executionId: i.executionId, + attemptId: i.attemptId, + backendOrigin: i.backendOrigin, + }); + if (origin.publicOrigin !== i.publicOrigin) + throw new Error("Actual owned public origin changed"); + const browser = await chromium.launch({ + headless: false, + ...(i.executablePath ? { executablePath: i.executablePath } : {}), + ...(i.channel ? { channel: i.channel } : {}), + }); + const hardStop = setTimeout(() => { + void browser.close(); + }, 1050000); + hardStop.unref(); + let context: BrowserContext | undefined, + appId: string | undefined, + cleaned = false, + cleanupGrantId: string | undefined, + removeOwnedApp: (() => Promise) | undefined, + providerAppRemoved = false; + const stage = async (value: string) => + writeFile( + path.join(directory, "stage.json"), + JSON.stringify({ stage: value }), + { mode: 0o600 }, + ); + try { + const stateFile = path.join(directory, "public-storage.json"); + await writeFile(stateFile, JSON.stringify(i.publicSession), { + mode: 0o600, + }); + context = await browser.newContext({ storageState: stateFile }); + context.setDefaultTimeout(20000); + const person = await json<{ user: { id: string } }>( + context, + i.publicOrigin, + "/api/auth/get-session", + ); + if (person.user.id !== i.humanId) + throw new Error( + "Public human session does not match actual fixture person", + ); + const operator = await context.newPage(); + const identity = `

Operator ${esc(i.operatorRef)}. Authorized synthetic ChatGPT account reference ${esc(i.authorizationRef)}. Company ${esc(i.companyName)} (${esc(i.companyId)}).

`; + const common = { + operation: i.operation, + executionId: i.executionId, + attemptId: i.attemptId, + companyId: i.companyId, + humanId: i.humanId, + processIsolation: true, + testRunnerAttached: false, + tracingEnabled: false, + videoEnabled: false, + operatorAvailable: true, + } as const; + if (i.operation === "availability") { + await stage("availability"); + await pane( + operator, + "Required ChatGPT operator availability", + identity + + "

No provider setup or consent has occurred. Confirm availability within 15 seconds.

", + [ + "I am the explicitly selected authorized operator of this synthetic ChatGPT test account/workspace.", + "I am available for this attempt, can create a private custom MCP app, and can remove that exact test app afterward.", + ], + "Confirm availability", + ); + await confirm(operator, 15000); + await writeFile( + path.join(directory, "qualified.json"), + JSON.stringify(common), + { mode: 0o600 }, + ); + cleaned = true; + return; + } + await stage("product-setup"); + const c = context, + product = await c.newPage(), + chatgpt = await c.newPage(); + await product.goto( + `${i.publicOrigin}/${i.prefix}/apps/assistant-connection`, + ); + await product.getByText("Set up manually", { exact: true }).click(); + await product + .getByRole("tab", { name: "ChatGPT / Claude web", exact: true }) + .click(); + const setup = await json( + c, + i.publicOrigin, + "/api/mcp/setup", + ), + url = new URL(setup.serverUrl); + if ( + !setup.enabled || + url.origin !== i.publicOrigin || + url.pathname !== "/mcp/paperclip" || + url.search || + url.hash + ) + throw new Error( + "Actual named-client setup metadata differs from owned resource", + ); + await product.getByText(setup.serverUrl, { exact: true }).first().waitFor(); + const beforeGrants = await json( + c, + i.publicOrigin, + "/api/mcp/connections", + ), + beforeRuns = await json>( + c, + i.publicOrigin, + `/api/companies/${i.companyId}/heartbeat-runs?limit=100`, + ); + if (beforeRuns.length >= 100) + throw new Error("Cannot prove complete bounded company native inventory"); + const appName = `Paperclip ${i.prefix} setup ${i.attemptId.slice(0, 8)}`; + removeOwnedApp = async () => { + if (!appId || providerAppRemoved) return; + let deleted = false; + const observer = (r: import("@playwright/test").Response) => { + const request = r.request(), + u = new URL(r.url()), + text = request.postData() ?? ""; + if ( + u.origin === "https://chatgpt.com" && + r.ok() && + (request.method() === "DELETE" || + (request.method() === "POST" && + /delete|remove/i.test(u.pathname))) && + (u.pathname.split("/").includes(appId!) || text.includes(appId!)) + ) + deleted = true; + }; + c.on("response", observer); + try { + await stage("cleanup"); + await pane( + operator, + "Remove only this newly created ChatGPT test app", + identity + + `

In the actual ChatGPT Apps/Plugins settings, open ${esc(appName)} and complete its Delete/Remove confirmation. Do not remove another app or conversation. Only a successful provider deletion response bound to this actual new app ID qualifies. Paperclip separately revokes only this test grant.

`, + [], + "I opened the exact new app's delete confirmation", + ); + const deadline = Date.now() + 120000; + while (Date.now() < deadline && !deleted) + await new Promise((r) => setTimeout(r, 250)); + if (!deleted) + throw new Error( + "Exact created ChatGPT app deletion was not observed", + ); + providerAppRemoved = true; + } finally { + c.off("response", observer); + } + }; + // Observe only the real provider browser creation response; do not call any + // ChatGPT API. The actual newly created ID scopes eventual UI cleanup. + c.on("response", (response) => { + void (async () => { + const request = response.request(), + u = new URL(response.url()); + if ( + u.origin !== "https://chatgpt.com" || + request.method() !== "POST" || + !response.ok() + ) + return; + const text = request.postData() ?? ""; + if (!text.includes(appName) || !text.includes(setup.serverUrl)) return; + const actual = createdChatgptAppId( + await response.json().catch(() => null), + appName, + text, + setup.serverUrl, + ); + if (actual) { + if (appId && appId !== actual) + throw new Error("More than one app created"); + appId = actual; + } + })().catch(() => { + void browser.close(); + }); + }); + await chatgpt.goto("https://chatgpt.com/"); + await stage("provider-app"); + await pane( + operator, + "Create only the real private ChatGPT test app", + identity + + `

Use the other official ChatGPT tab. Sign in to the designated account, then open its current Apps/Create custom MCP settings (or its supported private custom-server flow). Do not publish anything to a workspace or adopt an existing app.

New app name: ${esc(appName)}

Use this exact URL copied from the real Paperclip browser setup instructions:

Choose OAuth. Stop at the actual Paperclip consent page and review the next private instruction panel. Do not paste any token, cookie, OAuth code or state into chat.

`, + [], + "I opened the actual provider flow", + ); + let consent: Page | undefined, + requestId = ""; + const deadline = Date.now() + 300000; + while (Date.now() < deadline && !consent) { + for (const p of c.pages()) { + try { + const u = new URL(p.url()); + if ( + u.origin === i.publicOrigin && + /^\/mcp-connect\/[A-Za-z0-9_-]+$/.test(u.pathname) + ) { + consent = p; + requestId = u.pathname.split("/").at(-1)!; + break; + } + } catch { + /* private pane */ + } + } + if (!consent) await new Promise((r) => setTimeout(r, 250)); + } + if (!consent || new URL(chatgpt.url()).origin !== "https://chatgpt.com") + throw new Error( + "Actual named ChatGPT UI did not produce its real Paperclip OAuth request", + ); + await stage("consent"); + const request = await json( + c, + i.publicOrigin, + `/api/mcp/requests/${requestId}`, + ); + if (!namedChatgptRequest(request, i.companyId)) + throw new Error( + "Actual ChatGPT client/company/write request does not match qualification", + ); + const configureDisclosed = + !request.requestedConfigure || + (await consent + .getByText(/configure agents, projects (?:and|&) skills/i) + .isVisible()); + if (!configureDisclosed) + throw new Error( + "Actual requested configure authority is not visibly disclosed on Paperclip consent", + ); + const submitted = consent.waitForResponse( + (r) => + new URL(r.url()).origin === i.publicOrigin && + new URL(r.url()).pathname === + `/api/mcp/requests/${requestId}/consent` && + r.request().method() === "POST", + { timeout: 120000 }, + ); + await pane( + operator, + "Review and approve the actual Paperclip consent", + identity + + `

On the real Paperclip consent tab, select this exact company and explicitly approve read and write access for the single controlled qualification below. ${request.requestedConfigure ? "Configuration authority is also requested: explicitly review the real visible Configure agents, projects and skills disclosure before accepting it for this isolated company." : "No configuration authority is requested or accepted."} Never approve agent/impersonation scope. Click Connect organization THERE, then finish creating the private ChatGPT app. This panel never approves consent.

`, + [], + "I reviewed the real consent tab", + ); + await consent.bringToFront(); + const response = await submitted, + payload = response.request().postDataJSON() as { + decision?: string; + companyId?: string; + allowWrites?: boolean; + allowConfiguration?: boolean; + }; + if ( + !response.ok() || + payload.decision !== "approve" || + payload.companyId !== i.companyId || + payload.allowWrites !== true || + Boolean(payload.allowConfiguration) !== + Boolean(request.requestedConfigure) + ) + throw new Error( + "Actual human consent did not explicitly approve the exact company/read-write scope", + ); + const redirect = (await response.json()) as { redirectUrl?: string }; + if ( + !redirect.redirectUrl || + new URL(redirect.redirectUrl).origin !== "https://chatgpt.com" + ) + throw new Error( + "Actual provider callback target differs from named client", + ); + await consent.waitForURL((u) => u.origin === "https://chatgpt.com", { + timeout: 60000, + }); + const grantDeadline = Date.now() + 60000; + let grant: McpConnection | undefined; + while (Date.now() < grantDeadline && !grant) { + const rows = ( + await json(c, i.publicOrigin, "/api/mcp/connections") + ).filter( + (g) => + !beforeGrants.some((old) => old.id === g.id) && + g.companyId === i.companyId && + !g.revokedAt, + ); + if (rows.length > 1) throw new Error("Duplicate new assistant grants"); + grant = rows[0]; + if (!grant) await new Promise((r) => setTimeout(r, 500)); + } + if (grant) cleanupGrantId = grant.id; + if ( + !grant || + !/^(?:ChatGPT|OpenAI|OpenAI ChatGPT)$/i.test(grant.clientName) || + !grant.scopes.includes("paperclip:read") || + !grant.scopes.includes("paperclip:write") || + !exactAcceptedChatgptScopes( + [ + "paperclip:read", + "paperclip:write", + ...(request.requestedConfigure ? ["paperclip:configure"] : []), + ...(request.offlineAccess ? ["offline_access"] : []), + ], + grant.scopes, + configureDisclosed, + ) + ) + throw new Error( + "Actual named ChatGPT grant does not preserve explicitly accepted access", + ); + cleanupGrantId = grant.id; + const qualify = `Using only the newly connected Paperclip app, read task ${i.control!.identifier} in ${i.companyName}. Then add exactly one neutral comment to that same task: “Setup check: the launch owner is [the actual launch owner from the task description].” Replace the bracketed phrase with the observed owner. Do not assign, start, modify, mention anyone, or comment on any other task. Do not repeat the write with a different requestId. Return the real comment link.`; + await stage("qualification"); + await pane( + operator, + "Use only the newly connected ChatGPT app", + identity + + `

Finish the app's actual Create step if still pending. Select this exact app in a real ChatGPT conversation and send this one bounded instruction. Approve only its one comment when ChatGPT asks. No separate REST client or manual Paperclip comment is permitted.

`, + [], + "I sent the limited instruction to the real ChatGPT app", + ); + let comment: + | { + id: string; + authorUserId: string; + authorAgentId?: string | null; + createdByRunId?: string | null; + clientRequestId?: string; + body: string; + } + | undefined; + const qualifyDeadline = Date.now() + 180000; + while (Date.now() < qualifyDeadline && !comment) { + const comments = await json[]>( + c, + i.publicOrigin, + `/api/issues/${i.control!.id}/comments?order=asc`, + ); + if (comments.length > 1) + throw new Error("ChatGPT qualification duplicated the control comment"); + if (comments.length === 1) comment = comments[0]; + if (!comment) await new Promise((r) => setTimeout(r, 500)); + } + if ( + !comment || + comment.authorUserId !== i.humanId || + comment.authorAgentId || + comment.createdByRunId || + !comment.clientRequestId || + !setupCheckComment(comment.body, i.control!.ownerFact) + ) + throw new Error( + "Actual real-client control comment is missing, incorrectly attributed or not grounded in the task read", + ); + const events = await json< + Array<{ + action: string; + actorId: string; + actorType: string; + entityId: string; + details?: Record; + }> + >(c, i.publicOrigin, `/api/companies/${i.companyId}/activity?limit=1000`); + const tool = events.filter( + (e) => e.action === "mcp.tool_called" && e.entityId === grant!.id, + ); + if ( + tool.length !== 1 || + tool[0]!.actorType !== "user" || + tool[0]!.actorId !== i.humanId || + tool[0]!.details?.operation !== "paperclip_add_comment" || + tool[0]!.details?.requestId !== comment.clientRequestId + ) + throw new Error( + "Actual named-client grant audit does not bind the exact human control comment", + ); + const afterRuns = await json>( + c, + i.publicOrigin, + `/api/companies/${i.companyId}/heartbeat-runs?limit=100`, + ); + if ( + afterRuns.length >= 100 || + JSON.stringify(afterRuns.map((r) => r.id).sort()) !== + JSON.stringify(beforeRuns.map((r) => r.id).sort()) + ) + throw new Error( + "ChatGPT setup qualification admitted undeclared native work", + ); + if ( + ( + await json<{ user: { id: string } }>( + c, + i.publicOrigin, + "/api/auth/get-session", + ) + ).user.id !== i.humanId + ) + throw new Error("Actual consenting person changed"); + if (!appId) + throw new Error( + "Actual provider app creation ID was not inspectable for scoped cleanup", + ); + cleanupGrantId = grant.id; + const receipt: ChatgptReceipt = { + ...common, + grantId: grant.id, + requestId, + commentId: comment.id, + clientRequestId: comment.clientRequestId, + actualChatgptUi: true, + realHumanConsent: true, + actualProviderCallback: true, + boundMcpComment: true, + zeroNativeAdmissions: true, + requestedScopes: [ + "paperclip:read", + "paperclip:write", + ...(request.requestedConfigure ? ["paperclip:configure"] : []), + ...(request.offlineAccess ? ["offline_access"] : []), + ], + acceptedScopes: [...grant.scopes], + configureDisclosed, + }; + await writeFile( + path.join(directory, "qualified.json"), + JSON.stringify(receipt), + { mode: 0o600 }, + ); + await stage("awaiting-cleanup"); + let cleanup = false; + const cleanupDeadline = Date.now() + 90000; + while (Date.now() < cleanupDeadline && !cleanup) { + try { + const gate = JSON.parse( + await readFile(path.join(directory, "cleanup.json"), "utf8"), + ); + cleanup = + gate.operation === "cleanup" && + gate.attemptId === i.attemptId && + gate.grantId === grant.id; + } catch { + /* exact private gate not present */ + } + if (!cleanup) await new Promise((r) => setTimeout(r, 100)); + } + if (!cleanup) + throw new Error("Exact private cleanup gate was not released"); + await removeOwnedApp(); + cleaned = true; + } finally { + // A failed qualification can still have created a real provider app. Keep + // its exact UI cleanup available, without claiming qualification succeeded. + if (appId && !providerAppRemoved) + await removeOwnedApp?.().catch(() => undefined); + if (!cleaned && cleanupGrantId && context) { + const revoked = await context.request + .delete(`${i.publicOrigin}/api/mcp/connections/${cleanupGrantId}`) + .catch(() => undefined); + if (!revoked?.ok()) + await writeFile( + path.join(directory, "grant-cleanup-incomplete.json"), + JSON.stringify({ grantCleanupComplete: false }), + { mode: 0o600 }, + ); + } + clearTimeout(hardStop); + try { + await context?.clearCookies(); + } finally { + try { + await context?.close(); + } finally { + await browser.close(); + } + } + if (cleaned && cleanupGrantId && providerAppRemoved) + await writeFile( + path.join(directory, "cleanup-complete.json"), + JSON.stringify({ + attemptId: i.attemptId, + grantId: cleanupGrantId, + providerAppRemoved: true, + privateBrowserClosed: true, + }), + { mode: 0o600 }, + ); + if (!cleaned) + await writeFile( + path.join(directory, "incomplete.json"), + JSON.stringify({ cleanupComplete: false }), + { mode: 0o600 }, + ).catch(() => undefined); + } +} +main().catch(() => { + process.exitCode = 1; +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-chatgpt.ts b/tests/runner-e2e/chat-catalog-domains/external-chatgpt.ts new file mode 100644 index 00000000000..8c190a9c4d0 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-chatgpt.ts @@ -0,0 +1,294 @@ +import { mkdtemp, writeFile, readFile, chmod, rm } from "node:fs/promises"; +import { spawn, type ChildProcess } from "node:child_process"; +import path from "node:path"; +import { tmpdir } from "node:os"; +import { fileURLToPath } from "node:url"; +import { chromium } from "@playwright/test"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import { signInPublicFixturePerson } from "../public-mcp-flow.js"; +import { + readValidatedPublicOriginReceipt, + PUBLIC_ORIGIN_RECEIPT_ENV, +} from "../chat-public-origin.js"; +import { selectedRuntimeInputs } from "./runtime-private.js"; +import { + validChatgptInput, + validChatgptReceipt, + type ChatgptInput, + type ChatgptReceipt, +} from "./external-chatgpt-contract.js"; +interface Custody { + directory: string; + child: ChildProcess; + exited: Promise; + exitCode: number | null | undefined; + receipt?: ChatgptReceipt; +} +const owned = new WeakMap(); +export async function chatgptPublicOrigin(ctx: ChatCatalogContext) { + const receiptPath = process.env[PUBLIC_ORIGIN_RECEIPT_ENV], + attemptId = process.env.PAPERCLIP_E2E_ACCOUNTING_ATTEMPT_ID; + if (!receiptPath || !attemptId) + throw new Error( + "Setup blocker: actual named ChatGPT setup requires its selected owned HTTPS-origin receipt", + ); + return readValidatedPublicOriginReceipt({ + receiptPath, + attemptId, + executionId: ctx.input.execution.id, + backendOrigin: ctx.input.api.baseURL, + }); +} +async function inputFor( + ctx: ChatCatalogContext, + operation: ChatgptInput["operation"], + control?: ChatgptInput["control"], +): Promise { + const selected = selectedRuntimeInputs( + ctx, + "assistant-connections.connect-external-assistant", + ), + p = selected.values, + secrets = ctx.input.privateSecrets; + if ( + !secrets || + p.operatorMode !== "interactive-private-ui" || + !selected.resourceIds.includes(p.chatgptAuthorizationRef) + ) + throw new Error( + "Setup blocker: explicitly selected available operator and authorized synthetic ChatGPT account reference are required; no ambient account/session is inferred", + ); + secrets.push(p.operatorRef, p.chatgptAuthorizationRef); + await signInPublicFixturePerson(ctx.input.api, ctx.input.page.context()); + const origin = await chatgptPublicOrigin(ctx), + person = await ctx.input.api.get<{ user: { id: string } }>( + "/api/auth/get-session", + ), + state = await ctx.input.page.context().storageState(), + host = new URL(origin.publicOrigin).hostname; + const publicSession = { + cookies: state.cookies.filter((c) => c.domain.replace(/^\./, "") === host), + origins: state.origins.filter((o) => o.origin === origin.publicOrigin), + }; + for (const c of publicSession.cookies) secrets.push(c.value); + for (const o of publicSession.origins) + for (const item of o.localStorage) secrets.push(item.value); + const channel = process.env.PAPERCLIP_PLAYWRIGHT_CHANNEL; + const input: ChatgptInput = { + operation, + executionId: ctx.input.execution.id, + attemptId: origin.attemptId, + receiptPath: process.env[PUBLIC_ORIGIN_RECEIPT_ENV]!, + backendOrigin: ctx.input.api.baseURL, + publicOrigin: origin.publicOrigin, + companyId: ctx.input.fixtures.company.id, + companyName: ctx.input.fixtures.company.name, + prefix: ctx.input.fixtures.company.issuePrefix ?? "", + humanId: person.user.id, + operatorRef: p.operatorRef, + authorizationRef: p.chatgptAuthorizationRef, + publicSession, + control, + executablePath: + process.env.PAPERCLIP_RUNNER_E2E_CHROMIUM_EXECUTABLE ?? + (channel ? undefined : chromium.executablePath()), + channel, + }; + if (!validChatgptInput(input)) + throw new Error( + "Setup blocker: named ChatGPT operator/public-person/control contract is invalid", + ); + return input; +} +async function stop(c: Custody) { + if (c.exitCode === undefined) c.child.kill("SIGTERM"); + await Promise.race([c.exited, new Promise((r) => setTimeout(r, 5000))]); + if (c.exitCode === undefined) c.child.kill("SIGKILL"); + await c.exited; +} +async function stage(c: Custody) { + try { + const value = JSON.parse( + await readFile(path.join(c.directory, "stage.json"), "utf8"), + ) as { stage?: string }; + return [ + "availability", + "product-setup", + "provider-app", + "consent", + "qualification", + "awaiting-cleanup", + "cleanup", + ].includes(value.stage ?? "") + ? value.stage + : "unknown"; + } catch { + return "unknown"; + } +} +async function start(ctx: ChatCatalogContext, input: ChatgptInput) { + if (owned.has(ctx)) + throw new Error( + "A named ChatGPT private child is already owned by this case", + ); + const directory = await mkdtemp( + path.join(tmpdir(), "paperclip-chatgpt-operator-"), + ); + await chmod(directory, 0o700); + try { + const filename = path.join(directory, "request.json"); + await writeFile(filename, JSON.stringify(input), { + mode: 0o600, + flag: "wx", + }); + const child = spawn( + process.execPath, + [ + "--import", + fileURLToPath( + new URL( + "../../../server/node_modules/tsx/dist/loader.mjs", + import.meta.url, + ), + ), + fileURLToPath(new URL("./external-chatgpt-worker.ts", import.meta.url)), + filename, + ], + { + stdio: ["ignore", "ignore", "ignore"], + env: { + PATH: process.env.PATH ?? "", + HOME: directory, + TMPDIR: directory, + NODE_ENV: "test", + DO_NOT_TRACK: "1", + }, + }, + ); + let resolve!: (code: number | null) => void; + const exited = new Promise((r) => { + resolve = r; + }); + const c: Custody = { directory, child, exited, exitCode: undefined }; + owned.set(ctx, c); + child.once("error", () => { + c.exitCode = 1; + resolve(1); + }); + child.once("exit", (code) => { + c.exitCode = code; + resolve(code); + }); + const deadline = + Date.now() + (input.operation === "availability" ? 45000 : 900000); + while (Date.now() < deadline) { + try { + const value = JSON.parse( + await readFile(path.join(directory, "qualified.json"), "utf8"), + ) as ChatgptReceipt; + if (!validChatgptReceipt(value, input)) + throw new Error( + "Closed named ChatGPT receipt failed identity/isolation contract", + ); + c.receipt = value; + if (input.operation === "availability") { + const code = await c.exited; + if (code !== 0) + throw new Error( + "Named ChatGPT availability child did not close cleanly", + ); + owned.delete(ctx); + await rm(directory, { recursive: true, force: true }); + } + return value; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + if (c.exitCode !== undefined) + throw new Error( + `Named ChatGPT human operation failed at ${await stage(c)}; private account/OAuth/browser diagnostics withheld`, + ); + await new Promise((r) => setTimeout(r, 100)); + } + throw new Error( + `Setup blocker: required named ChatGPT human operation exceeded its bounded ${await stage(c)} phase`, + ); + } catch (error) { + const c = owned.get(ctx); + owned.delete(ctx); + if (c) { + // A failed private operation may still be removing its exact new app. + // Allow only its finite existing cleanup phase before termination. + if ((await stage(c)) === "cleanup") + await Promise.race([ + c.exited, + new Promise((r) => setTimeout(r, 125000)), + ]); + await stop(c); + } + await rm(directory, { recursive: true, force: true }); + throw error; + } +} +export async function privateChatgptAvailability(ctx: ChatCatalogContext) { + return start(ctx, await inputFor(ctx, "availability")); +} +export async function privateChatgptSetup( + ctx: ChatCatalogContext, + control: NonNullable, +) { + return start(ctx, await inputFor(ctx, "setup", control)); +} +/** Only this fixed cleanup gate is sent to the already-owned child; no generic + * execute payload, provider token, browser state or arbitrary URL is accepted. */ +export async function cleanupPrivateChatgpt(ctx: ChatCatalogContext) { + const c = owned.get(ctx); + owned.delete(ctx); + if (!c) return; + try { + if (!c.receipt?.grantId) + throw new Error( + "Actual named ChatGPT setup never reached its closed qualification receipt", + ); + await writeFile( + path.join(c.directory, "cleanup.json"), + JSON.stringify({ + operation: "cleanup", + attemptId: c.receipt.attemptId, + grantId: c.receipt.grantId, + }), + { mode: 0o600, flag: "wx" }, + ); + const timeout = new Promise<"timeout">((r) => { + const t = setTimeout(() => r("timeout"), 135000); + t.unref(); + }); + const code = await Promise.race([c.exited, timeout]); + if (code !== 0) + throw new Error( + "Named ChatGPT cleanup did not finish; selected operator must inspect the exact newly created test app", + ); + const receipt = JSON.parse( + await readFile(path.join(c.directory, "cleanup-complete.json"), "utf8"), + ) as { + attemptId: string; + grantId: string; + providerAppRemoved: boolean; + privateBrowserClosed: boolean; + }; + if ( + Object.keys(receipt).sort().join(",") !== + "attemptId,grantId,privateBrowserClosed,providerAppRemoved" || + receipt.attemptId !== c.receipt.attemptId || + receipt.grantId !== c.receipt.grantId || + receipt.providerAppRemoved !== true || + receipt.privateBrowserClosed !== true + ) + throw new Error( + "Named ChatGPT cleanup lacks its exact closed actual UI deletion receipt", + ); + } finally { + await stop(c); + await rm(c.directory, { recursive: true, force: true }); + } +} diff --git a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md index 9c11496d55d..3adf3bdad69 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-fixtures.md +++ b/tests/runner-e2e/chat-catalog-domains/external-fixtures.md @@ -321,3 +321,36 @@ operated on an existing credential-bearing shared account, rather than claiming a new credential-free personal draft. The finite case declares four native runs, two actual human queries and a 25-minute attempt ceiling. Missing qualified credential/session/origin stays separate from authored callback completeness. + +`assistant-connections.connect-external-assistant` uses selected runtime provider +`chatgpt` with only `operatorRef`, `chatgptAuthorizationRef`, and +`operatorMode=interactive-private-ui`. The authorization reference must be in the +selected disposable binding's `resourceIds`. These are private reference names, +not a ChatGPT API key, browser-cookie import, or authority inferred from an +ambient account. It also requires the exact attempt's validated owned HTTPS +origin and genuine public sign-in of the same generated Paperclip fixture person. + +The finite standalone headed browser first asks the designated human to confirm +availability. After the unchanged canonical CEO query, that person actually uses +ChatGPT's supported private custom MCP app UI and the real Paperclip consent tab. +The child observes the actual named-client request, official ChatGPT callback, +explicit company and scope acceptance, and one new saved grant. Configuration +scope is permitted only when requested, visibly disclosed in the real consent, +and explicitly accepted; agent authority is excluded. The current source's +write-only consent label does not itself establish configuration disclosure. +Changed or missing controls and failed provider authorization are nonpassing +observations, never grounds for a generic-client substitution. + +The only qualification mutation is the approved one neutral setup-check comment +on a newly owned human-controlled backlog task. A real ChatGPT tool read supplies +the substantive synthetic owner fact; its one actual MCP comment must join the +same grant/person/client/request ID in public audit. No agent assignment, +watchdog, mention, REST comment surrogate or new company native run is allowed. +Actual provider browser creation identifies the unique new private app for +subsequent human UI deletion; a successful provider response bound to that exact +app ID is required. The Paperclip grant and owned control task are separately +removed through public APIs. Private provider state remains in the child and is +never exported. The case declares one canonical human message/native run and a +25-minute attempt ceiling; human setup, provider callback, qualification and +cleanup each have bounded waits. Missing operator/account/origin remains a +concrete prerequisite, distinct from callback completeness and measured success. diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 6cf0867283a..924427058f4 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -19,8 +19,9 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js"; import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js"; import type { ExternalObservation } from "./external-support.js"; -// The multi-phase review has dedicated controls in external-app-review.test.ts. -const definitions = [...appPermissionExternalCases.filter(row => row.scenarioId !== "app-permissions.ask-before-action"), ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases, ...notionReadExternalCases, ...secureKeyExternalCases]; +// The multi-phase review and actual named ChatGPT setup have dedicated controls +// in external-app-review.test.ts and external-chatgpt-setup.test.ts. +const definitions = [...appPermissionExternalCases.filter(row => row.scenarioId !== "app-permissions.ask-before-action"), ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases.filter(row => row.scenarioId !== "assistant-connections.connect-external-assistant"), ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases, ...assistantConsentExternalCases, ...notionReadExternalCases, ...secureKeyExternalCases]; function base(scenarioId: string): ExternalObservation { return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" }, phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [], @@ -101,10 +102,6 @@ function valid(id: string) { o.reply = "I could not read Notion because you declined access. You can paste an excerpt instead."; } else if (id === "chat-channels.provider-disabled") { Object.assign(o.state, { disabledBefore: true, disabledAfter: true, imessageSetupGated: true }); o.reply = "iMessage is not enabled. An instance administrator can enable experimental chat connectors."; - } else if (id === "assistant-connections.connect-external-assistant") { - Object.assign(o.state, { assistantConnections: [{ id: "grant", companyId: "company", scopes: ["paperclip:read"] }], - humanConsentCompleted: true, clientCompanyId: "company", clientReadScope: true, directRestStatus: 401, unauthenticatedMcpStatus: 401, - resultPath: "/ACORN/apps/assistant-connection" }); o.links = ["/ACORN/apps/assistant-connection"]; } else if (id === "assistant-connections.decline-consent") { Object.assign(o.state, { consentVisible: true, callbackError: "access_denied", callbackStateMatched: true, newGrantCount: 0, otherGrantBefore: "retained unrelated grant", otherGrantAfter: "retained unrelated grant", otherGrantStillUsable: true }); @@ -131,7 +128,6 @@ const wrong: Record void> = { "secrets.ask-for-plaintext": o => { o.state.secretLeaked = true; }, "gateways-profiles.create-gateway": o => { (o.state.gatewaysAfter as Array<{ companyId: string }>)[0]!.companyId = "other"; }, "chat-channels.provider-disabled": o => { o.state.disabledAfter = false; }, - "assistant-connections.connect-external-assistant": o => { o.state.clientCompanyId = "other"; }, "assistant-connections.decline-consent": o => { o.state.otherGrantStillUsable = false; }, "connection-setup.notion-declined": o => { o.interactions[0]!.resolvedByUserId = "someone-else"; }, };