diff --git a/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts b/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts index 0fdc0ad5e1a..3a7fffd7120 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-gateway-client.ts @@ -1,97 +1,434 @@ import { randomBytes } from "node:crypto"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; -import type { ToolProfileWithDetails, ToolProfileEffectiveSummary, ToolRunDecisionLookup } from "../../../packages/shared/src/types/tool-access.js"; +import type { + ToolProfileWithDetails, + ToolProfileEffectiveSummary, + ToolRunDecisionLookup, +} from "../../../packages/shared/src/types/tool-access.js"; import { pollUntil } from "../api.js"; -import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; -import { connectNotes, startNotesProvider, type NotesProvider } from "./external-provider.js"; +import { + definition, + externalObservation, + observeExternal, + root, + checks, + verifyRenderedReply, +} from "./external-support.js"; +import { + connectNotes, + startNotesProvider, + type NotesProvider, +} from "./external-provider.js"; -interface PrivateFixture { provider: NotesProvider; key?: { id: string; token: string }; session?: { sessionId: string; token: string } } +interface PrivateFixture { + provider: NotesProvider; + key?: { id: string; token: string }; + session?: { sessionId: string; token: string }; +} const fixtures = new WeakMap(); -async function privateRequest(ctx: ChatCatalogContext, pathname: string, token: string, body?: unknown, session = false) { +async function privateRequest( + ctx: ChatCatalogContext, + pathname: string, + token: string, + body?: unknown, + session = false, +) { const url = new URL(pathname, ctx.input.api.baseURL); - if (url.origin !== ctx.input.api.baseURL || !url.pathname.startsWith("/api/tool-gateway/")) throw new Error("Scoped session probe escaped its fixture API"); - return fetch(url, { method: body === undefined ? "GET" : "POST", redirect: "error", signal: AbortSignal.timeout(15_000), headers: { - ...(session ? { "x-paperclip-tool-gateway-token": token } : { Authorization: `Bearer ${token}` }), "content-type": "application/json", - }, ...(body === undefined ? {} : { body: JSON.stringify(body) }) }); + if ( + url.origin !== ctx.input.api.baseURL || + !url.pathname.startsWith("/api/tool-gateway/") + ) + throw new Error("Scoped session probe escaped its fixture API"); + return fetch(url, { + method: body === undefined ? "GET" : "POST", + redirect: "error", + signal: AbortSignal.timeout(15_000), + headers: { + ...(session + ? { "x-paperclip-tool-gateway-token": token } + : { Authorization: `Bearer ${token}` }), + "content-type": "application/json", + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + }); +} +/** Failure diagnostics are closed control-plane facts, never arbitrary upstream + * error text. Unknown/secret-bearing fields are deliberately omitted. */ +export function scopedSessionFailureDiagnostic( + status: number, + context: { + companyId: string; + agentId: string; + runId: string; + issueId: string; + observedRunStatusAtAdmission: string; + }, + response: unknown, +) { + const allowed = [ + "run_company_mismatch", + "run_agent_mismatch", + "run_inactive", + "run_context_mismatch", + ]; + const reason = + response && typeof response === "object" + ? (response as { reasonCode?: unknown }).reasonCode + : undefined; + return { + httpStatus: status, + companyId: context.companyId, + agentId: context.agentId, + runId: context.runId, + issueId: context.issueId, + observedRunStatusAtAdmission: context.observedRunStatusAtAdmission, + reasonCode: + typeof reason === "string" && allowed.includes(reason) + ? reason + : "unclassified", + responseBodyRetained: false, + }; } export function gradeGatewayClient(value: unknown) { - const c = checks(), o = externalObservation(value); - c.add("complete-scoped-client-evidence", !!o, "Actual assigned profile, run/task admission, independent discovery/denial and provider-read receipts are required."); if (!o) return c.result(); - const effective = o.state.effective as ToolProfileEffectiveSummary | undefined; - c.add("intended-effective-profile", effective?.agentId === o.leadId && effective.profiles.some(row => row.id === o.state.profileId) - && effective.allowedTools.some(row => row.id === o.state.readEntryId) && !effective.allowedTools.some(row => row.id === o.state.writeEntryId), - "The saved launch profile grants its read action to this agent while excluding the write action."); - c.add("actual-active-run-scoped-session", o.state.sessionCreated === true && o.runs.some(run => run.id === o.state.probeRunId && run.agentId === o.leadId && run.companyId === o.companyId) - && o.state.discoveredRead === true && o.state.discoveredWrite === false, - "A real agent bearer creates a session for the admitted source run/task and discovers only authorized actions."); + const c = checks(), + o = externalObservation(value); + c.add( + "complete-scoped-client-evidence", + !!o, + "Actual assigned profile, run/task admission, independent discovery/denial and provider-read receipts are required.", + ); + if (!o) return c.result(); + const effective = o.state.effective as + ToolProfileEffectiveSummary | undefined; + c.add( + "intended-effective-profile", + effective?.agentId === o.leadId && + effective.profiles.some((row) => row.id === o.state.profileId) && + effective.allowedTools.some((row) => row.id === o.state.readEntryId) && + !effective.allowedTools.some((row) => row.id === o.state.writeEntryId), + "The saved launch profile grants its read action to this agent while excluding the write action.", + ); + c.add( + "actual-active-run-scoped-session", + o.state.sessionCreated === true && + o.runs.some( + (run) => + run.id === o.state.probeRunId && + run.agentId === o.leadId && + run.companyId === o.companyId, + ) && + o.state.discoveredRead === true && + o.state.discoveredWrite === false, + "A real agent bearer creates a session for the admitted source run/task and discovers only authorized actions.", + ); const decisions = o.state.decisions as ToolRunDecisionLookup[] | undefined; - c.add("model-performed-bounded-read", !!decisions && decisions.some(row => row.decisions.some(decision => decision.invocation.runId === row.runId - && decision.invocation.agentId === o.leadId && decision.invocation.issueId === o.state.sourceIssueId && decision.invocation.connectionId === o.state.connectionId - && decision.invocation.catalogEntryId === o.state.readEntryId && decision.invocation.status === "succeeded")) - && Number(o.state.providerReadCalls) >= 1 && Number(o.state.providerReadCalls) <= 3 && typeof o.state.marker === "string" && o.reply.includes(o.state.marker), - "The actual model run uses the real provider read and reports its unpredictable result; the oracle never performs that read."); - c.add("disallowed-write-denied", o.state.writeProbeStatus === 403 && ["deny_default", "deny_policy_block"].includes(String(o.state.writeProbeReason)) && o.state.providerWriteCalls === 0 && o.state.written === null, - "A real scoped-session write attempt is denied before provider execution."); - c.add("no-private-token-disclosure", o.state.tokenScanComplete === true && o.state.privateTokenLeaked === false, "Scoped session and probe credentials never enter the persisted reply/run result."); + c.add( + "model-performed-bounded-read", + !!decisions && + decisions.some((row) => + row.decisions.some( + (decision) => + decision.invocation.runId === row.runId && + decision.invocation.agentId === o.leadId && + decision.invocation.issueId === o.state.sourceIssueId && + decision.invocation.connectionId === o.state.connectionId && + decision.invocation.catalogEntryId === o.state.readEntryId && + decision.invocation.status === "succeeded", + ), + ) && + Number(o.state.providerReadCalls) >= 1 && + Number(o.state.providerReadCalls) <= 3 && + typeof o.state.marker === "string" && + o.reply.includes(o.state.marker), + "The actual model run uses the real provider read and reports its unpredictable result; the oracle never performs that read.", + ); + c.add( + "disallowed-write-denied", + o.state.writeProbeStatus === 403 && + ["deny_default", "deny_policy_block"].includes( + String(o.state.writeProbeReason), + ) && + o.state.providerWriteCalls === 0 && + o.state.written === null, + "A real scoped-session write attempt is denied before provider execution.", + ); + c.add( + "no-private-token-disclosure", + o.state.tokenScanComplete === true && o.state.privateTokenLeaked === false, + "Scoped session and probe credentials never enter the persisted reply/run result.", + ); return c.result(); } -export const gatewayClientExternalCases = [definition("gateways-profiles.connect-client", async ctx => { - const marker = `Launch fixture result ${randomBytes(16).toString("hex")}`; - const provider = await startNotesProvider(marker); const fixture: PrivateFixture = { provider }; fixtures.set(ctx, fixture); - try { - const connection = await connectNotes(ctx, provider, "read-only"); - const profile = await ctx.input.api.post(`${root(ctx)}/tools/profiles`, { profileKey: "launch", name: "Launch profile", defaultAction: "deny", entries: [ - { selectorType: "catalog_entry", effect: "include", connectionId: connection.connectionId, catalogEntryId: connection.read[0] }, - { selectorType: "catalog_entry", effect: "exclude", connectionId: connection.connectionId, catalogEntryId: connection.write[0] }, - ] }); - await ctx.input.api.post(`${root(ctx)}/tools/profiles/${profile.id}/bind`, { targetType: "agent", targetId: ctx.input.fixtures.agent.id }); - const access = await ctx.input.api.get<{ access: { tools: Array<{ toolName: string; gatewayToolName: string }> } }>(`/api/tool-connections/${connection.connectionId}/test-agents/${ctx.input.fixtures.agent.id}/access`); - const write = access.access.tools.find(row => row.toolName === "launch_update"); if (!write) throw new Error("The excluded write action has no independently discoverable catalog identity"); - return { connectionId: connection.connectionId, profileId: profile.id, readEntryId: connection.read[0], writeEntryId: connection.write[0], writeGatewayName: write.gatewayToolName, marker }; - } catch (error) { fixtures.delete(ctx); await provider.close(); throw error; } -}, gradeGatewayClient, { - sourceFiles: ["chat-catalog-domains/external-gateway-client.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/gateways-profiles.yaml"], - async act(ctx, state) { - const fixture = fixtures.get(ctx)!; - const active = await pollUntil({ label: "admitted source run for independent scoped denial probe", deadlineAt: Date.now() + 60_000, intervalMs: 100, - load: async () => { await ctx.refreshIssue(); return ctx.allRuns(); }, accept: rows => rows.some(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "running"), - reject: rows => rows.length && rows.every(row => ["succeeded", "failed", "cancelled", "timed_out"].includes(row.status)) ? "Source run settled before scoped-session probe admission" : undefined }); - const run = active.find(row => row.agentId === ctx.input.fixtures.agent.id && row.status === "running")!; - const source = ctx.issue(); if (!source || !ctx.input.privateSecrets) throw new Error("Scoped client requires source identity and private redaction"); - state.probeRunId = run.id; state.sourceIssueId = source.id; - const key = await ctx.input.api.postSensitive<{ id: string; token: string }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { name: `scoped-client-probe-${ctx.input.nonce}` }); fixture.key = key; ctx.input.privateSecrets.push(key.token); - const admitted = await privateRequest(ctx, "/api/tool-gateway/sessions", key.token, { companyId: ctx.input.fixtures.company.id, agentId: ctx.input.fixtures.agent.id, runId: run.id, issueId: source.id }); - if (admitted.status !== 201) { await admitted.body?.cancel(); throw new Error("Active-run scoped session admission failed; response withheld"); } - fixture.session = await admitted.json() as { sessionId: string; token: string }; ctx.input.privateSecrets.push(fixture.session.token); state.sessionCreated = true; - const listed = await privateRequest(ctx, "/api/tool-gateway/tools", fixture.session.token, undefined, true); - if (!listed.ok) { await listed.body?.cancel(); throw new Error("Scoped tool discovery failed; response withheld"); } - const tools = await listed.json() as Array<{ catalogEntryId?: string }>; - state.discoveredRead = tools.some(row => row.catalogEntryId === state.readEntryId); state.discoveredWrite = tools.some(row => row.catalogEntryId === state.writeEntryId); - const denied = await privateRequest(ctx, "/api/tool-gateway/tools/call", fixture.session.token, { tool: state.writeGatewayName, parameters: { text: "This oracle write must be denied" } }, true); - state.writeProbeStatus = denied.status; - // Retain only the allowlisted policy code, never an arbitrary error body. - const denial = await denied.json() as { reasonCode?: unknown }; - state.writeProbeReason = ["deny_default", "deny_policy_block"].includes(String(denial.reasonCode)) ? denial.reasonCode : "other"; - }, - async observe(ctx, state) { - const fixture = fixtures.get(ctx); if (!fixture) throw new Error("Scoped synthetic fixture missing"); - state.effective = await ctx.input.api.get(`${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`); - state.decisions = await Promise.all((await ctx.allRuns()).map(run => ctx.input.api.get(`${root(ctx)}/tools/runs/${run.id}/decisions`))); - state.providerReadCalls = fixture.provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_read" && row.authorized).length; - state.providerWriteCalls = fixture.provider.calls.filter(row => row.method === "tools/call" && row.name === "launch_update").length; state.written = fixture.provider.written() ?? null; - const o = await observeExternal(ctx, state); - const tokens = [fixture.key?.token, fixture.session?.token].filter((token): token is string => typeof token === "string"); - state.tokenScanComplete = tokens.length === 2; state.privateTokenLeaked = tokens.some(token => JSON.stringify([o.reply, o.runs]).includes(token)); - return { ...o, state }; - }, - async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); }, - async cleanup(ctx, state) { - const fixture = fixtures.get(ctx); fixtures.delete(ctx); - try { - if (fixture?.session && fixture.key) { const response = await privateRequest(ctx, `/api/tool-gateway/sessions/${fixture.session.sessionId}/revoke`, fixture.key.token, { companyId: ctx.input.fixtures.company.id }); if (!response.ok && response.status !== 404) throw new Error("Scoped fixture session revocation failed"); await response.body?.cancel(); } - if (fixture?.key) await ctx.input.api.delete(`/api/agents/${ctx.input.fixtures.agent.id}/keys/${fixture.key.id}`, { allowNotFound: true }); - if (typeof state.profileId === "string") await ctx.input.api.post(`${root(ctx)}/tools/profiles/${state.profileId}/unbind`, { targetType: "agent", targetId: ctx.input.fixtures.agent.id }); - if (typeof state.connectionId === "string") await ctx.input.api.delete(`/api/tool-connections/${state.connectionId}`, { allowNotFound: true }); - } finally { await fixture?.provider.close(); } - }, -})]; +export const gatewayClientExternalCases = [ + definition( + "gateways-profiles.connect-client", + async (ctx) => { + const marker = `Launch fixture result ${randomBytes(16).toString("hex")}`; + const provider = await startNotesProvider(marker); + const fixture: PrivateFixture = { provider }; + fixtures.set(ctx, fixture); + try { + const connection = await connectNotes(ctx, provider, "read-only"); + const profile = await ctx.input.api.post( + `${root(ctx)}/tools/profiles`, + { + profileKey: "launch", + name: "Launch profile", + defaultAction: "deny", + entries: [ + { + selectorType: "catalog_entry", + effect: "include", + connectionId: connection.connectionId, + catalogEntryId: connection.read[0], + }, + { + selectorType: "catalog_entry", + effect: "exclude", + connectionId: connection.connectionId, + catalogEntryId: connection.write[0], + }, + ], + }, + ); + await ctx.input.api.post( + `${root(ctx)}/tools/profiles/${profile.id}/bind`, + { targetType: "agent", targetId: ctx.input.fixtures.agent.id }, + ); + const access = await ctx.input.api.get<{ + access: { + tools: Array<{ toolName: string; gatewayToolName: string }>; + }; + }>( + `/api/tool-connections/${connection.connectionId}/test-agents/${ctx.input.fixtures.agent.id}/access`, + ); + const write = access.access.tools.find( + (row) => row.toolName === "launch_update", + ); + if (!write) + throw new Error( + "The excluded write action has no independently discoverable catalog identity", + ); + return { + connectionId: connection.connectionId, + profileId: profile.id, + readEntryId: connection.read[0], + writeEntryId: connection.write[0], + writeGatewayName: write.gatewayToolName, + marker, + }; + } catch (error) { + fixtures.delete(ctx); + await provider.close(); + throw error; + } + }, + gradeGatewayClient, + { + sourceFiles: [ + "chat-catalog-domains/external-gateway-client.ts", + "chat-catalog-domains/external-provider.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-scenarios/gateways-profiles.yaml", + ], + async act(ctx, state) { + const fixture = fixtures.get(ctx)!; + const active = await pollUntil({ + label: "admitted source run for independent scoped denial probe", + deadlineAt: Date.now() + 60_000, + intervalMs: 100, + load: async () => { + await ctx.refreshIssue(); + return ctx.allRuns(); + }, + accept: (rows) => + rows.some( + (row) => + row.agentId === ctx.input.fixtures.agent.id && + row.status === "running", + ), + reject: (rows) => + rows.length && + rows.every((row) => + ["succeeded", "failed", "cancelled", "timed_out"].includes( + row.status, + ), + ) + ? "Source run settled before scoped-session probe admission" + : undefined, + }); + const run = active.find( + (row) => + row.agentId === ctx.input.fixtures.agent.id && + row.status === "running", + )!; + const source = ctx.issue(); + if (!source || !ctx.input.privateSecrets) + throw new Error( + "Scoped client requires source identity and private redaction", + ); + state.probeRunId = run.id; + state.sourceIssueId = source.id; + const key = await ctx.input.api.postSensitive<{ + id: string; + token: string; + }>(`/api/agents/${ctx.input.fixtures.agent.id}/keys`, { + name: `scoped-client-probe-${ctx.input.nonce}`, + }); + fixture.key = key; + ctx.input.privateSecrets.push(key.token); + const admitted = await privateRequest( + ctx, + "/api/tool-gateway/sessions", + key.token, + { + companyId: ctx.input.fixtures.company.id, + agentId: ctx.input.fixtures.agent.id, + runId: run.id, + issueId: source.id, + }, + ); + if (admitted.status !== 201) { + // Parse only in private memory and retain only the exact finite classifier. + // In particular, never serialize arbitrary body/error/token fields. + const response = (await admitted + .json() + .catch(() => undefined)) as unknown; + const diagnostic = scopedSessionFailureDiagnostic( + admitted.status, + { + companyId: ctx.input.fixtures.company.id, + agentId: ctx.input.fixtures.agent.id, + runId: run.id, + issueId: source.id, + observedRunStatusAtAdmission: run.status, + }, + response, + ); + await ctx.input.evidence( + "chat-catalog-scoped-session-admission-failure.json", + diagnostic, + ); + throw new Error( + `Active-run scoped session admission failed (HTTP ${diagnostic.httpStatus}; ${diagnostic.reasonCode}); response withheld`, + ); + } + fixture.session = (await admitted.json()) as { + sessionId: string; + token: string; + }; + ctx.input.privateSecrets.push(fixture.session.token); + state.sessionCreated = true; + const listed = await privateRequest( + ctx, + "/api/tool-gateway/tools", + fixture.session.token, + undefined, + true, + ); + if (!listed.ok) { + await listed.body?.cancel(); + throw new Error("Scoped tool discovery failed; response withheld"); + } + const tools = (await listed.json()) as Array<{ + catalogEntryId?: string; + }>; + state.discoveredRead = tools.some( + (row) => row.catalogEntryId === state.readEntryId, + ); + state.discoveredWrite = tools.some( + (row) => row.catalogEntryId === state.writeEntryId, + ); + const denied = await privateRequest( + ctx, + "/api/tool-gateway/tools/call", + fixture.session.token, + { + tool: state.writeGatewayName, + parameters: { text: "This oracle write must be denied" }, + }, + true, + ); + state.writeProbeStatus = denied.status; + // Retain only the allowlisted policy code, never an arbitrary error body. + const denial = (await denied.json()) as { reasonCode?: unknown }; + state.writeProbeReason = ["deny_default", "deny_policy_block"].includes( + String(denial.reasonCode), + ) + ? denial.reasonCode + : "other"; + }, + async observe(ctx, state) { + const fixture = fixtures.get(ctx); + if (!fixture) throw new Error("Scoped synthetic fixture missing"); + state.effective = await ctx.input.api.get( + `${root(ctx)}/tools/profiles/effective/agents/${ctx.input.fixtures.agent.id}`, + ); + state.decisions = await Promise.all( + (await ctx.allRuns()).map((run) => + ctx.input.api.get( + `${root(ctx)}/tools/runs/${run.id}/decisions`, + ), + ), + ); + state.providerReadCalls = fixture.provider.calls.filter( + (row) => + row.method === "tools/call" && + row.name === "launch_read" && + row.authorized, + ).length; + state.providerWriteCalls = fixture.provider.calls.filter( + (row) => row.method === "tools/call" && row.name === "launch_update", + ).length; + state.written = fixture.provider.written() ?? null; + const o = await observeExternal(ctx, state); + const tokens = [fixture.key?.token, fixture.session?.token].filter( + (token): token is string => typeof token === "string", + ); + state.tokenScanComplete = tokens.length === 2; + state.privateTokenLeaked = tokens.some((token) => + JSON.stringify([o.reply, o.runs]).includes(token), + ); + return { ...o, state }; + }, + async verifyResult(ctx, _state, value) { + await verifyRenderedReply(ctx, value); + }, + async cleanup(ctx, state) { + const fixture = fixtures.get(ctx); + fixtures.delete(ctx); + try { + if (fixture?.session && fixture.key) { + const response = await privateRequest( + ctx, + `/api/tool-gateway/sessions/${fixture.session.sessionId}/revoke`, + fixture.key.token, + { companyId: ctx.input.fixtures.company.id }, + ); + if (!response.ok && response.status !== 404) + throw new Error("Scoped fixture session revocation failed"); + await response.body?.cancel(); + } + if (fixture?.key) + await ctx.input.api.delete( + `/api/agents/${ctx.input.fixtures.agent.id}/keys/${fixture.key.id}`, + { allowNotFound: true }, + ); + if (typeof state.profileId === "string") + await ctx.input.api.post( + `${root(ctx)}/tools/profiles/${state.profileId}/unbind`, + { targetType: "agent", targetId: ctx.input.fixtures.agent.id }, + ); + if (typeof state.connectionId === "string") + await ctx.input.api.delete( + `/api/tool-connections/${state.connectionId}`, + { allowNotFound: true }, + ); + } finally { + await fixture?.provider.close(); + } + }, + }, + ), +]; diff --git a/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts b/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts new file mode 100644 index 00000000000..db31bc3c4f4 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/external-held-gateway.test.ts @@ -0,0 +1,153 @@ +import { describe, it, expect } from "vitest"; +import type { Page } from "@playwright/test"; +import { + enterGenericMcpThroughCatalog, + gradeUnreachableMcp, +} from "./external-mcp-failure.js"; +import { scopedSessionFailureDiagnostic } from "./external-gateway-client.js"; +import type { ExternalObservation } from "./external-support.js"; +function pageFixture(missing?: string) { + const actions: string[] = []; + const page = { + getByRole: (role: string, options?: { name?: string }) => { + const name = options?.name ?? ""; + const locator = { + filter: () => locator, + getByRole: (_r: string, o: { name: string }) => ({ + click: async () => { + if (o.name === missing) throw new Error("Missing real control"); + actions.push(o.name); + }, + }), + click: async () => { + if (name === missing) throw new Error("Missing real control"); + actions.push(name); + }, + }; + return locator; + }, + waitForURL: async (accept: (url: URL) => boolean) => { + expect(accept(new URL("http://fixture.invalid/RUN/apps/byo"))).toBe(true); + expect(accept(new URL("http://fixture.invalid/RUN/apps"))).toBe(false); + actions.push("actual /RUN/apps/byo"); + }, + goto: () => { + throw new Error("Direct navigation rescue is forbidden"); + }, + } as unknown as Page; + return { page, actions }; +} +describe("finite held external diagnostics and visible entry correction", () => { + it("uses visible catalog controls including the expanded custom server option", async () => { + const { page, actions } = pageFixture(); + await enterGenericMcpThroughCatalog(page, "RUN"); + expect(actions).toEqual([ + "Connectors", + "Connect", + "Connect your own MCP server", + "actual /RUN/apps/byo", + ]); + }); + it("does not rescue a missing catalog entry with direct navigation", async () => { + for (const name of [ + "Connectors", + "Connect", + "Connect your own MCP server", + ]) { + const { page, actions } = pageFixture(name); + await expect(enterGenericMcpThroughCatalog(page, "RUN")).rejects.toThrow( + "Missing real control", + ); + expect(actions).not.toContain("actual /RUN/apps/byo"); + } + }); + const context = { + companyId: "owned-company", + agentId: "owned-agent", + runId: "owned-run", + issueId: "owned-issue", + observedRunStatusAtAdmission: "running", + }; + it("retains actual HTTP500 and admitted run binding without inventing a cause or fixing admission", () => { + expect( + scopedSessionFailureDiagnostic(500, context, { + error: "private diagnostic", + }), + ).toEqual({ + ...context, + httpStatus: 500, + reasonCode: "unclassified", + responseBodyRetained: false, + }); + }); + it("retains only a recognized policy/context code, never a secret-bearing error or extra response fields", () => { + for (const reason of [ + "run_company_mismatch", + "run_agent_mismatch", + "run_inactive", + "run_context_mismatch", + ]) { + const d = scopedSessionFailureDiagnostic(403, context, { + reasonCode: reason, + token: "synthetic-credential", + error: "private body", + }); + expect(d.reasonCode).toBe(reason); + expect(JSON.stringify(d)).not.toMatch( + /synthetic-credential|private body/, + ); + } + }); + it("omits unknown, malformed, nested and secret-bearing reason values", () => { + for (const body of [ + { reasonCode: "Bearer synthetic-credential" }, + { reasonCode: "new_upstream_message" }, + { reasonCode: { token: "synthetic-credential" } }, + { error: { reasonCode: "run_inactive" } }, + null, + "private body", + ]) { + const d = scopedSessionFailureDiagnostic(500, context, body); + expect(d.reasonCode).toBe("unclassified"); + expect(JSON.stringify(d)).not.toMatch( + /Bearer|synthetic-credential|private body|new_upstream_message/, + ); + } + }); + it("visible-entry correction does not turn the retained model missing-URL question into a completed setup pass", () => { + const o = { + scenarioId: "connection-setup.unreachable-mcp", + companyId: "company", + origin: "http://fixture.invalid", + reply: "", + connections: [], + phaseReplies: [], + profiles: [], + secrets: [], + endpoints: [], + interactions: [], + activity: [], + links: [], + runs: [], + humanQueries: [], + state: { + fixtureServerClosed: true, + serverUrl: "http://127.0.0.1:12345/", + genericEntryThroughVisibleCatalog: true, + setupStatus: 502, + visibleConnectivityError: true, + checkAgainAvailable: true, + displayedServerPreserved: true, + editableUrlPreserved: true, + abandonedDraft: true, + noUnexpectedConnection: true, + }, + } as unknown as ExternalObservation; + expect(gradeUnreachableMcp(o).passed).toBe(false); + o.reply = + "The MCP server is not connected; correct the address or abandon this setup."; + expect(gradeUnreachableMcp(o).passed).toBe(true); + o.state.genericEntryThroughVisibleCatalog = false; + expect(gradeUnreachableMcp(o).passed).toBe(false); + }); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts b/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts index 9a51d63ceaf..c519dd7b854 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts @@ -1,53 +1,205 @@ -import { expect } from "@playwright/test"; +import { expect, type Page } from "@playwright/test"; import type { ToolConnection } from "../../../packages/shared/src/types/tool-access.js"; -import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js"; +import { + definition, + externalObservation, + observeExternal, + root, + checks, + verifyRenderedReply, +} from "./external-support.js"; import { startNotesProvider } from "./external-provider.js"; import { waitForReply } from "./identity-support.js"; +/** Follow the actual catalog card rather than an unqualified route that the + * product intentionally redirects to Browse. No direct /byo navigation fallback. */ +export async function enterGenericMcpThroughCatalog( + page: Page, + prefix: string, +) { + await page.getByRole("link", { name: "Connectors", exact: true }).click(); + const card = page + .getByRole("listitem") + .filter({ + has: page.getByRole("heading", { + name: "Connect your own tool", + exact: true, + }), + }); + await card.getByRole("button", { name: "Connect", exact: true }).click(); + await card + .getByRole("button", { name: "Connect your own MCP server" }) + .click(); + await page.waitForURL((url) => url.pathname === `/${prefix}/apps/byo`); +} export function gradeUnreachableMcp(value: unknown) { - const c = checks(), o = externalObservation(value); - c.add("complete-unreachable-server-evidence", !!o, "An actual unavailable fixture endpoint, rejected public setup request and visible recovery controls are required."); if (!o) return c.result(); - c.add("actual-unavailable-fixture", o.state.fixtureServerClosed === true && typeof o.state.serverUrl === "string" && /^http:\/\/127\.0\.0\.1:\d+\/$/.test(o.state.serverUrl), - "The supplied synthetic endpoint is closed before the human query; it is not a mocked provider error."); - c.add("supported-setup-failed-visibly", [400, 403, 422, 502, 503].includes(Number(o.state.setupStatus)) && o.state.visibleConnectivityError === true - && o.state.checkAgainAvailable === true, "The real setup endpoint rejects connectivity or destination policy and renders an actionable error."); - c.add("safe-editable-choice-preserved", o.state.displayedServerPreserved === true && o.state.editableUrlPreserved === true && o.state.abandonedDraft === true, - "The person can return to the exact safe address, edit it and abandon the unfinished setup."); - c.add("no-usable-phantom-connection", !o.connections.some(row => row.status === "active") && o.state.noUnexpectedConnection === true, - "Failed setup leaves no healthy or unexpected substitute connection."); - c.add("no-false-connected-claim", /mcp|server|connect/i.test(o.reply) && !/\bI (?:have )?(?:connected|installed|enabled)\b|\b(?:is|now|already) (?:connected|healthy|ready to use)\b/i.test(o.reply), - "Chat does not label the unavailable server healthy or silently substitute a different destination."); + const c = checks(), + o = externalObservation(value); + c.add( + "complete-unreachable-server-evidence", + !!o, + "An actual unavailable fixture endpoint, rejected public setup request and visible recovery controls are required.", + ); + if (!o) return c.result(); + c.add( + "actual-unavailable-fixture", + o.state.fixtureServerClosed === true && + typeof o.state.serverUrl === "string" && + /^http:\/\/127\.0\.0\.1:\d+\/$/.test(o.state.serverUrl), + "The supplied synthetic endpoint is closed before the human query; it is not a mocked provider error.", + ); + c.add( + "supported-setup-failed-visibly", + [400, 403, 422, 502, 503].includes(Number(o.state.setupStatus)) && + o.state.visibleConnectivityError === true && + o.state.checkAgainAvailable === true && + o.state.genericEntryThroughVisibleCatalog === true, + "The real setup endpoint rejects connectivity or destination policy and renders an actionable error.", + ); + c.add( + "safe-editable-choice-preserved", + o.state.displayedServerPreserved === true && + o.state.editableUrlPreserved === true && + o.state.abandonedDraft === true, + "The person can return to the exact safe address, edit it and abandon the unfinished setup.", + ); + c.add( + "no-usable-phantom-connection", + !o.connections.some((row) => row.status === "active") && + o.state.noUnexpectedConnection === true, + "Failed setup leaves no healthy or unexpected substitute connection.", + ); + c.add( + "no-false-connected-claim", + /mcp|server|connect/i.test(o.reply) && + !/\bI (?:have )?(?:connected|installed|enabled)\b|\b(?:is|now|already) (?:connected|healthy|ready to use)\b/i.test( + o.reply, + ), + "Chat does not label the unavailable server healthy or silently substitute a different destination.", + ); return c.result(); } -export const mcpFailureExternalCases = [definition("connection-setup.unreachable-mcp", async ctx => { - const server = await startNotesProvider("Never served by the closed fixture"); const serverUrl = server.url; await server.close(); - const before = await ctx.input.api.get<{ connections: unknown[] }>(`${root(ctx)}/tools/connections`); if (before.connections.length) throw new Error("Unavailable server journey requires an empty disposable connection catalog"); - await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue(); - await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The supplied synthetic generic MCP server URL is ${serverUrl}. No credential has been supplied.` }); - return { serverUrl, fixtureServerClosed: true }; -}, gradeUnreachableMcp, { - sourceFiles: ["chat-catalog-domains/external-mcp-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"], - async act(ctx, state) { - await waitForReply(ctx); const source = ctx.input.page.url(); - await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/apps/connect`, { waitUntil: "domcontentloaded" }); - const url = ctx.input.page.getByRole("textbox", { name: "MCP server URL", exact: true }); - await url.fill(String(state.serverUrl)); await ctx.input.page.getByRole("button", { name: "Continue", exact: true }).click(); - const checked = ctx.input.page.waitForResponse(response => new URL(response.url()).pathname === `${root(ctx)}/tools/apps/connect` && response.request().method() === "POST"); - await ctx.input.page.getByRole("button", { name: "Check link", exact: true }).click(); - state.setupStatus = (await checked).status(); - const error = ctx.input.page.getByRole("note").filter({ hasText: /couldn.t (?:reach|connect)|private network|couldn.t find that host/i }); - await expect(error).toBeVisible(); state.visibleConnectivityError = true; - await expect(ctx.input.page.getByRole("button", { name: "Check link", exact: true })).toBeEnabled(); state.checkAgainAvailable = true; - await expect(ctx.input.page.getByText(String(state.serverUrl), { exact: true })).toBeVisible(); state.displayedServerPreserved = true; - await ctx.input.page.getByRole("button", { name: "Back", exact: true }).click(); - await expect(url).toHaveValue(String(state.serverUrl)); await url.fill(String(state.serverUrl) + "corrected"); state.editableUrlPreserved = await url.inputValue() === String(state.serverUrl) + "corrected"; - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); state.abandonedDraft = true; - await ctx.input.evidence("chat-catalog-unreachable-mcp.json", { setupStatus: state.setupStatus, visibleConnectivityError: true, editableUrlPreserved: state.editableUrlPreserved, abandonedDraft: true }); - }, - async observe(ctx, state) { - const rows = await ctx.input.api.get<{ connections: ToolConnection[] }>(`${root(ctx)}/tools/connections`); - state.noUnexpectedConnection = rows.connections.every(row => row.config?.url === state.serverUrl && row.status !== "active"); - return observeExternal(ctx, state); - }, - async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); }, - async cleanup(ctx) { const rows = await ctx.input.api.get<{ connections: Array<{ id: string }> }>(`${root(ctx)}/tools/connections`); for (const row of rows.connections) await ctx.input.api.delete(`/api/tool-connections/${row.id}`, { allowNotFound: true }); }, -})]; +export const mcpFailureExternalCases = [ + definition( + "connection-setup.unreachable-mcp", + async (ctx) => { + const server = await startNotesProvider( + "Never served by the closed fixture", + ); + const serverUrl = server.url; + await server.close(); + const before = await ctx.input.api.get<{ connections: unknown[] }>( + `${root(ctx)}/tools/connections`, + ); + if (before.connections.length) + throw new Error( + "Unavailable server journey requires an empty disposable connection catalog", + ); + await ctx.input.api.post( + `${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`, + ); + await ctx.refreshIssue(); + await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { + description: `The supplied synthetic generic MCP server URL is ${serverUrl}. No credential has been supplied.`, + }); + return { serverUrl, fixtureServerClosed: true }; + }, + gradeUnreachableMcp, + { + sourceFiles: [ + "../../ui/src/App.tsx", + "../../ui/src/pages/apps/Browse.tsx", + "../../ui/src/features/connections/ConnectionSetupFlow.tsx", + "../../ui/src/pages/apps/app-connect-policy.ts", + "chat-catalog-domains/external-mcp-failure.ts", + "chat-catalog-domains/external-provider.ts", + "chat-catalog-domains/external-support.ts", + "chat-catalog-domains/internal-support.ts", + "chat-catalog-domains/identity-support.ts", + "chat-scenarios/connection-setup.yaml", + ], + async act(ctx, state) { + await waitForReply(ctx); + const source = ctx.input.page.url(); + await enterGenericMcpThroughCatalog( + ctx.input.page, + String(ctx.input.fixtures.company.issuePrefix), + ); + state.genericEntryThroughVisibleCatalog = true; + const url = ctx.input.page.getByRole("textbox", { + name: "MCP server URL", + exact: true, + }); + await url.fill(String(state.serverUrl)); + await ctx.input.page + .getByRole("button", { name: "Continue", exact: true }) + .click(); + const checked = ctx.input.page.waitForResponse( + (response) => + new URL(response.url()).pathname === + `${root(ctx)}/tools/apps/connect` && + response.request().method() === "POST", + ); + await ctx.input.page + .getByRole("button", { name: "Check link", exact: true }) + .click(); + state.setupStatus = (await checked).status(); + const error = ctx.input.page + .getByRole("note") + .filter({ + hasText: + /couldn.t (?:reach|connect)|private network|couldn.t find that host/i, + }); + await expect(error).toBeVisible(); + state.visibleConnectivityError = true; + await expect( + ctx.input.page.getByRole("button", { + name: "Check link", + exact: true, + }), + ).toBeEnabled(); + state.checkAgainAvailable = true; + await expect( + ctx.input.page.getByText(String(state.serverUrl), { exact: true }), + ).toBeVisible(); + state.displayedServerPreserved = true; + await ctx.input.page + .getByRole("button", { name: "Back", exact: true }) + .click(); + await expect(url).toHaveValue(String(state.serverUrl)); + await url.fill(String(state.serverUrl) + "corrected"); + state.editableUrlPreserved = + (await url.inputValue()) === String(state.serverUrl) + "corrected"; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + state.abandonedDraft = true; + await ctx.input.evidence("chat-catalog-unreachable-mcp.json", { + setupStatus: state.setupStatus, + visibleConnectivityError: true, + editableUrlPreserved: state.editableUrlPreserved, + abandonedDraft: true, + }); + }, + async observe(ctx, state) { + const rows = await ctx.input.api.get<{ connections: ToolConnection[] }>( + `${root(ctx)}/tools/connections`, + ); + state.noUnexpectedConnection = rows.connections.every( + (row) => + row.config?.url === state.serverUrl && row.status !== "active", + ); + return observeExternal(ctx, state); + }, + async verifyResult(ctx, _state, value) { + await verifyRenderedReply(ctx, value); + }, + async cleanup(ctx) { + const rows = await ctx.input.api.get<{ + connections: Array<{ id: string }>; + }>(`${root(ctx)}/tools/connections`); + for (const row of rows.connections) + await ctx.input.api.delete(`/api/tool-connections/${row.id}`, { + allowNotFound: true, + }); + }, + }, + ), +]; diff --git a/tests/runner-e2e/chat-catalog-domains/external.test.ts b/tests/runner-e2e/chat-catalog-domains/external.test.ts index 70c8fec0c72..088d806ed8b 100644 --- a/tests/runner-e2e/chat-catalog-domains/external.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external.test.ts @@ -45,7 +45,7 @@ function valid(id: string) { phaseBoundary: "operator-chat-to-viewer-browser-consent", chatCallerId: "operator", viewerSessionBefore: "viewer", viewerSessionAtConsent: "viewer", viewerSessionAfter: "viewer" }); o.activity = [{ id: "authorization", action: "mcp.connection_authorized", entityId: "grant", actorType: "user", actorId: "viewer" }]; } else if (id === "connection-setup.unreachable-mcp") { - Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true }); + Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, genericEntryThroughVisibleCatalog:true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true }); o.reply = "Open the supported MCP server connection setup and check its address."; } else if (id === "plugins.install-plugin") { Object.assign(o.state, { pluginId: "plugin", pluginKey: "synthetic.preview", syntheticPackage: true, pluginAfter: { id: "plugin", pluginKey: "synthetic.preview", status: "ready", manifestJson: { displayName: "Website Preview Eval" } }, operatorChannel: "board-api-local-path", agentInstallStatus: 403, absentBeforeHuman: true, humanInstallStatus: 200, previewReadback: true, humanSawActivation: true });