test(evals): cover unavailable MCP setup recovery

tests/runner-e2e/chat-catalog-domains/external-definitions.ts

@@ -12,3 +12,4 @@ export { pluginExternalCases } from "./external-plugins.js";
 export { assistantDeviceExternalCases } from "./external-assistant-device.js";
 export { gatewayClientExternalCases } from "./external-gateway-client.js";
 export { pluginInstallExternalCases } from "./external-plugin-install.js";
+export { mcpFailureExternalCases } from "./external-mcp-failure.js";

tests/runner-e2e/chat-catalog-domains/external-mcp-failure.ts

@@ -0,0 +1,53 @@
+import { expect } from "@playwright/test";
+import type { ToolConnection } from "../../../packages/shared/src/types/tool-access.js";
+import { definition, externalObservation, observeExternal, root, checks, verifyRenderedReply } from "./external-support.js";
+import { startNotesProvider } from "./external-provider.js";
+import { waitForReply } from "./identity-support.js";
+export function gradeUnreachableMcp(value: unknown) {
+  const c = checks(), o = externalObservation(value);
+  c.add("complete-unreachable-server-evidence", !!o, "An actual unavailable fixture endpoint, rejected public setup request and visible recovery controls are required."); if (!o) return c.result();
+  c.add("actual-unavailable-fixture", o.state.fixtureServerClosed === true && typeof o.state.serverUrl === "string" && /^http:\/\/127\.0\.0\.1:\d+\/$/.test(o.state.serverUrl),
+    "The supplied synthetic endpoint is closed before the human query; it is not a mocked provider error.");
+  c.add("supported-setup-failed-visibly", [400, 403, 422, 502, 503].includes(Number(o.state.setupStatus)) && o.state.visibleConnectivityError === true
+    && o.state.checkAgainAvailable === true, "The real setup endpoint rejects connectivity or destination policy and renders an actionable error.");
+  c.add("safe-editable-choice-preserved", o.state.displayedServerPreserved === true && o.state.editableUrlPreserved === true && o.state.abandonedDraft === true,
+    "The person can return to the exact safe address, edit it and abandon the unfinished setup.");
+  c.add("no-usable-phantom-connection", !o.connections.some(row => row.status === "active") && o.state.noUnexpectedConnection === true,
+    "Failed setup leaves no healthy or unexpected substitute connection.");
+  c.add("no-false-connected-claim", /mcp|server|connect/i.test(o.reply) && !/\bI (?:have )?(?:connected|installed|enabled)\b|\b(?:is|now|already) (?:connected|healthy|ready to use)\b/i.test(o.reply),
+    "Chat does not label the unavailable server healthy or silently substitute a different destination.");
+  return c.result();
+}
+export const mcpFailureExternalCases = [definition("connection-setup.unreachable-mcp", async ctx => {
+  const server = await startNotesProvider("Never served by the closed fixture"); const serverUrl = server.url; await server.close();
+  const before = await ctx.input.api.get<{ connections: unknown[] }>(`${root(ctx)}/tools/connections`); if (before.connections.length) throw new Error("Unavailable server journey requires an empty disposable connection catalog");
+  await ctx.input.api.post(`${root(ctx)}/chats/${ctx.input.fixtures.agent.id}`); await ctx.refreshIssue();
+  await ctx.input.api.patch(`/api/issues/${ctx.issue()!.id}`, { description: `The supplied synthetic generic MCP server URL is ${serverUrl}. No credential has been supplied.` });
+  return { serverUrl, fixtureServerClosed: true };
+}, gradeUnreachableMcp, {
+  sourceFiles: ["chat-catalog-domains/external-mcp-failure.ts", "chat-catalog-domains/external-provider.ts", "chat-catalog-domains/external-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/connection-setup.yaml"],
+  async act(ctx, state) {
+    await waitForReply(ctx); const source = ctx.input.page.url();
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/apps/connect`, { waitUntil: "domcontentloaded" });
+    const url = ctx.input.page.getByRole("textbox", { name: "MCP server URL", exact: true });
+    await url.fill(String(state.serverUrl)); await ctx.input.page.getByRole("button", { name: "Continue", exact: true }).click();
+    const checked = ctx.input.page.waitForResponse(response => new URL(response.url()).pathname === `${root(ctx)}/tools/apps/connect` && response.request().method() === "POST");
+    await ctx.input.page.getByRole("button", { name: "Check link", exact: true }).click();
+    state.setupStatus = (await checked).status();
+    const error = ctx.input.page.getByRole("note").filter({ hasText: /couldn.t (?:reach|connect)|private network|couldn.t find that host/i });
+    await expect(error).toBeVisible(); state.visibleConnectivityError = true;
+    await expect(ctx.input.page.getByRole("button", { name: "Check link", exact: true })).toBeEnabled(); state.checkAgainAvailable = true;
+    await expect(ctx.input.page.getByText(String(state.serverUrl), { exact: true })).toBeVisible(); state.displayedServerPreserved = true;
+    await ctx.input.page.getByRole("button", { name: "Back", exact: true }).click();
+    await expect(url).toHaveValue(String(state.serverUrl)); await url.fill(String(state.serverUrl) + "corrected"); state.editableUrlPreserved = await url.inputValue() === String(state.serverUrl) + "corrected";
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); state.abandonedDraft = true;
+    await ctx.input.evidence("chat-catalog-unreachable-mcp.json", { setupStatus: state.setupStatus, visibleConnectivityError: true, editableUrlPreserved: state.editableUrlPreserved, abandonedDraft: true });
+  },
+  async observe(ctx, state) {
+    const rows = await ctx.input.api.get<{ connections: ToolConnection[] }>(`${root(ctx)}/tools/connections`);
+    state.noUnexpectedConnection = rows.connections.every(row => row.config?.url === state.serverUrl && row.status !== "active");
+    return observeExternal(ctx, state);
+  },
+  async verifyResult(ctx, _state, value) { await verifyRenderedReply(ctx, value); },
+  async cleanup(ctx) { const rows = await ctx.input.api.get<{ connections: Array<{ id: string }> }>(`${root(ctx)}/tools/connections`); for (const row of rows.connections) await ctx.input.api.delete(`/api/tool-connections/${row.id}`, { allowNotFound: true }); },
+})];

tests/runner-e2e/chat-catalog-domains/external.test.ts

@@ -3,6 +3,7 @@ import { assistantConnectionExternalCases } from "./external-assistant-connectio
 import { chatChannelExternalCases } from "./external-chat-channels.js";
 import { gatewayClientExternalCases } from "./external-gateway-client.js";
 import { pluginInstallExternalCases } from "./external-plugin-install.js";
+import { mcpFailureExternalCases } from "./external-mcp-failure.js";
 import { assistantDeviceExternalCases } from "./external-assistant-device.js";
 import { pluginExternalCases } from "./external-plugins.js";
 import { appCatalogExternalCases } from "./external-app-catalog.js";
@@ -14,7 +15,7 @@ import { connectionSetupExternalCases } from "./external-connection-setup.js";
 import { externalPrerequisiteReceipt, externalRequirements, validateExternalConfig, selectedExternalBinding, selectedExternalCredential } from "./external-fixtures.js";
 import type { ExternalObservation } from "./external-support.js";
 
-const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases];
+const definitions = [...appPermissionExternalCases, ...teamExternalCases, ...secretExternalCases, ...gatewayExternalCases, ...connectionSetupExternalCases, ...assistantConnectionExternalCases, ...chatChannelExternalCases, ...appCatalogExternalCases, ...pluginExternalCases, ...assistantDeviceExternalCases, ...gatewayClientExternalCases, ...pluginInstallExternalCases, ...mcpFailureExternalCases];
 function base(scenarioId: string): ExternalObservation {
   return { scenarioId, origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", state: { initialRunId: "initial" },
     phaseReplies: [], connections: [], profiles: [], secrets: [], endpoints: [], interactions: [], tasks: [], projects: [], goals: [], comments: [],
@@ -22,7 +23,10 @@ function base(scenarioId: string): ExternalObservation {
 }
 function valid(id: string) {
   const o = base(id);
-  if (id === "plugins.install-plugin") {
+  if (id === "connection-setup.unreachable-mcp") {
+    Object.assign(o.state, { serverUrl: "http://127.0.0.1:8848/", fixtureServerClosed: true, setupStatus: 400, visibleConnectivityError: true, checkAgainAvailable: true, displayedServerPreserved: true, editableUrlPreserved: true, abandonedDraft: true, noUnexpectedConnection: true });
+    o.reply = "Open the supported MCP server connection setup and check its address.";
+  } else if (id === "plugins.install-plugin") {
     Object.assign(o.state, { pluginId: "plugin", pluginKey: "synthetic.preview", syntheticPackage: true, pluginAfter: { id: "plugin", pluginKey: "synthetic.preview", status: "ready", manifestJson: { displayName: "Website Preview Eval" } }, operatorChannel: "board-api-local-path", agentInstallStatus: 403, absentBeforeHuman: true, humanInstallStatus: 200, previewReadback: true, humanSawActivation: true });
     o.reply = "An instance administrator can install Website Preview Eval through plugin management.";
     o.activity = [{ id: "activity", entityId: "plugin", actorType: "user", action: "plugin.installed" }];
@@ -90,6 +94,7 @@ function valid(id: string) {
   return o;
 }
 const wrong: Record<string, (o: ExternalObservation) => void> = {
+  "connection-setup.unreachable-mcp": o => { o.connections = [{ id: "phantom", status: "active" }]; },
   "plugins.install-plugin": o => { o.state.pluginAfter = { id: "plugin", pluginKey: "synthetic.preview", status: "error" }; },
   "gateways-profiles.connect-client": o => { o.state.providerWriteCalls = 1; },
   "assistant-connections.device-login": o => { o.state.elapsedBeforeExpiryProbe = 500; },