PaperClipAI/tests/e2e/apps-prosumer-mcp-flow.spec.ts

import { expect, test, type APIRequestContext, type Page } from "@playwright/test";
import { createServer, type Server } from "node:http";
import { listenOnFetchAllowedPort } from "./fetch-allowed-port";

// prosumer MCP flow — QA harness for the prosumer Connect-an-app flow on top of the
// tool-access foundation. Covers the M-series happy path (gallery + key paste
// → choose access → install → success), the expired-key reconnect path,
// the Needs-attention surface, and a regression check that /apps/advanced
// still mounts.
//
// The spec boots the shared local_trusted Playwright webServer (see
// playwright.config), spawns a small in-process mock HTTP MCP server that
// responds to tools/list with read-only and write-side-effect tools, then
// drives the wizard via the Apps UI for evidence + screenshots.

const SCREENSHOT_DIR = "test-results";

type Seed = { companyId: string; prefix: string };

async function newCompany(request: APIRequestContext, label: string): Promise<Seed> {
  const res = await request.post("/api/companies", { data: { name: `prosumer MCP flow ${label} ${Date.now()}` } });
  expect(res.ok(), `create company failed ${res.status()}: ${await res.text()}`).toBe(true);
  const company = await res.json();
  return {
    companyId: company.id,
    prefix: company.issuePrefix ?? company.prefix ?? company.urlKey ?? "E2E",
  };
}

// ---- Mock MCP HTTP fixture --------------------------------------------------
// Minimal MCP JSON-RPC server. /catalog refresh hits this with method
// `tools/list`; the gateway calls it with `tools/call`. We expose one
// read-only and one write tool so setup can apply risk-based ask-first defaults.

type MockMcpServer = { url: string; close: () => Promise<void>; captures: Array<{ method: string; params: unknown }> };

async function startMockMcp(options: { expectedHeader?: string } = {}): Promise<MockMcpServer> {
  const captures: Array<{ method: string; params: unknown }> = [];
  const server: Server = createServer(async (req, res) => {
    if (req.method !== "POST") {
      res.writeHead(405).end();
      return;
    }
    if (options.expectedHeader && req.headers.authorization !== options.expectedHeader) {
      res.writeHead(401, { "Content-Type": "application/json", "WWW-Authenticate": "Bearer" });
      res.end(JSON.stringify({ jsonrpc: "2.0", id: null, error: { code: -32000, message: "Unauthorized" } }));
      return;
    }
    const chunks: Buffer[] = [];
    for await (const chunk of req) chunks.push(chunk as Buffer);
    let payload: { id?: string | number; method?: string; params?: unknown } = {};
    try {
      payload = JSON.parse(Buffer.concat(chunks).toString("utf8") || "{}");
    } catch {
      // fall through to method routing — will land in default
    }
    captures.push({ method: String(payload.method ?? "<unknown>"), params: payload.params });

    if (payload.method === "tools/list") {
      res.writeHead(200, { "Content-Type": "application/json" });
      res.end(
        JSON.stringify({
          jsonrpc: "2.0",
          id: payload.id ?? null,
          result: {
            tools: [
              {
                name: "list_widgets",
                title: "List widgets",
                description: "Read-only listing of widgets.",
                inputSchema: { type: "object", properties: {}, additionalProperties: false },
              },
              {
                // Namespaced name (namespaced tool names): real MCP servers prefix tool names
                // ("github:create_issue"). The classifier must still see the "create"
                // verb and land this under "Can make changes" with the toggle OFF —
                // the old leading-anchor regex fell through to read-only.
                name: "qa10864:create_widget",
                title: "Create widget",
                description: "Creates a widget — has side effects.",
                inputSchema: {
                  type: "object",
                  properties: { name: { type: "string" } },
                  required: ["name"],
                  additionalProperties: false,
                },
              },
            ],
          },
        })
      );
      return;
    }
    if (payload.method === "tools/call") {
      res.writeHead(200, { "Content-Type": "application/json" });
      res.end(JSON.stringify({ jsonrpc: "2.0", id: payload.id ?? null, result: { content: [{ type: "text", text: "ok" }] } }));
      return;
    }
    res.writeHead(200, { "Content-Type": "application/json" });
    res.end(JSON.stringify({ jsonrpc: "2.0", id: payload.id ?? null, result: {} }));
  });
  const port = await listenOnFetchAllowedPort(server);
  return {
    url: `http://127.0.0.1:${port}/`,
    captures,
    close: () => new Promise<void>((resolve) => server.close(() => resolve())),
  };
}

// ---- Helpers ----------------------------------------------------------------

async function gotoApps(page: Page, prefix: string) {
  await page.goto(`/${prefix}/apps/connections`);
}

async function gotoConnect(page: Page, prefix: string) {
  await page.goto(`/${prefix}/apps`);
  await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 30_000 });
  const customConnector = page
    .getByRole("list", { name: "Connector list" })
    .getByRole("listitem")
    .filter({ hasText: "Connect your own tool" });
  await customConnector.getByRole("button", { name: "Connect", exact: true }).click();
  await customConnector.getByRole("button", { name: "Connect your own MCP server" }).click();
}

async function gotoAdvanced(page: Page, prefix: string) {
  await page.goto(`/${prefix}/apps/advanced`);
}

async function gotoNeedsAttention(page: Page, prefix: string) {
  await page.goto(`/${prefix}/apps/connections`);
}

// ---- Tests ------------------------------------------------------------------

test.describe.serial("prosumer MCP flow prosumer MCP flow", () => {
  test.setTimeout(180_000); // vite-dev cold bundling is slow on first hit

  let mock: MockMcpServer;

  test.beforeAll(async () => {
    mock = await startMockMcp();
  });

  test.afterAll(async () => {
    await mock?.close();
  });

  test("Connect wizard happy path: link mode → success", async ({ page, request }) => {
    const seed = await newCompany(request, "connect");

    await gotoConnect(page, seed.prefix);

    // Browse launches the BYO link-mode connect wizard.
    await expect(page.getByText("Connect your own MCP server", { exact: true })).toBeVisible({ timeout: 30_000 });
    await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-01-gallery.png`, fullPage: true });

    // Use the "Connect with a link" path against the mock MCP server.
    const linkInput = page.getByPlaceholder("https://example.com/actions");
    await linkInput.fill(mock.url);
    await page.getByRole("button", { name: "Continue" }).click();

    // There is no separate Access step: the link opens the key screen, which
    // states the default access in one line. The mock needs no key, and a
    // credential challenge from the server is what would ask for one.
    await expect(page.getByRole("heading", { name: "Connect your own MCP server" })).toBeVisible({ timeout: 15_000 });
    await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-02-key-step.png`, fullPage: true });

    // Submit (button label is "Check link").
    await page.getByRole("button", { name: /Check link/i }).click();

    // A successful generic probe commits discovered actions and the stated
    // access defaults transactionally, so the key check lands on success.
    await expect(page.getByRole("heading", { name: /is ready\.$/i })).toBeVisible({ timeout: 30_000 });
    await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-05-success.png`, fullPage: true });

    // Verify the mock saw a tools/list call from the catalog refresh.
    expect(mock.captures.some((c) => c.method === "tools/list")).toBe(true);

    // Return through the UI so a page reload cannot hide a stale catalog cache.
    await page.getByRole("button", { name: "View connection" }).click();
    await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 15_000 });
    await expect(page.getByRole("heading", { name: `${new URL(mock.url).host} for the organization`, exact: true })).toBeVisible();
    await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-06-apps-list.png`, fullPage: true });
  });

  test("Expired key → health sweep → Needs attention → reconnect → green", async ({ page, request }) => {
    const seed = await newCompany(request, "attention");

    // Spawn a dedicated mock we can break partway through.
    const ephemeral = await startMockMcp();
    try {
      const connect = await request.post(`/api/companies/${seed.companyId}/tools/apps/connect`, {
        data: { link: ephemeral.url, credentialValues: { "credentials.authorization": "qa-token" } },
      });
      expect(connect.ok(), `connect failed ${connect.status()}: ${await connect.text()}`).toBe(true);
      const connectResult = await connect.json();
      const connectionId = connectResult.connectionId as string;
      const enabledCatalogEntryIds = (connectResult.actions?.readOnly ?? []).map(
        (action: { catalogEntryId: string }) => action.catalogEntryId,
      );
      const finish = await request.post(`/api/companies/${seed.companyId}/tools/apps/${connectionId}/finish`, {
        data: { enabledCatalogEntryIds, askFirstCatalogEntryIds: [], access: "all_agents" },
      });
      expect(finish.ok(), `finish failed ${finish.status()}: ${await finish.text()}`).toBe(true);

      // Break the mock so the next health-check fails (simulates expired key /
      // dead remote — the same observable shape the server uses to mark a
      // connection unhealthy).
      await ephemeral.close();

      const health = await request.post(`/api/tool-connections/${connectionId}/health-check`);
      expect(health.status(), `health-check should fail with the mock down`).toBe(502);

      const before = await request.get(`/api/tool-connections/${connectionId}`);
      const beforeBody = await before.json();
      expect(beforeBody.healthStatus).not.toBe("ok");

      // Needs-attention page should surface this connection.
      await gotoNeedsAttention(page, seed.prefix);
      await expect(page.getByRole("heading", { name: "Connectors" })).toBeVisible({ timeout: 30_000 });
      await expect(page.getByText("Needs attention", { exact: true }).first()).toBeVisible({ timeout: 30_000 });
      await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-07-needs-attention.png`, fullPage: true });

      // App detail should expose the reconnect call-to-action.
      await page.goto(`/${seed.prefix}/apps/${connectionId}`);
      await expect(page.getByRole("button", { name: /Reconnect|Replace key/i }).first()).toBeVisible({ timeout: 20_000 });
      await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-08-app-detail-reconnect.png`, fullPage: true });

      // Bring the mock back so reconnect succeeds.
      const recovered = await startMockMcp();
      try {
        // The reconnect endpoint replaces credentials but does not change the URL,
        // so we update the connection URL through PATCH first to point at the
        // recovered mock — this mirrors what users do when the remote moves.
        const repatch = await request.patch(`/api/tool-connections/${connectionId}`, {
          data: { config: { url: recovered.url } },
        });
        expect(repatch.ok(), `patch url failed ${repatch.status()}: ${await repatch.text()}`).toBe(true);

        await page.reload();
        await page.getByLabel("App key", { exact: true }).fill("fresh-key");
        await page.getByRole("button", { name: "Check & reconnect", exact: true }).click();
        await expect(page.getByText("Reconnected", { exact: true })).toBeVisible();
        await expect(page.getByRole("heading", { name: "This app needs reconnecting" })).toHaveCount(0);
        await expect(page.getByText("Still not working", { exact: true })).toHaveCount(0);

        const after = await request.get(`/api/tool-connections/${connectionId}`);
        const afterBody = await after.json();
        expect(afterBody.healthStatus).toBe("ok");
      } finally {
        await recovered.close();
      }
    } finally {
      await ephemeral.close().catch(() => {});
    }
  });

  test("/apps/advanced mounts the paste-config path", async ({ page, request }) => {
    const seed = await newCompany(request, "advanced");

    await gotoAdvanced(page, seed.prefix);
    await expect(page.getByRole("heading", { name: "Advanced setup" })).toBeVisible({ timeout: 20_000 });
    await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-09-advanced-default.png`, fullPage: true });

    await expect(page.getByRole("link", { name: "Paste a config" })).toBeVisible();
    await expect(page.getByRole("link", { name: /Run your own|Self host|Stdio|Local/i })).toHaveCount(0);
    await page.screenshot({ path: `${SCREENSHOT_DIR}/prosumer-mcp-10-advanced-paste-tab.png`, fullPage: true });
  });
});