PaperClipAI/tests/e2e/connection-reviews.spec.ts

import { expect, test, type APIRequestContext } from "@playwright/test";
import { readFile, writeFile } from "node:fs/promises";
import { startReviewProvider } from "../fixtures/connection-review-provider";

type Json = Record<string, unknown>;
type Seed = { companyId: string; prefix: string };
type Agent = { id: string; name: string };

async function json<T = Json>(
  response: Awaited<ReturnType<APIRequestContext["get"]>>,
): Promise<T> {
  expect(
    response.ok(),
    `${response.url()} failed ${response.status()}: ${await response.text()}`,
  ).toBe(true);
  return (await response.json()) as T;
}

async function newCompany(request: APIRequestContext): Promise<Seed> {
  const company = await json<{ id: string; issuePrefix: string }>(
    await request.post("/api/companies", {
      data: { name: `Connection review E2E ${Date.now()}` },
    }),
  );
  return { companyId: company.id, prefix: company.issuePrefix };
}

async function createAgent(
  request: APIRequestContext,
  companyId: string,
  name: string,
): Promise<Agent> {
  return await json<Agent>(
    await request.post(`/api/companies/${companyId}/agents`, {
      data: {
        name,
        role: "qa",
        title: "Connection intent fixture agent",
        capabilities: "Exercises deterministic connection intent wiring.",
        adapterType: "process",
        adapterConfig: {
          command: process.execPath,
          args: ["--input-type=module", "-e", "process.exit(0)"],
        },
      },
    }),
  );
}

function reviewAgentScript(connectionId: string, query: string) {
  return `
const base = process.env.PAPERCLIP_API_URL + "/api";

const headers = { authorization: "Bearer " + process.env.PAPERCLIP_API_KEY, "content-type": "application/json", "x-paperclip-run-id": process.env.PAPERCLIP_RUN_ID };
const api = async (path, method = "GET", body) => {
  const response = await fetch(base + path, { method, headers, body: body === undefined ? undefined : JSON.stringify(body) });
  if (!response.ok) throw new Error(await response.text());
  return await response.json();
};
const describePages = (raw) => {
  let value = raw;
  if (typeof value === "string") { try { value = JSON.parse(value); } catch {} }
  const content = typeof value === "string" ? value : typeof value?.content === "string" ? value.content : value?.data?.content?.filter(item => item.type === "text").map(item => item.text).join(" ") ?? "No pages returned.";
  return "I found these recent pages: " + content.replace(/^Pages: /, "") + ".";
};
const issueId = process.env.PAPERCLIP_TASK_ID ?? (await api("/heartbeat-runs/" + process.env.PAPERCLIP_RUN_ID)).contextSnapshot.issueId;
const interactions = await api("/issues/" + issueId + "/interactions");
const review = interactions.find(i => i.payload?.toolAction);
if (review?.status === "pending") { console.log("Still waiting for the existing review; no retry."); process.exit(0); }
if (review && review.status !== "pending") {
  const result = review.result?.toolAction;
  if (review.status === "accepted" && (!result || ["approved", "executing"].includes(result.status))) {
    console.log("Review accepted; waiting for the stored execution result. No retry.");
    process.exit(0);
  }
  const message = review.status === "rejected" ? "Review declined. No pages were read." : result?.status === "failed" ? "Read failed: " + result.errorMessage : ["expired", "cancelled"].includes(review.status) ? "The review expired or was cancelled. No pages were read." : describePages(result?.resultSummary);
  await api("/issues/" + issueId + "/comments", "POST", { body: message });
  await api("/issues/" + issueId, "PATCH", { status: "done" });
  console.log(message);
  process.exit(0);
}
const session = await api("/tool-gateway/sessions", "POST", { runId: process.env.PAPERCLIP_RUN_ID, ttlMs: 60000 });
const gatewayHeaders = { "x-paperclip-tool-gateway-token": session.token, "content-type": "application/json" };
const tools = await (await fetch(base + "/tool-gateway/tools", { headers: gatewayHeaders })).json();
const tool = tools.find(t => t.connectionId === ${JSON.stringify(connectionId)} && t.upstreamToolName === "notion:list_pages");
if (!tool) throw new Error("Missing fixture connection");
const call = await fetch(base + "/tool-gateway/tools/call", { method: "POST", headers: gatewayHeaders, body: JSON.stringify({ tool: tool.name, parameters: { query: ${JSON.stringify(query)} } }) });
const result = await call.json();
if (!call.ok) {
  if (result.reasonCode !== "approval_required" && result.code !== "approval_required" && !JSON.stringify(result).includes("approval_required")) throw new Error(JSON.stringify(result));
  await api("/issues/" + issueId, "PATCH", { status: "in_review" });
  console.log("Waiting for human review. No retry.");
  process.exit(0);
}
await api("/issues/" + issueId + "/comments", "POST", { body: describePages(result.result) });
await api("/issues/" + issueId, "PATCH", { status: "done" });
`;
}

for (const journey of [
  "approve",
  "decline",
  "always",
  "failure",
  "restart",
] as const) {
  test(`connection review: ${journey}, synchronized task history and actual continuation`, async ({
    page,
    context,
    request,
  }, testInfo) => {
    test.setTimeout(180_000);
    test.skip(
      journey === "restart" && !process.env.PAPERCLIP_REVIEW_RESTART_FILE,
      "Use connection-reviews.config.ts for controlled server restart",
    );
    let suppressReviewEvents = journey === "decline";
    if (journey === "decline") {
      // Reproduce a review arriving between the initial fetch and subscription:
      // return an empty first snapshot and drop its live creation notification.
      const initialSnapshots = new Set<string>();
      await page.route("**/api/issues/*/interactions", async (route) => {
        const url = route.request().url();
        if (route.request().method() === "GET" && !initialSnapshots.has(url)) {
          initialSnapshots.add(url);
          await route.fulfill({ json: [] });
        } else await route.continue();
      });
      await page.routeWebSocket("**/api/companies/*/events/ws", (socket) => {
        const server = socket.connectToServer();
        server.onMessage((message) => {
          if (suppressReviewEvents && String(message).includes("issue.thread_interaction_")) return;
          socket.send(message);
        });
      });
    }
    const provider = await startReviewProvider();
    try {
      const seed = await newCompany(request);
      const agent = await createAgent(request, seed.companyId, "Page reader");
      await page.goto(`/${seed.prefix}/apps`);
      const connector = page
        .getByRole("list", { name: "Connector list" })
        .getByRole("listitem")
        .filter({ hasText: "Connect your own tool" });
      await connector
        .getByRole("button", { name: "Connect", exact: true })
        .click();
      await connector
        .getByRole("button", { name: "Connect your own MCP server" })
        .click();
      await page
        .getByPlaceholder("https://example.com/actions")
        .fill(provider.url);
      await page.getByRole("button", { name: "Continue", exact: true }).click();
      await page.getByRole("button", { name: /Check link/i }).click();
      await expect(
        page.getByRole("heading", { name: /is ready/i }),
      ).toBeVisible({ timeout: 30_000 });
      const {
        connections: [connection],
      } = await json<{ connections: Array<{ id: string }> }>(
        await request.get(`/api/companies/${seed.companyId}/tools/connections`),
      );
      await json(
        await request.put(`/api/tool-connections/${connection.id}/installs`, {
          data: { installs: [{ targetType: "agent", targetId: agent.id }] },
        }),
      );
      await page.goto(`/${seed.prefix}/apps/${connection.id}/permissions`);
      await page
        .getByRole("radio", { name: "List fixture pages: Ask first" })
        .click();
      const configureAgent = async (query: string) =>
        json(
          await request.patch(`/api/agents/${agent.id}`, {
            data: {
              adapterConfig: {
                command: process.execPath,
                args: [
                  "--input-type=module",
                  "-e",
                  reviewAgentScript(connection.id, query),
                ],
              },
              replaceAdapterConfig: true,
            },
          }),
        );
      await configureAgent(journey === "failure" ? "fail" : "recent");
      const issue = await json<{ id: string; identifier: string }>(
        await request.post(`/api/companies/${seed.companyId}/issues`, {
          data: {
            title: "Read recent pages",
            status: "in_progress",
            assigneeAgentId: agent.id,
          },
        }),
      );
      await page.goto(`/${seed.prefix}/issues/${issue.identifier}`);
      await expect(
        page.getByRole("button", { name: "Approve & run", exact: true }),
      ).toBeVisible({ timeout: 45_000 });
      suppressReviewEvents = false;
      const originatingReviews = await json<Array<{ id: string; sourceRunId: string | null }>>(await request.get(`/api/issues/${issue.id}/interactions`));
      const originatingReview = originatingReviews[0];
      const calls = () =>
        provider.captures.filter((c) => c.method === "tools/call").length;
      expect(calls()).toBe(0);
      await page.screenshot({
        path: testInfo.outputPath("pending-review.png"),
      });
      await page
        .getByRole("button", { name: "Dismiss Approve tool action" })
        .click();
      await expect(
        page.getByRole("button", { name: "Approve & run", exact: true }),
      ).not.toBeVisible();
      await page.reload();
      await expect(page.getByRole("button", { name: "Review request", exact: true })).toBeVisible({ timeout: 30_000 });
      await expect(
        page.getByRole("button", { name: "Approve & run", exact: true }),
      ).not.toBeVisible();
      await json(
        await request.post(`/api/issues/${issue.id}/comments`, {
          data: { body: "Keeping this review pending while I check." },
        }),
      );
      await page
        .getByRole("button", { name: "Review request", exact: true })
        .click();
      await expect(
        page.getByRole("button", { name: "Approve & run", exact: true }),
      ).toBeVisible();
      const queue = await context.newPage();
      await queue.goto(`/${seed.prefix}/apps/review`);
      await expect(
        queue.getByRole("button", { name: "Decline", exact: true }),
      ).toBeVisible({ timeout: 20_000 });
      if (journey === "restart") {
        const control = process.env.PAPERCLIP_REVIEW_RESTART_FILE!;
        const token = String(Date.now());
        await writeFile(control, `restart:${token}`);
        await expect
          .poll(() => readFile(control, "utf8"), { timeout: 60_000 })
          .toBe(`started:${token}`);
        await expect
          .poll(
            async () => {
              try {
                return (await request.get("/api/health")).ok();
              } catch {
                return false;
              }
            },
            { timeout: 60_000 },
          )
          .toBe(true);
        await page.reload();
        await queue.reload();
        await expect(
          page.getByRole("button", { name: "Approve & run", exact: true }),
        ).not.toBeVisible();
        await page.getByRole("button", { name: "Review request", exact: true }).click();
        await expect(
          page.getByRole("button", { name: "Approve & run", exact: true }),
        ).toBeVisible();
        expect(calls()).toBe(0);
      }
      if (journey === "decline") {
        // Resolve from Connections and observe the still-open task tab update.
        await queue
          .getByRole("button", { name: "Decline", exact: true })
          .click();

      } else {
        if (journey === "always") {
          await page.getByRole("button", { name: "Approval options", exact: true }).click();
          await page.getByRole("menuitem", { name: "Always allow", exact: true }).click();
        } else {
          await page.getByRole("button", { name: "Approve & run", exact: true }).click();
        }
      }
      await expect
        .poll(
          async () =>
            (
              await json<Array<{ body: string }>>(
                await request.get(`/api/issues/${issue.id}/comments`),
              )
            )
              .map((c) => c.body)
              .join("\n"),
          { timeout: 60_000 },
        )
        .toContain(
          journey === "decline"
            ? "Review declined"
            : journey === "failure"
              ? "Read failed"
              : "Roadmap",
        );
      const replies = await json<Array<{ body: string; authorAgentId: string | null; createdByRunId: string | null }>>(await request.get(`/api/issues/${issue.id}/comments`));
      const reply = replies.find(comment => comment.authorAgentId === agent.id)!;
      expect(reply.createdByRunId).toBeTruthy();
      expect(originatingReview.sourceRunId).toBeTruthy();
      expect(reply.createdByRunId).not.toBe(originatingReview.sourceRunId);
      expect(reply.body).not.toContain('"content":');
      if (!["decline", "failure"].includes(journey)) {
        await expect(page.getByRole("button", { name: "Show result details" })).toBeVisible();
        await expect(page.locator("pre")).not.toBeVisible();
        await page.getByRole("button", { name: "Show result details" }).click();
        await expect(page.locator("pre")).toContainText('"content":');
        await page.getByRole("button", { name: "Hide result details" }).click();
      }
      expect(calls()).toBe(journey === "decline" ? 0 : 1);
      await expect(
        page.getByRole("button", { name: "Approve & run", exact: true }),
      ).not.toBeVisible();
      await expect(
        queue.getByRole("button", { name: "Decline", exact: true }),
      ).not.toBeVisible({ timeout: 20_000 });
      await page.bringToFront();
      await page.reload();
      await expect(page.getByTestId("task-chat-history-content")).toBeVisible({
        timeout: 30_000,
      });
      await expect(
        page
          .getByText(
            journey === "decline"
              ? "Declined"
              : journey === "failure"
                ? "Execution failed"
                : "Succeeded",
            { exact: false },
          )
          .first(),
      ).toBeVisible();
      await page.screenshot({
        path: testInfo.outputPath("resolved-review.png"),
      });
      if (journey === "always") {
        await configureAgent("different search options");
        const later = await json<{ id: string }>(
          await request.post(`/api/companies/${seed.companyId}/issues`, {
            data: {
              title: "Read pages with different arguments",
              status: "in_progress",
              assigneeAgentId: agent.id,
            },
          }),
        );
        await expect
          .poll(
            async () =>
              (
                await json<Array<{ body: string }>>(
                  await request.get(`/api/issues/${later.id}/comments`),
                )
              )
                .map((c) => c.body)
                .join("\n"),
            { timeout: 45_000 },
          )
          .toContain("Roadmap");
        expect(calls()).toBe(2);
        expect(
          await json<unknown[]>(
            await request.get(`/api/issues/${later.id}/interactions`),
          ),
        ).toHaveLength(0);
      }
      await testInfo.attach("journey", {
        body: JSON.stringify(
          {
            source: "local MCP fixture; no live Notion",
            journey,
            ...seed,
            issue,
            agent,
            providerCalls: calls(),
          },
          null,
          2,
        ),
        contentType: "application/json",
      });
      await queue.close();
    } finally {
      await provider.close();
    }
  });
}