PaperClipAI/scripts/smoke/posthog-live.mjs

#!/usr/bin/env node

import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import {
  assertSanitizedEvidence,
  extractProjectSummary,
  parsePosthogLiveArguments,
  parseSanitizedAgentProof,
  PosthogLivePreflightError,
  preflightFailureMessage,
  preparePosthogLiveSmoke,
} from "./posthog-live-lib.mjs";

const TARGET_COMPANY_PREFIX = "PAP";
const TARGET_AGENT_NAME = "CodexCoderPro";
const PROJECT_GET = "project-get";
const PROJECT_SETTINGS_UPDATE = "project-settings-update";
const EXCLUDED_PROJECT_SWITCHERS = new Set(["switch-project", "switch-organization"]);
const DEFAULT_AGENT_TIMEOUT_MS = 15 * 60_000;

class SmokeFailure extends Error {
  constructor(checkpoint, code, details = null) {
    super(`${checkpoint}:${code}`);
    this.name = "SmokeFailure";
    this.checkpoint = checkpoint;
    this.code = code;
    this.details = details;
  }
}

function fail(checkpoint, code, details = null) {
  throw new SmokeFailure(checkpoint, code, details);
}

function asArray(value, key) {
  if (Array.isArray(value)) return value;
  if (value && typeof value === "object" && Array.isArray(value[key])) return value[key];
  return [];
}

function escapeRegex(value) {
  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

function screenshotFile(outputDirectory, name) {
  return path.join(outputDirectory, name);
}

function safeConnectionConfig(connection, expectedProjectId) {
  const methodConfig = connection?.config?.methodConfig;
  if (!methodConfig || typeof methodConfig !== "object" || Array.isArray(methodConfig)) {
    fail("C.connection-detail", "method_config_missing");
  }
  if (connection.config?.connectionMethodKey !== "mcp-oauth") {
    fail("C.connection-detail", "oauth_method_not_recorded");
  }
  if (String(methodConfig.projectId ?? "") !== expectedProjectId) {
    fail("C.connection-detail", "wrong_project_pin");
  }
  if (methodConfig.readOnly !== false || methodConfig.mode !== "tools") {
    fail("C.connection-detail", "unexpected_posthog_scope");
  }
  if ((methodConfig.features ?? "") !== "" || (methodConfig.tools ?? "") !== "") {
    fail("C.connection-detail", "unexpected_tool_filter");
  }
  if (!connection.config?.oauth?.connectedAt) {
    fail("B.oauth-callback", "oauth_not_connected");
  }
}

function assertNoCredentialMaterial(value, secrets, checkpoint) {
  const serialized = JSON.stringify(value);
  for (const secret of secrets) {
    if (secret && serialized.includes(secret)) fail(checkpoint, "credential_material_visible");
  }
  if (/[?&](?:code|state|access_token|refresh_token)=/i.test(serialized)) {
    fail(checkpoint, "oauth_query_material_visible");
  }
  const forbiddenValueKey = /^(?:password|accessToken|access_token|refreshToken|refresh_token|oauthCode|oauth_code|clientSecret|client_secret|cookie|authorizationHeader)$/i;
  const visit = (candidate, seen = new Set()) => {
    if (!candidate || typeof candidate !== "object" || seen.has(candidate)) return;
    seen.add(candidate);
    for (const [key, child] of Object.entries(candidate)) {
      if (forbiddenValueKey.test(key) && child !== null && child !== "") {
        fail(checkpoint, "raw_credential_field_visible");
      }
      visit(child, seen);
    }
  };
  visit(value);
}

async function apiJson(request, baseUrl, method, pathname, data, checkpoint, expectedStatuses = [200]) {
  let response;
  try {
    const origin = new URL(baseUrl).origin;
    response = await request.fetch(new URL(pathname, baseUrl).toString(), {
      method,
      ...(data === undefined ? {} : { data }),
      headers: {
        accept: "application/json",
        origin,
        referer: `${origin}/`,
      },
      timeout: 30_000,
    });
  } catch {
    fail(checkpoint, "request_failed");
  }
  if (!expectedStatuses.includes(response.status())) {
    fail(checkpoint, `http_${response.status()}`);
  }
  try {
    return await response.json();
  } catch {
    fail(checkpoint, "invalid_json");
  }
}

async function waitFor(checkpoint, fn, { timeoutMs = 120_000, intervalMs = 2_000 } = {}) {
  const deadline = Date.now() + timeoutMs;
  let lastValue;
  while (Date.now() < deadline) {
    lastValue = await fn();
    if (lastValue) return lastValue;
    await new Promise((resolve) => setTimeout(resolve, intervalMs));
  }
  fail(checkpoint, "timed_out");
  return lastValue;
}

async function expectVisible(locator, checkpoint, code, timeout = 30_000) {
  try {
    await locator.waitFor({ state: "visible", timeout });
  } catch {
    fail(checkpoint, code);
  }
}

async function gotoPaperclipPage(
  page,
  url,
  readyLocator,
  checkpoint,
  code,
  { attempts = 3, timeout = 15_000 } = {},
) {
  for (let attempt = 1; attempt <= attempts; attempt += 1) {
    try {
      await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30_000 });
      await readyLocator.waitFor({ state: "visible", timeout });
      return;
    } catch {
      if (attempt < attempts) await page.waitForTimeout(500);
    }
  }
  fail(checkpoint, code);
}

async function stabilizeCompanyContext(page, config, companyId) {
  const galleryPath = `/api/companies/${companyId}/tools/gallery`;
  for (let attempt = 1; attempt <= 3; attempt += 1) {
    try {
      const galleryResponsePromise = page.waitForResponse(
        (response) => response.request().method() === "GET"
          && new URL(response.url()).pathname === galleryPath,
        { timeout: 30_000 },
      );
      await page.goto(new URL(`/${TARGET_COMPANY_PREFIX}/apps`, config.baseUrl).toString(), {
        waitUntil: "domcontentloaded",
        timeout: 30_000,
      });
      const response = await galleryResponsePromise;
      if (!response.ok()) continue;
      const posthogAction = page.getByRole("button", {
        name: /^(?:Connect for PostHog|Add another PostHog account)$/,
      }).first();
      await posthogAction.waitFor({ state: "visible", timeout: 30_000 });
      // The company-prefixed route and selected-company provider settle in
      // separate renders. Clicking the tile immediately can carry the prior
      // company's gallery cache into the setup effect and redirect back out.
      await page.waitForTimeout(2_000);
      return;
    } catch {
      if (attempt < 3) await page.waitForTimeout(500);
    }
  }
  fail("A.company-context", "posthog_gallery_context_missing");
}

async function openPosthogSetupFromGallery(page, config, companyId) {
  for (let attempt = 1; attempt <= 3; attempt += 1) {
    await stabilizeCompanyContext(page, config, companyId);
    const addAnother = page.getByRole("button", { name: /^Add another PostHog account$/ }).first();
    const connect = page.getByRole("button", { name: /^Connect for PostHog$/ }).first();
    const action = await addAnother.isVisible().catch(() => false) ? addAnother : connect;
    try {
      await action.click({ timeout: 5_000 });
      await page.getByRole("button", { name: "Sign in with PostHog" }).waitFor({
        state: "visible",
        timeout: 30_000,
      });
      return;
    } catch {
      if (attempt < 3) await page.waitForTimeout(500);
    }
  }
  fail("A.setup-route", "oauth_method_missing");
}

async function safePageState(page, resourceFailures, paperclipOrigin) {
  let current;
  try {
    current = new URL(page.url());
  } catch {
    return { location: "invalid", resourceFailures };
  }
  const bodyText = await page.locator("body").innerText().catch(() => "");
  return {
    location: current.origin === paperclipOrigin
      ? `${current.hostname}${current.pathname}`
      : current.hostname,
    headingCount: await page.getByRole("heading").count().catch(() => 0),
    buttonCount: await page.getByRole("button").count().catch(() => 0),
    methodSignals: {
      posthogSignIn: /sign in with posthog/i.test(bodyText),
      personalApiKey: /personal api key/i.test(bodyText),
      connectApp: /connect an app/i.test(bodyText),
    },
    resourceFailures,
  };
}

async function clickVisibleButton(page, names) {
  for (const name of names) {
    for (const role of ["button", "link"]) {
      const control = page.getByRole(role, { name, exact: false }).filter({ visible: true }).first();
      if (await control.count()) {
        try {
          await control.click({ timeout: 2_000 });
          return true;
        } catch {
          // Provider pages often replace their form between locator creation
          // and click. The next loop re-reads the current DOM.
        }
      }
    }
  }
  return false;
}

async function selectPosthogCloudRegion(page) {
  const region = (process.env.POSTHOG_CLOUD_REGION || "us").trim().toLowerCase();
  if (!new Set(["us", "eu"]).has(region)) {
    fail("B.oauth-callback", "unsupported_cloud_region");
  }
  const expectedHost = `${region}.posthog.com`;
  const links = page.getByRole("link");
  for (let index = 0; index < await links.count(); index += 1) {
    const link = links.nth(index);
    const href = await link.getAttribute("href");
    if (!href) continue;
    try {
      const target = new URL(href, page.url());
      if (target.hostname !== expectedHost) continue;
      await page.goto(target.toString(), { waitUntil: "domcontentloaded", timeout: 30_000 });
      return true;
    } catch {
      // The provider can replace this chooser while the link is being read.
      // The next authorization-loop iteration re-evaluates it.
    }
  }
  return false;
}

async function completePosthogAuthorization(page, config) {
  const paperclipOrigin = new URL(config.baseUrl).origin;
  const providerTimeoutMs = Number(process.env.POSTHOG_PROVIDER_TIMEOUT_MS || 4 * 60_000);
  const deadline = Date.now() + (Number.isFinite(providerTimeoutMs) && providerTimeoutMs > 0
    ? providerTimeoutMs
    : 4 * 60_000);
  let providerState = null;
  let credentialFormSubmitted = false;
  while (Date.now() < deadline) {
    let current;
    try {
      current = new URL(page.url());
    } catch {
      fail("B.oauth-callback", "invalid_navigation_url");
    }
    if (current.origin === paperclipOrigin && current.pathname.includes("/apps/")) return;

    const emailInput = page.locator('input[type="email"], input[name="email"], input[autocomplete="username"]').filter({ visible: true }).first();
    const passwordInput = page.locator('input[type="password"], input[name="password"], input[autocomplete="current-password"]').filter({ visible: true }).first();
    const identityFieldVisible = await emailInput.count() > 0;
    const credentialFieldVisible = await passwordInput.count() > 0;
    const credentialForm = page.locator("form").filter({ has: passwordInput }).first();
    const submitControl = credentialForm.locator('button[type="submit"], input[type="submit"]').filter({ visible: true }).first();
    const consentControlVisible = await page.getByRole("button", {
      name: /^(?:authorize|allow|approve|grant access|accept)$/i,
    }).first().isVisible().catch(() => false);
    const bodyText = await page.locator("body").innerText().catch(() => "");
    const linkHrefs = await page.getByRole("link").evaluateAll((links) =>
      links.map((link) => link.getAttribute("href")).filter(Boolean)
    ).catch(() => []);
    const linkTargets = Array.from(new Set(linkHrefs.map((href) => {
      try {
        const target = new URL(href, current.origin);
        return `${target.hostname}${target.pathname}`;
      } catch {
        return "invalid";
      }
    })));
    providerState = {
      host: current.hostname,
      path: current.pathname.slice(0, 200),
      identityFieldVisible,
      credentialFieldVisible,
      credentialFormVisible: await credentialForm.count() > 0,
      submitControlVisible: await submitControl.count() > 0,
      submitControlDisabled: await submitControl.isDisabled().catch(() => false),
      consentControlVisible,
      alertCount: await page.getByRole("alert").count(),
      headingCount: await page.getByRole("heading").count(),
      buttonCount: await page.getByRole("button").count(),
      linkCount: await page.getByRole("link").count(),
      linkTargets,
      frameCount: page.frames().length,
      semanticSignals: {
        signIn: /\b(?:sign in|log in)\b/i.test(bodyText),
        continue: /\bcontinue\b/i.test(bodyText),
        consent: /\b(?:authorize|allow|approve|grant access|accept)\b/i.test(bodyText),
        loading: /\b(?:loading|preparing|opening|redirecting)\b/i.test(bodyText),
        workspace: /\b(?:workspace|organization|project)\b/i.test(bodyText),
        error: /\b(?:error|invalid|failed|problem|went wrong)\b/i.test(bodyText),
      },
    };
    if (current.hostname === "oauth.posthog.com" && await selectPosthogCloudRegion(page)) {
      await page.waitForTimeout(500);
      continue;
    }
    if (await emailInput.count()) {
      const currentValue = await emailInput.inputValue().catch(() => "");
      if (!currentValue) await emailInput.fill(config.email);
    }

    if (await passwordInput.count()) {
      const currentValue = await passwordInput.inputValue().catch(() => "");
      if (!currentValue) await passwordInput.fill(config.password);
      if (!credentialFormSubmitted) {
        if (await submitControl.count() && !await submitControl.isDisabled().catch(() => true)) {
          await submitControl.click({ noWaitAfter: true, timeout: 2_000 }).catch(() => {});
        } else {
          const submitted = await clickVisibleButton(page, [
            /^sign in$/i,
            /^log in$/i,
            /^login$/i,
            /^continue$/i,
            /sign in with email/i,
            /log in with email/i,
            /login with email/i,
          ]);
          if (!submitted) await passwordInput.press("Enter").catch(() => {});
        }
        credentialFormSubmitted = true;
      }
    } else if (await emailInput.count()) {
      await clickVisibleButton(page, [/^continue$/i, /^next$/i, /continue with email/i, /sign in with email/i]);
    } else {
      await clickVisibleButton(page, [
        /^authorize$/i,
        /^allow$/i,
        /^approve$/i,
        /^grant access$/i,
        /^accept$/i,
        /^continue$/i,
        /^sign in$/i,
        /^log in$/i,
      ]);
    }
    await page.waitForTimeout(500);
  }
  fail("B.oauth-callback", "provider_authorization_timed_out", { providerState });
}

async function safeScreenshot(page, outputPath, config, checkpoint) {
  const current = new URL(page.url());
  if (current.origin !== new URL(config.baseUrl).origin) fail(checkpoint, "screenshot_not_on_paperclip");
  for (const queryKey of ["code", "state", "token", "access_token", "refresh_token"]) {
    if (current.searchParams.has(queryKey)) fail(checkpoint, "credential_query_in_screenshot_url");
  }
  const bodyText = await page.locator("body").innerText();
  if (bodyText.includes(config.password) || /[?&](?:code|state|access_token|refresh_token)=/i.test(bodyText)) {
    fail(checkpoint, "credential_material_in_screenshot");
  }
  await page.screenshot({
    path: outputPath,
    fullPage: true,
    animations: "disabled",
    mask: [page.getByText(config.email, { exact: false })],
  });
}

function catalogFacts(catalog, checkpoint) {
  const active = catalog.filter((entry) => entry.status !== "removed");
  const projectGet = active.find((entry) => entry.toolName === PROJECT_GET);
  const projectSettings = active.find((entry) => entry.toolName === PROJECT_SETTINGS_UPDATE);
  if (!projectGet || !projectGet.isReadOnly) fail(checkpoint, "project_get_missing_or_not_read_only");
  if (!projectSettings || projectSettings.isReadOnly) fail(checkpoint, "project_settings_update_missing_or_not_write");
  for (const excluded of EXCLUDED_PROJECT_SWITCHERS) {
    if (active.some((entry) => entry.toolName === excluded)) fail(checkpoint, `excluded_${excluded}_present`);
  }
  return { active, projectGet, projectSettings };
}

async function finishAgentOnlySetup(request, config, companyId, connectionId, catalog, agentId) {
  const { active } = catalogFacts(catalog, "C.catalog-policy");
  const enabledCatalogEntryIds = active.filter((entry) => entry.isReadOnly).map((entry) => entry.id);
  await apiJson(
    request,
    config.baseUrl,
    "POST",
    `/api/companies/${companyId}/tools/apps/${connectionId}/finish`,
    {
      enabledCatalogEntryIds,
      askFirstCatalogEntryIds: [],
      reviewedCatalogEntryIds: active.filter((entry) => entry.status === "quarantined").map((entry) => entry.id),
      access: { agentIds: [agentId] },
    },
    "C.catalog-policy",
  );
  await apiJson(
    request,
    config.baseUrl,
    "PUT",
    `/api/tool-connections/${connectionId}/installs`,
    { installs: [{ targetType: "agent", targetId: agentId }] },
    "C.agent-install",
  );
}

async function findConnectionIdByName(request, config, companyId, connectionName) {
  const connectionsResponse = await apiJson(
    request,
    config.baseUrl,
    "GET",
    `/api/companies/${companyId}/tools/connections`,
    undefined,
    "F.cleanup-recovery",
  );
  const matching = asArray(connectionsResponse, "connections").filter(
    (connection) => connection.name === connectionName && connection.status !== "archived",
  );
  if (matching.length > 1) fail("F.cleanup-recovery", "duplicate_test_connections");
  return matching[0]?.id ?? null;
}

async function cleanupConnection(
  request,
  config,
  companyId,
  connectionId,
  connectionName,
  { requireInstalledState = true } = {},
) {
  const removed = await apiJson(
    request,
    config.baseUrl,
    "DELETE",
    `/api/tool-connections/${connectionId}`,
    undefined,
    "F.cleanup",
  );
  const receipt = removed.removal;
  if (!receipt || (requireInstalledState && (
    receipt.installsRemoved < 1
    || receipt.appProfileBindingsRemoved < 1
    || receipt.credentialRefsCleared + receipt.secretsRevoked < 1
    || !["deleted", "archived"].includes(receipt.appProfile)
  ))) {
    fail("F.cleanup", "incomplete_removal_receipt");
  }

  const connectionsResponse = await apiJson(
    request,
    config.baseUrl,
    "GET",
    `/api/companies/${companyId}/tools/connections`,
    undefined,
    "F.cleanup-verification",
  );
  const remaining = asArray(connectionsResponse, "connections").filter(
    (connection) => connection.name === connectionName && connection.status !== "archived",
  );
  if (remaining.length > 0) fail("F.cleanup-verification", "test_connection_remains");

  const profilesResponse = await apiJson(
    request,
    config.baseUrl,
    "GET",
    `/api/companies/${companyId}/tools/profiles`,
    undefined,
    "F.cleanup-verification",
  );
  if (asArray(profilesResponse, "profiles").some((profile) => profile.profileKey === `app:${connectionId}` && profile.status === "active")) {
    fail("F.cleanup-verification", "active_profile_remains");
  }
  const pendingResponse = await apiJson(
    request,
    config.baseUrl,
    "GET",
    `/api/companies/${companyId}/tools/action-requests?status=pending`,
    undefined,
    "F.cleanup-verification",
  );
  if (asArray(pendingResponse, "actionRequests").some((item) => (item.connectionId ?? item.request?.connectionId) === connectionId)) {
    fail("F.cleanup-verification", "pending_action_remains");
  }
  return {
    credentialsTornDown: receipt.credentialRefsCleared + receipt.secretsRevoked > 0,
    accessBindingsRemoved: receipt.appProfileBindingsRemoved,
    installsRemoved: receipt.installsRemoved,
    appProfile: receipt.appProfile,
    pendingActions: 0,
    remainingConnections: 0,
  };
}

async function runSmoke({ config, chromium }) {
  const startedAt = new Date();
  const runKey = startedAt.toISOString().replace(/[:.]/g, "-");
  const connectionName = `PostHog live self-test ${startedAt.toISOString()}`;
  const outputDirectory = process.env.POSTHOG_EVIDENCE_DIR
    ? path.resolve(process.env.POSTHOG_EVIDENCE_DIR)
    : path.join(process.env.PAPERCLIP_RUN_SCRATCH_DIR || process.cwd(), `posthog-live-${runKey}`);
  await mkdir(outputDirectory, { recursive: true });

  const summary = {
    schemaVersion: 1,
    smoke: "posthog_mcp_live",
    passed: false,
    startedAt: startedAt.toISOString(),
    completedAt: null,
    target: { companyPrefix: TARGET_COMPANY_PREFIX, projectId: config.projectId },
    connection: null,
    catalog: null,
    boardTest: null,
    freshRun: null,
    cleanup: null,
    screenshots: [],
    failure: null,
  };

  let browser;
  let context;
  let page;
  let connectionId = null;
  let companyId = null;
  let cleanupComplete = false;
  let caughtFailure = null;
  let activeCheckpoint = "A.browser-launch";
  const resourceFailures = [];

  try {
    browser = await chromium.launch({ headless: process.env.POSTHOG_SMOKE_HEADED !== "1" });
    context = await browser.newContext({
      viewport: { width: 1440, height: 1000 },
      acceptDownloads: false,
      serviceWorkers: "block",
    });
    page = await context.newPage();
    page.on("requestfailed", (request) => {
      if (!["document", "script", "stylesheet", "xhr", "fetch"].includes(request.resourceType())) return;
      try {
        const target = new URL(request.url());
        if (target.origin !== new URL(config.baseUrl).origin) return;
        resourceFailures.push({
          target: `${target.hostname}${target.pathname}`,
          resourceType: request.resourceType(),
          error: request.failure()?.errorText ?? "unknown",
        });
        if (resourceFailures.length > 12) resourceFailures.shift();
      } catch {
        // Ignore malformed resource URLs rather than copying them into evidence.
      }
    });

    activeCheckpoint = "A.paperclip-login";
    await gotoPaperclipPage(
      page,
      new URL("/auth?next=/", config.baseUrl).toString(),
      page.locator("#email"),
      "A.paperclip-login",
      "email_field_missing",
    );
    await page.locator("#email").fill(config.email);
    await page.locator("#password").fill(config.password);
    const loginResponsePromise = page.waitForResponse((response) =>
      response.request().method() === "POST" && new URL(response.url()).pathname === "/api/auth/sign-in/email",
    );
    await page.getByRole("button", { name: /^sign in$/i }).click();
    const loginResponse = await loginResponsePromise;
    if (!loginResponse.ok()) fail("A.paperclip-login", `http_${loginResponse.status()}`);
    await page.waitForURL((url) => url.pathname !== "/auth", { timeout: 30_000 }).catch(() => {
      fail("A.paperclip-login", "login_redirect_missing");
    });

    activeCheckpoint = "A.company-selection";
    const companiesResponse = await apiJson(context.request, config.baseUrl, "GET", "/api/companies", undefined, "A.company-selection");
    const company = asArray(companiesResponse, "companies").find((candidate) => candidate.issuePrefix === TARGET_COMPANY_PREFIX);
    if (!company) fail("A.company-selection", "pap_company_missing");
    companyId = company.id;

    activeCheckpoint = "C.agent-scope";
    const agentsResponse = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/companies/${companyId}/agents`,
      undefined,
      "C.agent-scope",
    );
    const agent = asArray(agentsResponse, "agents").find((candidate) => candidate.name === TARGET_AGENT_NAME);
    if (!agent) fail("C.agent-scope", "codex_coder_pro_missing");

    activeCheckpoint = "A.setup-route";
    await openPosthogSetupFromGallery(page, config, companyId);
    await expectVisible(page.getByRole("button", { name: "Use a personal API key" }), "A.setup-route", "api_key_method_missing");

    activeCheckpoint = "B.oauth-setup";
    await page.getByRole("button", { name: "Sign in with PostHog" }).click();
    const nameInput = page.locator('input[placeholder="My app"]');
    await nameInput.fill(connectionName);
    await page.locator('input[placeholder="12345"]').fill(config.projectId);
    const readOnlyScope = page.locator("div").filter({ hasText: /^Read-only mode/ }).filter({ has: page.getByRole("switch") }).first();
    const readOnlySwitch = readOnlyScope.getByRole("switch");
    if (await readOnlySwitch.getAttribute("aria-checked") !== "false") fail("B.oauth-setup", "read_only_default_changed");
    await page.getByText("Advanced", { exact: true }).click();
    const featuresInput = page.locator("label", { hasText: "Feature groups" }).locator("..").locator("textarea");
    const toolsInput = page.locator("label", { hasText: "Individual tools" }).locator("..").locator("textarea");
    if (await featuresInput.inputValue() !== "" || await toolsInput.inputValue() !== "") {
      fail("B.oauth-setup", "tool_filters_not_blank");
    }
    const responseMode = page.locator("label", { hasText: "Tool response mode" }).locator("..").locator("select");
    if (await responseMode.inputValue() !== "tools") fail("B.oauth-setup", "individual_tools_mode_not_selected");

    activeCheckpoint = "B.oauth-start";
    const connectResponsePromise = page.waitForResponse(
      (response) => {
        const target = new URL(response.url());
        return response.request().method() === "POST"
          && target.pathname === `/api/companies/${companyId}/tools/apps/connect`;
      },
      { timeout: 120_000 },
    );
    await page.getByRole("button", { name: "Continue to sign in" }).click({ noWaitAfter: true });
    const connectResponse = await connectResponsePromise;
    if (!connectResponse.ok()) fail("B.oauth-start", `http_${connectResponse.status()}`);
    let connectResult = null;
    try {
      connectResult = await connectResponse.json();
    } catch {
      // A successful create immediately redirects the page to PostHog. Chromium
      // can discard that response body during the cross-origin navigation, so
      // recover the uniquely named draft instead of orphaning it.
    }
    connectionId = connectResult?.connectionId ?? await waitFor(
      "B.oauth-start",
      () => findConnectionIdByName(context.request, config, companyId, connectionName),
      { timeoutMs: 15_000, intervalMs: 500 },
    );
    if (typeof connectionId !== "string" || !connectionId) fail("B.oauth-start", "connection_id_missing");

    activeCheckpoint = "B.oauth-callback";
    await completePosthogAuthorization(page, config);
    const permissionsPath = `/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`;
    await gotoPaperclipPage(
      page,
      new URL(permissionsPath, config.baseUrl).toString(),
      page.getByText("PostHog connected", { exact: true }),
      "B.oauth-callback",
      "connected_state_missing",
      { attempts: 3, timeout: 30_000 },
    );

    activeCheckpoint = "C.connection-detail";
    let connection = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/tool-connections/${connectionId}`,
      undefined,
      "C.connection-detail",
    );
    safeConnectionConfig(connection, config.projectId);
    assertNoCredentialMaterial(connection, [config.password], "C.connection-detail");
    summary.connection = {
      id: connectionId,
      name: connectionName,
      authentication: "oauth",
      projectId: config.projectId,
      status: connection.status,
      healthStatus: connection.healthStatus,
    };
    const connectedShot = "01-connected-setup.png";
    await safeScreenshot(page, screenshotFile(outputDirectory, connectedShot), config, "F.connected-screenshot");
    summary.screenshots.push(connectedShot);

    activeCheckpoint = "C.catalog-policy";
    let catalogResponse = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/tool-connections/${connectionId}/catalog`,
      undefined,
      "C.catalog-policy",
    );
    let catalog = asArray(catalogResponse, "catalog");
    let facts = catalogFacts(catalog, "C.catalog-policy");
    await finishAgentOnlySetup(context.request, config, companyId, connectionId, catalog, agent.id);

    const health = await apiJson(
      context.request,
      config.baseUrl,
      "POST",
      `/api/tool-connections/${connectionId}/health-check`,
      {},
      "C.health-check",
    );
    if (health.connection?.healthStatus !== "healthy") fail("C.health-check", "connection_not_healthy");
    const refreshed = await apiJson(
      context.request,
      config.baseUrl,
      "POST",
      `/api/tool-connections/${connectionId}/catalog/refresh`,
      {},
      "C.catalog-refresh",
    );
    catalog = asArray(refreshed, "catalog");
    facts = catalogFacts(catalog, "C.catalog-refresh");
    await finishAgentOnlySetup(context.request, config, companyId, connectionId, catalog, agent.id);

    connection = await apiJson(context.request, config.baseUrl, "GET", `/api/tool-connections/${connectionId}`, undefined, "C.connection-detail");
    safeConnectionConfig(connection, config.projectId);
    if (connection.status !== "active" || connection.healthStatus !== "healthy") {
      fail("C.connection-detail", "connection_not_active_and_healthy");
    }
    assertNoCredentialMaterial(connection, [config.password], "C.connection-detail");
    summary.connection.status = connection.status;
    summary.connection.healthStatus = connection.healthStatus;

    const uniqueConnections = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/companies/${companyId}/tools/connections`,
      undefined,
      "C.connection-detail",
    );
    if (asArray(uniqueConnections, "connections").filter((candidate) => candidate.name === connectionName).length !== 1) {
      fail("C.connection-detail", "duplicate_connection_detected");
    }

    const installs = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/tool-connections/${connectionId}/installs`,
      undefined,
      "C.agent-install",
    );
    const installRows = asArray(installs, "installs");
    if (installRows.length !== 1 || installRows[0].targetType !== "agent" || installRows[0].targetId !== agent.id) {
      fail("C.agent-install", "install_not_agent_only");
    }

    const testAgents = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/tool-connections/${connectionId}/test-agents`,
      undefined,
      "C.effective-policy",
    );
    const testAgent = asArray(testAgents, "agents").find((candidate) => candidate.id === agent.id);
    const projectGetAccess = testAgent?.effectiveAccess?.tools?.find((tool) => tool.toolName === PROJECT_GET);
    const projectSettingsAccess = testAgent?.effectiveAccess?.tools?.find((tool) => tool.toolName === PROJECT_SETTINGS_UPDATE);
    if (projectGetAccess?.decision !== "allowed" || projectSettingsAccess?.decision !== "off") {
      fail("C.effective-policy", "unexpected_effective_decision");
    }
    summary.catalog = {
      discoveredCount: refreshed.discoveredCount,
      projectGet: { catalogEntryId: facts.projectGet.id, toolName: PROJECT_GET, decision: "allowed" },
      projectSettingsUpdate: { catalogEntryId: facts.projectSettings.id, toolName: PROJECT_SETTINGS_UPDATE, decision: "off" },
      excludedToolsAbsent: [...EXCLUDED_PROJECT_SWITCHERS],
      accessAgentId: agent.id,
      installAgentId: agent.id,
      healthCheck: "healthy",
      catalogRefresh: "succeeded",
    };

    activeCheckpoint = "C.permissions-ui";
    await gotoPaperclipPage(
      page,
      new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/permissions`, config.baseUrl).toString(),
      page.getByText("Who can use it", { exact: true }),
      "C.permissions-ui",
      "permissions_panel_missing",
    );
    const projectGetPermission = page.locator(`[data-action-id="${facts.projectGet.id}"] select`);
    const projectSettingsPermission = page.locator(`[data-action-id="${facts.projectSettings.id}"] select`);
    await expectVisible(projectGetPermission, "C.permissions-ui", "project_get_permission_missing");
    await expectVisible(projectSettingsPermission, "C.permissions-ui", "project_settings_permission_missing");
    if (await projectGetPermission.inputValue() !== "allowed" || await projectSettingsPermission.inputValue() !== "off") {
      fail("C.permissions-ui", "permissions_ui_mismatch");
    }
    const permissionsShot = "02-scoped-permissions.png";
    await safeScreenshot(page, screenshotFile(outputDirectory, permissionsShot), config, "F.permissions-screenshot");
    summary.screenshots.push(permissionsShot);

    activeCheckpoint = "D.test-panel";
    await gotoPaperclipPage(
      page,
      new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/test`, config.baseUrl).toString(),
      page.getByLabel("Choose which agent to test as"),
      "D.test-panel",
      "agent_picker_missing",
    );
    await page.getByLabel("Choose which agent to test as").click();
    await page.getByLabel("Search agents").fill(TARGET_AGENT_NAME);
    await page.getByRole("button", { name: new RegExp(`^${escapeRegex(TARGET_AGENT_NAME)}`) }).click();
    await page.getByLabel("Find an action").fill(PROJECT_GET);
    const projectGetTitle = facts.projectGet.title ?? facts.projectGet.toolName;
    const actionRow = page.locator("button").filter({ hasText: projectGetTitle }).filter({ hasText: "Allowed" }).first();
    await expectVisible(actionRow, "D.test-panel", "project_get_allowed_row_missing");
    await actionRow.click();
    await expectVisible(page.getByText("This action takes no inputs."), "D.test-panel", "empty_input_form_missing");

    activeCheckpoint = "D.project-get";
    const boardTestStartedAt = Date.now();
    const testCallResponsePromise = page.waitForResponse((response) =>
      response.request().method() === "POST"
      && new URL(response.url()).pathname === `/api/tool-connections/${connectionId}/test-calls`,
    );
    await page.getByRole("button", { name: /^run$/i }).click();
    const testCallResponse = await testCallResponsePromise;
    if (!testCallResponse.ok()) fail("D.project-get", `http_${testCallResponse.status()}`);
    let testCall;
    try {
      testCall = await testCallResponse.json();
    } catch {
      fail("D.project-get", "invalid_json");
    }
    let testCallInput;
    try {
      testCallInput = testCallResponse.request().postDataJSON();
    } catch {
      fail("D.project-get", "request_body_unavailable");
    }
    if (testCallInput?.agentId !== agent.id
      || testCallInput?.toolName !== PROJECT_GET
      || !testCallInput.parameters
      || typeof testCallInput.parameters !== "object"
      || Array.isArray(testCallInput.parameters)
      || Object.keys(testCallInput.parameters).length !== 0) {
      fail("D.project-get", "nonempty_or_unexpected_input");
    }
    if (testCall.decision !== "allowed" || testCall.error || typeof testCall.invocationId !== "string") {
      fail("D.project-get", "gateway_call_not_allowed");
    }
    const boardProject = extractProjectSummary(testCall.result, config.projectId);
    if (!boardProject?.name) fail("D.project-get", "project_result_missing");
    await expectVisible(page.getByText(/^Worked\./), "D.project-get", "success_result_missing");
    const boardShot = "03-board-project-get.png";
    await safeScreenshot(page, screenshotFile(outputDirectory, boardShot), config, "F.board-test-screenshot");
    summary.screenshots.push(boardShot);
    summary.boardTest = {
      catalogEntryId: facts.projectGet.id,
      toolName: PROJECT_GET,
      invocationId: testCall.invocationId,
      decision: testCall.decision,
      httpStatus: testCallResponse.status(),
      resultStatus: "succeeded",
      project: boardProject,
      durationMs: Date.now() - boardTestStartedAt,
    };

    activeCheckpoint = "E.create-proof-issue";
    const parentIssueId = process.env.POSTHOG_PROOF_PARENT_ISSUE_ID || process.env.PAPERCLIP_TASK_ID;
    if (!parentIssueId) fail("E.create-proof-issue", "parent_issue_id_missing");
    const child = await apiJson(
      context.request,
      config.baseUrl,
      "POST",
      `/api/issues/${parentIssueId}/children`,
      {
        title: `PostHog installed-tool proof ${startedAt.toISOString()}`,
        description: [
          "Invoke exactly one installed PostHog action: the read-only upstream `project-get` tool, with an empty `{}` input.",
          `Verify the returned project ID is exactly ${config.projectId} and make no PostHog mutations.`,
          "Then post exactly one JSON object with keys `projectId`, `projectName`, and `invocationId` (the Paperclip invocation ID), and mark this issue done.",
          "Do not report tokens, cookies, authorization data, request headers, raw tool payloads, or any other fields.",
        ].join("\n\n"),
        status: "todo",
        workMode: "standard",
        priority: "medium",
        assigneeAgentId: agent.id,
        acceptanceCriteria: [
          `The installed PostHog project-get action returns project ${config.projectId}.`,
          "The comment contains only sanitized project ID/name and Paperclip invocation ID.",
          "No mutation is attempted.",
        ],
      },
      "E.create-proof-issue",
      [201],
    );
    if (child.status !== "todo") fail("E.create-proof-issue", "child_not_created_todo");
    activeCheckpoint = "E.fresh-agent-run";
    const observedStatuses = new Set(["todo"]);
    const finishedChild = await waitFor("E.fresh-agent-run", async () => {
      const issue = await apiJson(context.request, config.baseUrl, "GET", `/api/issues/${child.id}`, undefined, "E.fresh-agent-run");
      observedStatuses.add(issue.status);
      if (["blocked", "cancelled"].includes(issue.status)) fail("E.fresh-agent-run", `child_${issue.status}`);
      return issue.status === "done" ? issue : null;
    }, {
      timeoutMs: Number(process.env.POSTHOG_AGENT_TIMEOUT_MS || DEFAULT_AGENT_TIMEOUT_MS),
      intervalMs: 3_000,
    });
    if (!finishedChild.startedAt || !finishedChild.completedAt) fail("E.fresh-agent-run", "transition_timestamps_missing");

    const commentsResponse = await apiJson(
      context.request,
      config.baseUrl,
      "GET",
      `/api/issues/${child.id}/comments`,
      undefined,
      "E.agent-proof-comment",
    );
    const comments = asArray(commentsResponse, "comments");
    for (const comment of comments) assertNoCredentialMaterial(comment.body, [config.password], "E.agent-proof-comment");
    const agentComments = comments.filter((comment) =>
      comment.authorAgentId === agent.id || comment.derivedAuthorAgentId === agent.id,
    );
    const proofs = agentComments.map((comment) => parseSanitizedAgentProof(comment.body, config.projectId)).filter(Boolean);
    if (agentComments.length !== 1 || proofs.length !== 1) {
      fail("E.agent-proof-comment", "single_sanitized_proof_missing");
    }
    const [proof] = proofs;

    let finalConnectionActivity;
    const agentEvent = await waitFor("E.agent-audit", async () => {
      const activity = await apiJson(
        context.request,
        config.baseUrl,
        "GET",
        `/api/tool-connections/${connectionId}/activity?limit=100`,
        undefined,
        "E.agent-audit",
      );
      assertNoCredentialMaterial(activity, [config.password], "E.agent-audit");
      finalConnectionActivity = activity;
      return asArray(activity, "events").find((event) =>
        event.issueId === child.id
        && event.agentId === agent.id
        && event.toolName === PROJECT_GET
        && event.invocationId === proof.invocationId
        && event.outcome === "success",
      ) ?? null;
    }, { timeoutMs: 60_000, intervalMs: 2_000 });
    if (!agentEvent.runId) fail("E.agent-audit", "run_id_missing");
    if (agentEvent.requestSummary?.summary !== "{}") fail("E.agent-audit", "project_get_input_not_empty");
    const childToolEvents = asArray(finalConnectionActivity, "events").filter((event) => event.issueId === child.id && event.invocationId);
    if (childToolEvents.length === 0 || childToolEvents.some((event) => event.toolName !== PROJECT_GET)) {
      fail("E.agent-audit", "unexpected_upstream_action");
    }
    if (new Set(childToolEvents.map((event) => event.invocationId)).size !== 1) {
      fail("E.agent-audit", "project_get_invoked_more_than_once");
    }
    if (proof.projectName !== boardProject.name) fail("E.agent-proof-comment", "project_name_mismatch");

    const agentRun = await waitFor("E.agent-run-status", async () => {
      const run = await apiJson(
        context.request,
        config.baseUrl,
        "GET",
        `/api/heartbeat-runs/${agentEvent.runId}`,
        undefined,
        "E.agent-run-status",
      );
      if (["failed", "cancelled", "timed_out"].includes(run.status)) fail("E.agent-run-status", `run_${run.status}`);
      return run.status === "succeeded" ? run : null;
    }, { timeoutMs: 60_000, intervalMs: 2_000 });

    summary.freshRun = {
      issueId: child.id,
      issueIdentifier: child.identifier,
      transition: {
        created: "todo",
        enteredInProgress: observedStatuses.has("in_progress") || Boolean(finishedChild.startedAt),
        completed: "done",
      },
      runId: agentEvent.runId,
      runStatus: agentRun.status,
      invocationId: proof.invocationId,
      project: { id: proof.projectId, name: proof.projectName },
      auditOutcome: agentEvent.outcome,
      durationMs: agentEvent.latencyMs,
    };

    activeCheckpoint = "F.evidence";
    await gotoPaperclipPage(
      page,
      new URL(`/${TARGET_COMPANY_PREFIX}/issues/${child.identifier}`, config.baseUrl).toString(),
      page.getByText(child.title, { exact: true }).first(),
      "F.child-screenshot",
      "child_issue_missing",
    );
    const childShot = "04-fresh-agent-proof.png";
    await safeScreenshot(page, screenshotFile(outputDirectory, childShot), config, "F.child-screenshot");
    summary.screenshots.push(childShot);

    await gotoPaperclipPage(
      page,
      new URL(`/${TARGET_COMPANY_PREFIX}/apps/${connectionId}/activity`, config.baseUrl).toString(),
      page.getByText(PROJECT_GET, { exact: false }).first(),
      "F.activity-screenshot",
      "project_get_activity_missing",
    );
    const activityShot = "05-redacted-activity.png";
    await safeScreenshot(page, screenshotFile(outputDirectory, activityShot), config, "F.activity-screenshot");
    summary.screenshots.push(activityShot);

    activeCheckpoint = "F.cleanup";
    summary.cleanup = await cleanupConnection(context.request, config, companyId, connectionId, connectionName);
    cleanupComplete = true;
    summary.passed = true;
  } catch (error) {
    caughtFailure = error instanceof SmokeFailure ? error : new SmokeFailure(activeCheckpoint, "unexpected_error");
    if (page) {
      caughtFailure.details = {
        ...(caughtFailure.details ?? {}),
        pageState: await safePageState(page, resourceFailures, new URL(config.baseUrl).origin),
      };
    }
  } finally {
    if (!connectionId && companyId && context) {
      try {
        connectionId = await findConnectionIdByName(
          context.request,
          config,
          companyId,
          connectionName,
        );
      } catch (error) {
        summary.cleanup = {
          completed: false,
          code: error instanceof SmokeFailure ? error.code : "cleanup_recovery_failed",
        };
      }
    }
    if (connectionId && companyId && context && !cleanupComplete) {
      try {
        summary.cleanup = await cleanupConnection(
          context.request,
          config,
          companyId,
          connectionId,
          connectionName,
          { requireInstalledState: false },
        );
        cleanupComplete = true;
      } catch (error) {
        summary.cleanup = {
          completed: false,
          code: error instanceof SmokeFailure ? error.code : "cleanup_failed",
        };
        if (!caughtFailure) caughtFailure = new SmokeFailure("F.cleanup", "cleanup_failed");
      }
    }
    await context?.close().catch(() => {});
    await browser?.close().catch(() => {});
  }

  summary.completedAt = new Date().toISOString();
  if (caughtFailure) {
    summary.failure = {
      checkpoint: caughtFailure.checkpoint,
      code: caughtFailure.code,
      ...(caughtFailure.details ? { details: caughtFailure.details } : {}),
    };
  }
  assertSanitizedEvidence(summary);
  const summaryPath = path.join(outputDirectory, "summary.json");
  await writeFile(summaryPath, `${JSON.stringify(summary, null, 2)}\n`, { mode: 0o600 });

  if (caughtFailure) throw caughtFailure;
  return { outputDirectory, summaryPath, screenshots: summary.screenshots };
}

async function main() {
  let prepared;
  try {
    const options = parsePosthogLiveArguments(process.argv.slice(2));
    prepared = await preparePosthogLiveSmoke({
      baseUrl: options.baseUrl,
      loadBrowser: () => import("@playwright/test"),
    });
  } catch (error) {
    process.stderr.write(`${preflightFailureMessage(error)}\n`);
    process.exitCode = error instanceof PosthogLivePreflightError ? 2 : 1;
    return;
  }

  try {
    const result = await runSmoke({ config: prepared.config, chromium: prepared.browserModule.chromium });
    process.stdout.write(`PostHog live smoke passed. Sanitized evidence: ${result.outputDirectory}\n`);
  } catch (error) {
    const failure = error instanceof SmokeFailure ? error : new SmokeFailure("unexpected", "unexpected_error");
    process.stderr.write(`PostHog live smoke failed at ${failure.checkpoint} (${failure.code}).\n`);
    process.exitCode = 1;
  }
}

await main();