PaperClipAI/tests/runner-e2e/runner.spec.ts

import { randomBytes } from "node:crypto";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { expect, test, type Page } from "@playwright/test";
import { RunnerApi, pollUntil } from "./api.js";
import { buildRuntimeUsage, summarizeExecutionBilling } from "./billing.js";
import { runnerExecutionById } from "./catalog.js";
import { classifyFailure } from "./failure-classifier.js";
import { setupLiveFixtures, type LiveFixtureValues } from "./live-fixtures.js";
import { evaluateMatchers, type MatcherResult } from "./matchers.js";
import {
  isNonExecutingReviewFenceRun,
  numberedPlanStepCount,
} from "./run-observations.js";
import {
  assertSecretFree,
  findSecretLeakInJsonValues,
  normalizedSecrets,
  sanitizeJson,
} from "./redaction.js";
import {
  CREDENTIAL_NAMES,
  type CredentialName,
  type FailureClass,
  type RunnerE2EResult,
} from "./types.js";

interface IssueRecord {
  id: string;
  identifier?: string | null;
  companyId: string;
  title: string;
  status: string;
  workMode?: string;
  assigneeAgentId?: string | null;
  executionRunId?: string | null;
  checkoutRunId?: string | null;
}

interface CommentRecord {
  id: string;
  body?: string | null;
  authorType?: string | null;
  authorAgentId?: string | null;
  createdByRunId?: string | null;
  createdAt?: string;
}

interface RunRecord {
  id: string;
  companyId: string;
  agentId: string;
  status: string;
  runtimeMode?: string;
  continuationAttempt?: number;
  retryOfRunId?: string | null;
  runnerInstanceId?: string | null;
  contextSnapshot?: Record<string, unknown> | null;
  runnerProfileJson?: Record<string, unknown> | null;
  usageJson?: Record<string, unknown> | null;
  resultJson?: Record<string, unknown> | null;
  error?: string | null;
  errorCode?: string | null;
  startedAt?: string | null;
  finishedAt?: string | null;
}

interface EnvironmentLeaseRecord {
  id: string;
  issueId?: string | null;
  heartbeatRunId?: string | null;
  provider?: string | null;
  acquiredAt?: string | null;
  releasedAt?: string | null;
  updatedAt?: string | null;
  metadata?: Record<string, unknown> | null;
}

interface InteractionRecord {
  id: string;
  status: string;
  kind?: string;
  payload?: {
    acceptLabel?: string;
    rejectLabel?: string;
    target?: {
      type?: string;
      key?: string;
      revisionId?: string;
      revisionNumber?: number;
    };
    questions?: Array<{
      id: string;
      prompt: string;
      selectionMode: "single" | "multi";
      required?: boolean;
      options: Array<{ id: string; label: string }>;
    }>;
  };
}
interface IssueDocumentRecord {
  id: string;
  key: string;
  body?: string | null;
  latestRevisionId?: string | null;
  latestRevisionNumber?: number;
}
interface RunEventRecord {
  eventType?: string;
  payload?: Record<string, unknown> | null;
  sourceInstanceId?: string | null;
}
const TERMINAL_RUN_STATUSES = new Set([
  "succeeded",
  "interrupted",
  "failed",
  "cancelled",
  "timed_out",
]);
const DEFINITIVE_FAILURE_RUN_STATUSES = new Set([
  "failed",
  "cancelled",
  "timed_out",
]);

function definitiveRunFailure(runs: readonly RunRecord[]) {
  const failed = runs.find((run) =>
    DEFINITIVE_FAILURE_RUN_STATUSES.has(run.status),
  );
  if (!failed) return undefined;
  return `heartbeat run ${failed.id} ended ${failed.status}${failed.errorCode ? ` (${failed.errorCode})` : ""}${failed.error ? `: ${failed.error}` : ""}`;
}

const executionIds = (() => {
  const encoded = process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_IDS;
  if (encoded) {
    const parsed = JSON.parse(encoded) as unknown;
    if (
      !Array.isArray(parsed) ||
      parsed.length === 0 ||
      parsed.some((value) => typeof value !== "string")
    ) {
      throw new Error(
        "PAPERCLIP_RUNNER_E2E_EXECUTION_IDS must be a non-empty JSON string array",
      );
    }
    return parsed;
  }
  const single = process.env.PAPERCLIP_RUNNER_E2E_EXECUTION_ID;
  if (!single)
    throw new Error("PAPERCLIP_RUNNER_E2E_EXECUTION_IDS is required");
  return [single];
})();
const executions = executionIds.map(runnerExecutionById);
const attempt = Number(process.env.PAPERCLIP_RUNNER_E2E_ATTEMPT ?? "1");
const privateRoot = process.env.PAPERCLIP_RUNNER_E2E_PRIVATE_DIR;
const workspacePath = process.env.PAPERCLIP_RUNNER_E2E_WORKSPACE;
if (!privateRoot || !workspacePath)
  throw new Error("Runner E2E private/workspace paths are required");

function record(value: unknown): Record<string, unknown> {
  return value && typeof value === "object" && !Array.isArray(value)
    ? (value as Record<string, unknown>)
    : {};
}

function credentialValues(): Partial<Record<CredentialName, string>> {
  return Object.fromEntries(
    CREDENTIAL_NAMES.flatMap((name) => {
      const value = process.env[name]?.trim();
      return value ? [[name, value]] : [];
    }),
  );
}

async function writeSanitizedJson(
  directory: string,
  name: string,
  value: unknown,
  secrets: readonly string[],
) {
  const raw = `${JSON.stringify(value)}\n`;
  // Public API payloads may contain harmless provider-shaped fixtures or
  // opaque generated tokens. Reject exact campaign credentials before writing,
  // then sanitize all known shapes and assert the published JSON is clean.
  assertSecretFree(raw, secrets, name, { includeShapes: false });
  const safe = `${JSON.stringify(sanitizeJson(value, secrets), null, 2)}\n`;
  const leak = findSecretLeakInJsonValues(JSON.parse(safe), secrets);
  if (leak) throw new Error(`Secret leak in ${name}: ${leak}`);
  await mkdir(directory, { recursive: true });
  await writeFile(path.join(directory, name), safe, "utf8");
}

async function createTaskThroughUi(input: {
  page: Page;
  issuePrefix: string;
  agentName: string;
  title: string;
  prompt: string;
  workMode: "standard" | "planning" | "ask";
}) {
  const issuesUrl = `/${encodeURIComponent(input.issuePrefix)}/issues`;
  const newTask = input.page.getByRole("button", { name: "New Task" }).first();
  let bootstrapError: unknown;
  for (let bootstrapAttempt = 1; bootstrapAttempt <= 3; bootstrapAttempt += 1) {
    try {
      await input.page.goto(issuesUrl, {
        waitUntil: "domcontentloaded",
        timeout: 30_000,
      });
      await newTask.waitFor({ state: "visible", timeout: 20_000 });
      bootstrapError = undefined;
      break;
    } catch (error) {
      bootstrapError = error;
      if (bootstrapAttempt < 3) await input.page.waitForTimeout(1_000);
    }
  }
  if (bootstrapError) {
    throw new Error(
      `Browser bootstrap failed before task creation: ${bootstrapError instanceof Error ? bootstrapError.message : String(bootstrapError)}`,
      { cause: bootstrapError },
    );
  }
  await newTask.click();
  await input.page.getByPlaceholder("Task title").fill(input.title);
  await input.page
    .getByRole("dialog")
    .getByRole("textbox", { name: "editable markdown", exact: true })
    .fill(input.prompt);
  if (input.workMode !== "standard") {
    await input.page
      .getByRole("dialog")
      .locator(`[data-issue-work-mode-chip="standard"]`)
      .click();
    await input.page
      .locator(`[data-issue-work-mode="${input.workMode}"]`)
      .click();
  }
  await input.page
    .getByRole("button", { name: "Assignee", exact: true })
    .click();
  await input.page
    .getByPlaceholder("Search assignees...")
    .fill(input.agentName);
  await input.page.getByText(input.agentName, { exact: true }).last().click();
  await input.page
    .getByRole("button", { name: "Create Task", exact: true })
    .click();
}

function matchingRuns(runs: RunRecord[], issue: IssueRecord) {
  const explicit = new Set(
    [issue.executionRunId, issue.checkoutRunId].filter(Boolean),
  );
  return runs.filter((run) => {
    if (isNonExecutingReviewFenceRun(run)) return false;
    const context = record(run.contextSnapshot);
    return (
      context.issueId === issue.id ||
      context.taskId === issue.id ||
      explicit.has(run.id)
    );
  });
}

function isPendingPlanConfirmation(interaction: InteractionRecord) {
  return (
    interaction.kind === "request_confirmation" &&
    interaction.status === "pending" &&
    interaction.payload?.target?.type === "issue_document" &&
    interaction.payload.target.key === "plan" &&
    typeof interaction.payload.target.revisionId === "string"
  );
}

function isPendingQuestion(interaction: InteractionRecord) {
  return (
    interaction.kind === "ask_user_questions" &&
    interaction.status === "pending" &&
    Boolean(interaction.payload?.questions?.length)
  );
}

function normalizePlanMarkdown(body: string | null | undefined) {
  // Provider plan renderers may defensively escape underscores in plain-text
  // markers. The rendered document and semantic marker are identical.
  return (body ?? "").replaceAll("\\_", "_");
}

function renderedMarkerPattern(marker: string) {
  return new RegExp(
    marker
      .split("_")
      .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
      .join("\\\\?_"),
  );
}

async function expectPlanStageMarkerVisible(page: Page, marker: string) {
  const pattern = renderedMarkerPattern(marker);
  // Depending on the interaction presentation, the task thread either
  // expands the canonical Plan body inline or renders a compact `plan · vN`
  // confirmation card while the agent's adjacent visible message describes
  // the published revision. Backend assertions separately verify the exact
  // canonical document body, revision ID, step count, and pending interaction.
  await expect(
    page
      .getByTestId("task-chat-plan-preview")
      .filter({ hasText: pattern })
      .or(
        page.getByTestId("task-chat-agent-bubble").filter({ hasText: pattern }),
      )
      .last(),
  ).toBeVisible({ timeout: 30_000 });
}

for (const execution of executions) {
  const privateDir = path.join(privateRoot, "cases", execution.task.id);
  const resultPath = path.join(privateDir, "result.json");
  const snapshotsDir = path.join(privateDir, "snapshots");
  const deadlineMs = execution.task.attemptTimeoutMs[execution.environment.id];

  test(`${execution.id} completes through the browser and public APIs`, async ({
    page,
    request,
  }, testInfo) => {
    test.setTimeout(deadlineMs + 90_000);
    const startedAtMs = Date.now();
    const startedAt = new Date(startedAtMs).toISOString();
    const nonce = `${randomBytes(6).toString("hex")}-${attempt}`;
    const marker = execution.task.buildVisibleMarker(nonce);
    const title = execution.task.buildTitle(nonce);
    const prompt = execution.task.buildPrompt(nonce);
    const credentials = credentialValues();
    const secrets = normalizedSecrets(Object.values(credentials));
    const api = new RunnerApi(request);
    const consoleDiagnostics: Array<Record<string, unknown>> = [];
    const networkDiagnostics: Array<Record<string, unknown>> = [];
    let fixtures: LiveFixtureValues | undefined;
    let issue: IssueRecord | undefined;
    let selectedRuns: RunRecord[] = [];
    let runtimeLeases: EnvironmentLeaseRecord[] = [];
    let matcherResults: MatcherResult[] = [];
    const screenshots: NonNullable<RunnerE2EResult["screenshots"]> = [];
    let primaryError: unknown;
    let failureClassOverride: FailureClass | undefined;
    let cleanup: RunnerE2EResult["cleanup"] = "not_started";

    const captureScreenshot = async (
      id: string,
      label: string,
      file: string,
    ) => {
      const screenshotPath = path.join(privateDir, file);
      await page.screenshot({ path: screenshotPath, fullPage: true });
      await testInfo.attach(id, {
        path: screenshotPath,
        contentType: "image/png",
      });
      screenshots.push({ id, label, file });
    };

    const captureRuntimeLeases = async () => {
      if (!fixtures || execution.environment.id !== "daytona") return;
      const listed = await api.get<EnvironmentLeaseRecord[]>(
        `/api/environments/${fixtures.environment.id}/leases`,
      );
      const selectedRunIds = new Set(selectedRuns.map((run) => run.id));
      const relevant = listed.filter(
        (lease) =>
          lease.provider === "daytona" &&
          (selectedRunIds.size === 0 ||
            (lease.heartbeatRunId &&
              selectedRunIds.has(lease.heartbeatRunId)) ||
            (issue && lease.issueId === issue.id)),
      );
      runtimeLeases = relevant.length > 0 ? relevant : listed;
    };

    const captureFailureApiState = async () => {
      if (!fixtures || !issue) return;
      const capture = async <T>(operation: () => Promise<T>) =>
        operation().catch((error) => ({
          evidenceCaptureError:
            error instanceof Error ? error.message : String(error),
        }));
      const [currentIssue, listedRuns, comments, interactions] =
        await Promise.all([
          capture(() => api.get<IssueRecord>(`/api/issues/${issue!.id}`)),
          capture(() =>
            api.get<RunRecord[]>(
              `/api/companies/${fixtures!.company.id}/heartbeat-runs?agentId=${fixtures!.agent.id}&limit=20`,
            ),
          ),
          capture(() =>
            api.get<CommentRecord[]>(
              `/api/issues/${issue!.id}/comments?order=asc`,
            ),
          ),
          capture(() =>
            api.get<InteractionRecord[]>(
              `/api/issues/${issue!.id}/interactions`,
            ),
          ),
        ]);
      const taskRuns = Array.isArray(listedRuns)
        ? matchingRuns(listedRuns, "id" in currentIssue ? currentIssue : issue)
        : [];
      const detailedRuns = await Promise.all(
        taskRuns.map((candidate) =>
          capture(() =>
            api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`),
          ).then((value) => ("id" in value ? value : candidate)),
        ),
      );
      if (detailedRuns.length > 0) selectedRuns = detailedRuns;
      const runEvidence = await Promise.all(
        detailedRuns.map(async (candidate) => ({
          runId: candidate.id,
          log: await capture(() =>
            api.get<unknown>(
              `/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
            ),
          ),
          events: await capture(() =>
            api.get<RunEventRecord[]>(
              `/api/heartbeat-runs/${candidate.id}/events?limit=1000`,
            ),
          ),
        })),
      );
      await writeSanitizedJson(
        snapshotsDir,
        "api-state.json",
        {
          capturePhase: "failure",
          issue: currentIssue,
          runs: detailedRuns,
          comments,
          interactions,
          runEvidence,
        },
        secrets,
      );
    };

    page.on("console", (message) => {
      if (["error", "warning"].includes(message.type())) {
        consoleDiagnostics.push({
          type: message.type(),
          text: message.text(),
          location: message.location(),
        });
      }
    });
    page.on("requestfailed", (requestEvent) => {
      networkDiagnostics.push({
        method: requestEvent.method(),
        url: requestEvent.url(),
        failure: requestEvent.failure()?.errorText ?? null,
      });
    });
    page.on("response", (response) => {
      if (response.status() >= 400) {
        networkDiagnostics.push({
          method: response.request().method(),
          url: response.url(),
          status: response.status(),
          statusText: response.statusText(),
        });
      }
    });

    try {
      const initialExperimental = await api.get<{
        enableNativeRunner: boolean;
      }>("/api/instance/settings/experimental");
      expect(initialExperimental.enableNativeRunner).toBe(false);
      await api.patch("/api/instance/settings/experimental", {
        enableNativeRunner: true,
        ...(execution.profile.generation === "native" &&
        execution.environment.id === "daytona"
          ? { enableRunnerPreviewIngress: true }
          : {}),
      });

      fixtures = await setupLiveFixtures({
        api,
        execution,
        executionNonce: nonce,
        workspacePath,
        credentials,
        daytonaImage: process.env.PAPERCLIP_E2E_DAYTONA_IMAGE,
      });

      await writeSanitizedJson(
        snapshotsDir,
        "fixtures.json",
        {
          executionId: execution.id,
          companyId: fixtures.company.id,
          environmentId: fixtures.environment.id,
          agentId: fixtures.agent.id,
          secretIds: Object.fromEntries(
            Object.entries(fixtures.secretRefs).map(([name, ref]) => [
              name,
              ref?.secretId,
            ]),
          ),
          persistedAgent: await api.get<unknown>(
            `/api/agents/${fixtures.agent.id}`,
          ),
          persistedEnvironment: await api.get<unknown>(
            `/api/environments/${fixtures.environment.id}`,
          ),
        },
        secrets,
      );

      const issuePrefix = fixtures.company.issuePrefix;
      if (!issuePrefix)
        throw new Error(
          "Created fixture company did not return an issue prefix",
        );
      await createTaskThroughUi({
        page,
        issuePrefix,
        agentName: fixtures.agent.name,
        title,
        prompt,
        workMode: execution.task.workMode,
      });

      const deadlineAt = startedAtMs + deadlineMs;
      issue = await pollUntil({
        label: `UI-created issue ${title}`,
        deadlineAt,
        load: async () => {
          const issues = await api.get<IssueRecord[]>(
            `/api/companies/${fixtures!.company.id}/issues?q=${encodeURIComponent(title)}&limit=50`,
          );
          return issues.find((candidate) => candidate.title === title);
        },
        accept: (candidate): candidate is IssueRecord => Boolean(candidate),
      });
      if (!issue) throw new Error(`Issue ${title} disappeared after creation`);
      if (
        issue.companyId !== fixtures.company.id ||
        issue.assigneeAgentId !== fixtures.agent.id
      ) {
        throw new Error(
          "UI-created issue does not belong to the fixture company and agent",
        );
      }
      if (issue.workMode !== execution.task.workMode) {
        throw new Error(
          `UI-created issue work mode was ${String(issue.workMode)}; expected ${execution.task.workMode}`,
        );
      }

      await page.goto(
        `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
      );

      const loadTaskState = async () => {
        const [currentIssue, runs, comments, interactions] = await Promise.all([
          api.get<IssueRecord>(`/api/issues/${issue!.id}`),
          api.get<RunRecord[]>(
            `/api/companies/${fixtures!.company.id}/heartbeat-runs?agentId=${fixtures!.agent.id}&limit=20`,
          ),
          api.get<CommentRecord[]>(
            `/api/issues/${issue!.id}/comments?order=asc`,
          ),
          api.get<InteractionRecord[]>(`/api/issues/${issue!.id}/interactions`),
        ]);
        return {
          currentIssue,
          taskRuns: matchingRuns(runs, currentIssue),
          comments,
          interactions,
        };
      };

      let planLifecycleEvidence: Record<string, unknown> | null = null;
      let questionLifecycleEvidence: Record<string, unknown> | null = null;
      if (execution.task.flow === "plan_revision_acceptance") {
        const planMarkers = execution.task.buildPlanMarkers?.(nonce);
        const revisionRequest = execution.task.buildRevisionRequest?.(nonce);
        if (!planMarkers || !revisionRequest) {
          throw new Error(
            `Plan fixture ${execution.task.id} is missing lifecycle factories`,
          );
        }
        const draftState = await pollUntil({
          label: `initial plan confirmation for issue ${issue.id}`,
          deadlineAt,
          load: loadTaskState,
          accept: ({ taskRuns, interactions }) =>
            taskRuns.length >= 1 &&
            taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
            interactions.some(isPendingPlanConfirmation),
          reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
        });
        const draftInteraction = draftState.interactions.find(
          isPendingPlanConfirmation,
        )!;
        const draftPlan = await api.get<IssueDocumentRecord>(
          `/api/issues/${issue.id}/documents/plan`,
        );
        if (
          !normalizePlanMarkdown(draftPlan.body).includes(planMarkers.draft)
        ) {
          throw new Error(
            `Initial Plan document did not contain ${planMarkers.draft}`,
          );
        }
        if (numberedPlanStepCount(draftPlan.body) !== 2) {
          throw new Error(
            "Initial Plan must contain exactly two numbered steps",
          );
        }
        if (
          draftInteraction.payload?.target?.revisionId !==
          draftPlan.latestRevisionId
        ) {
          throw new Error(
            "Initial plan confirmation did not target the latest Plan revision",
          );
        }
        await page.goto(
          `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
          { waitUntil: "domcontentloaded" },
        );
        await expectPlanStageMarkerVisible(page, planMarkers.draft);
        await captureScreenshot(
          "plan-draft",
          "Initial plan awaiting revision",
          "plan-draft.png",
        );
        await page
          .getByRole("button", {
            name: draftInteraction.payload?.rejectLabel ?? "Reject",
            exact: true,
          })
          .last()
          .click();
        const revisionComposer = page
          .getByTestId("plan-revision-composer")
          .last();
        await expect(revisionComposer).toBeVisible({ timeout: 10_000 });
        await revisionComposer
          .locator('[contenteditable="true"], textarea')
          .first()
          .fill(revisionRequest);
        await page
          .getByRole("button", {
            name: draftInteraction.payload?.rejectLabel ?? "Reject",
            exact: true,
          })
          .last()
          .click();

        const revisedState = await pollUntil({
          label: `revised plan confirmation for issue ${issue.id}`,
          deadlineAt,
          load: loadTaskState,
          accept: ({ taskRuns, interactions }) =>
            taskRuns.length >= 2 &&
            taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
            interactions.some(
              (interaction) =>
                isPendingPlanConfirmation(interaction) &&
                interaction.id !== draftInteraction.id,
            ),
          reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
        });
        const revisedInteraction = revisedState.interactions.find(
          (interaction) =>
            isPendingPlanConfirmation(interaction) &&
            interaction.id !== draftInteraction.id,
        )!;
        const revisedPlan = await api.get<IssueDocumentRecord>(
          `/api/issues/${issue.id}/documents/plan`,
        );
        const normalizedRevisedPlan = normalizePlanMarkdown(revisedPlan.body);
        if (
          !normalizedRevisedPlan.includes(planMarkers.revised) ||
          normalizedRevisedPlan.includes(planMarkers.draft)
        ) {
          throw new Error(
            `Revised Plan must replace ${planMarkers.draft} with ${planMarkers.revised}`,
          );
        }
        if (numberedPlanStepCount(revisedPlan.body) !== 3) {
          throw new Error(
            "Revised Plan must contain exactly three numbered steps",
          );
        }
        if (
          revisedInteraction.payload?.target?.revisionId !==
            revisedPlan.latestRevisionId ||
          revisedPlan.latestRevisionId === draftPlan.latestRevisionId
        ) {
          throw new Error(
            "Revised confirmation did not target a new latest Plan revision",
          );
        }
        await page.goto(
          `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
          { waitUntil: "domcontentloaded" },
        );
        await expectPlanStageMarkerVisible(page, planMarkers.revised);
        await captureScreenshot(
          "plan-revised",
          "Revised plan awaiting acceptance",
          "plan-revised.png",
        );
        await page
          .getByRole("button", {
            name: revisedInteraction.payload?.acceptLabel ?? "Approve",
            exact: true,
          })
          .last()
          .click();
        planLifecycleEvidence = {
          draftInteraction,
          draftPlan,
          revisedInteraction,
          revisedPlan,
          revisionRequest,
        };
      } else if (execution.task.flow === "question_resume_completion") {
        const expectedAnswer = execution.task.buildQuestionAnswer?.(nonce);
        if (!expectedAnswer) {
          throw new Error(
            `Question fixture ${execution.task.id} is missing its answer factory`,
          );
        }
        const pendingState = await pollUntil({
          label: `pending user question for issue ${issue.id}`,
          deadlineAt,
          load: loadTaskState,
          accept: ({ taskRuns, interactions }) =>
            taskRuns.length >= 1 &&
            taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
            interactions.some(isPendingQuestion),
          reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
        });
        const questionInteraction =
          pendingState.interactions.find(isPendingQuestion)!;
        const labels =
          questionInteraction.payload?.questions?.flatMap((question) =>
            question.options.map((option) => option.label),
          ) ?? [];
        if (!labels.includes(expectedAnswer.optionLabel)) {
          throw new Error(
            `Question interaction omitted ${expectedAnswer.optionLabel}`,
          );
        }
        await page.goto(
          `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
          { waitUntil: "domcontentloaded" },
        );
        await expect(
          page
            .getByRole("button", {
              name: expectedAnswer.optionLabel,
              exact: true,
            })
            .last(),
        ).toBeVisible({ timeout: 30_000 });
        await captureScreenshot(
          "question-pending",
          "Structured question awaiting an answer",
          "question-pending.png",
        );
        await page
          .getByRole("button", {
            name: expectedAnswer.optionLabel,
            exact: true,
          })
          .last()
          .click();
        await page
          .getByRole("button", { name: "Submit answers", exact: true })
          .last()
          .click();
        questionLifecycleEvidence = {
          interaction: questionInteraction,
          answer: expectedAnswer.optionLabel,
          expectedMarker: expectedAnswer.expectedMarker,
        };
      } else if (execution.task.flow === "plan_approval_completion") {
        const planMarkers = execution.task.buildPlanMarkers?.(nonce);
        if (!planMarkers) {
          throw new Error(
            `Plan fixture ${execution.task.id} is missing its marker factory`,
          );
        }
        const pendingState = await pollUntil({
          label: `pending plan approval for issue ${issue.id}`,
          deadlineAt,
          load: loadTaskState,
          accept: ({ taskRuns, interactions }) =>
            taskRuns.length >= 1 &&
            taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)) &&
            interactions.some(isPendingPlanConfirmation),
          reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
        });
        const interaction = pendingState.interactions.find(
          isPendingPlanConfirmation,
        )!;
        const plan = await api.get<IssueDocumentRecord>(
          `/api/issues/${issue.id}/documents/plan`,
        );
        if (!normalizePlanMarkdown(plan.body).includes(planMarkers.draft)) {
          throw new Error(`Plan document did not contain ${planMarkers.draft}`);
        }
        if (numberedPlanStepCount(plan.body) !== 2) {
          throw new Error("Plan must contain exactly two numbered steps");
        }
        if (interaction.payload?.target?.revisionId !== plan.latestRevisionId) {
          throw new Error(
            "Plan confirmation did not target the latest Plan revision",
          );
        }
        await page.goto(
          `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
          { waitUntil: "domcontentloaded" },
        );
        await expectPlanStageMarkerVisible(page, planMarkers.draft);
        await captureScreenshot(
          "plan-pending",
          "Plan awaiting approval",
          "plan-pending.png",
        );
        await page
          .getByRole("button", {
            name: interaction.payload?.acceptLabel ?? "Approve",
            exact: true,
          })
          .last()
          .click();
        planLifecycleEvidence = { interaction, plan };
      }

      const terminal = await pollUntil({
        label: `issue ${issue.id} and heartbeat run terminal state`,
        deadlineAt,
        load: loadTaskState,
        accept: ({ currentIssue, taskRuns }) =>
          currentIssue.status === execution.task.expectedTerminalState.issue &&
          taskRuns.length >= execution.task.expectedRunCount &&
          taskRuns.every((run) => TERMINAL_RUN_STATUSES.has(run.status)),
        reject: ({ taskRuns }) => definitiveRunFailure(taskRuns),
      });

      issue = terminal.currentIssue;
      selectedRuns = terminal.taskRuns;
      if (selectedRuns.length !== execution.task.expectedRunCount) {
        const runLogs = await Promise.all(
          selectedRuns.map(async (candidate) => ({
            runId: candidate.id,
            log: await api
              .get<unknown>(
                `/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
              )
              .catch((error) => ({
                evidenceCaptureError:
                  error instanceof Error ? error.message : String(error),
              })),
          })),
        );
        await writeSanitizedJson(
          snapshotsDir,
          "api-state.json",
          { issue, runs: selectedRuns, runLogs, ...terminal },
          secrets,
        );
        throw new Error(
          `Expected exactly ${execution.task.expectedRunCount} task heartbeat run(s); observed ${selectedRuns.length}`,
        );
      }
      // The company run-list endpoint intentionally returns only a compact,
      // allowlisted context summary. Hydrate each selected run through the
      // public detail endpoint before asserting environment/lease metadata.
      selectedRuns = await Promise.all(
        selectedRuns.map((candidate) =>
          api.get<RunRecord>(`/api/heartbeat-runs/${candidate.id}`),
        ),
      );
      const run =
        selectedRuns.find(
          (candidate) => candidate.id === issue!.executionRunId,
        ) ?? selectedRuns[0];
      const [persistedAgentValue, persistedEnvironmentValue, runLogs] =
        await Promise.all([
          api.get<unknown>(`/api/agents/${fixtures.agent.id}`),
          api.get<unknown>(`/api/environments/${fixtures.environment.id}`),
          Promise.all(
            selectedRuns.map(async (candidate) => ({
              runId: candidate.id,
              log: await api
                .get<unknown>(
                  `/api/heartbeat-runs/${candidate.id}/log?limitBytes=1048576`,
                )
                .catch((error) => ({
                  evidenceCaptureError:
                    error instanceof Error ? error.message : String(error),
                })),
            })),
          ),
        ]);
      const runLog =
        runLogs.find((candidate) => candidate.runId === run.id)?.log ?? {};
      const persistedAgent = record(persistedAgentValue);
      const persistedEnvironment = record(persistedEnvironmentValue);
      const agentComments = terminal.comments.filter(
        (comment) =>
          comment.authorAgentId === fixtures!.agent.id ||
          selectedRuns.some(
            (candidate) => comment.createdByRunId === candidate.id,
          ),
      );
      // Message matchers intentionally use persisted agent comments only.
      // A semantic finish summary can differ from the actual user-facing text;
      // accepting it here would let backend metadata mask a truncated UI
      // response. The browser assertion below remains the visible source of
      // truth after the comment projection has settled.
      const message = agentComments
        .map((comment) => comment.body ?? "")
        .join("\n");
      const pendingInteractions = terminal.interactions.filter(
        (interaction) => interaction.status === "pending",
      );
      const invariantFailures: string[] = [];
      if (pendingInteractions.length > 0)
        invariantFailures.push(
          `expected no unresolved interaction; observed ${pendingInteractions.length}`,
        );
      for (const candidate of selectedRuns) {
        if (
          (candidate.continuationAttempt ?? 0) !== 0 ||
          candidate.retryOfRunId
        )
          invariantFailures.push(
            `expected run ${candidate.id} without a recovery continuation`,
          );
      }

      const context = record(run.contextSnapshot);
      const environmentContext = record(context.paperclipEnvironment);
      const workspaceContext = record(context.paperclipWorkspace);
      const environmentDriver =
        environmentContext.driver ?? persistedEnvironment.driver;
      const observedEnvironment =
        environmentDriver === "sandbox" ? "daytona" : environmentDriver;
      const observedRuntimeMode =
        run.runtimeMode ??
        (persistedAgent.adapterType === "paperclip_runner"
          ? "native"
          : "legacy");
      matcherResults = await evaluateMatchers(
        execution.task.buildMatchers(nonce, execution),
        {
          message,
          issueStatus: issue.status,
          runStatus: selectedRuns.every(
            (candidate) =>
              candidate.status === execution.task.expectedTerminalState.run,
          )
            ? execution.task.expectedTerminalState.run
            : selectedRuns.map((candidate) => candidate.status).join(","),
          runtimeMode: observedRuntimeMode,
          environment:
            typeof observedEnvironment === "string"
              ? observedEnvironment
              : undefined,
          json: {
            issue,
            run,
            comments: terminal.comments,
            interactions: terminal.interactions,
          },
        },
      );
      const failedMatchers = matcherResults.filter((result) => !result.passed);
      const observedEnvironmentId =
        environmentContext.id ??
        (execution.environment.id === "local"
          ? persistedAgent.defaultEnvironmentId
          : undefined);
      if (observedEnvironmentId !== fixtures.environment.id) {
        invariantFailures.push(
          `Expected environment ${fixtures.environment.id}; observed ${String(observedEnvironmentId)}`,
        );
      }
      if (
        execution.environment.id === "daytona" &&
        typeof environmentContext.leaseId !== "string"
      ) {
        invariantFailures.push(
          "expected a Daytona sandbox lease on the run context",
        );
      }
      if (execution.environment.id === "local") {
        const runLogContent = String(record(runLog).content ?? "");
        // The log endpoint returns NDJSON, so quotes inside each `chunk` are
        // escaped. Accept both that wire representation and a decoded chunk.
        const fallbackWorkspace =
          /Using fallback workspace \\\"([^"\\]+)\\\"/.exec(
            runLogContent,
          )?.[1] ??
          /Using fallback workspace "([^"]+)"/.exec(runLogContent)?.[1];
        const cwd = String(workspaceContext.cwd ?? fallbackWorkspace ?? "");
        const isolatedRoot = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT ?? "";
        if (!isolatedRoot || !cwd.startsWith(`${isolatedRoot}/`)) {
          invariantFailures.push(
            `local run workspace escaped the isolated root: ${cwd}`,
          );
        }
      }
      let runEvents: RunEventRecord[] = [];
      let runEventsCaptureError: string | null = null;
      if (execution.profile.generation === "native") {
        try {
          runEvents = await api.get<RunEventRecord[]>(
            `/api/heartbeat-runs/${run.id}/events?limit=1000`,
          );
        } catch (error) {
          runEventsCaptureError =
            error instanceof Error ? error.message : String(error);
          invariantFailures.push(
            `native run events query failed: ${runEventsCaptureError}`,
          );
        }
        const runnerInstanceObserved =
          Boolean(run.runnerInstanceId) ||
          runEvents.some(
            (event) =>
              typeof event.sourceInstanceId === "string" &&
              event.sourceInstanceId.length > 0 &&
              !event.sourceInstanceId.endsWith(":control"),
          );
        if (!runnerInstanceObserved) {
          invariantFailures.push(
            "expected native run events from a runner instance",
          );
        }
        const spanPayloads = runEvents
          .filter((event) => event.eventType === "run.performance.span")
          .map((event) => record(event.payload));
        const selectedTransport = spanPayloads.find(
          (payload) => payload.span === "runner.transport.selected",
        );
        const expectedTransport =
          execution.environment.id === "daytona"
            ? "provider_ingress"
            : "local_loopback";
        if (selectedTransport?.mode !== expectedTransport) {
          invariantFailures.push(
            `Expected native runner transport ${expectedTransport}; observed ${String(selectedTransport?.mode)}`,
          );
        }
        if (execution.environment.id === "daytona") {
          const authenticated = spanPayloads.some(
            (payload) =>
              payload.span === "runner.prp.authenticate" &&
              payload.outcome === "ok",
          );
          if (!authenticated) {
            invariantFailures.push(
              "expected authenticated native Daytona runner preview ingress",
            );
          }
        }
      }

      await writeSanitizedJson(
        snapshotsDir,
        "api-state.json",
        {
          issue,
          run,
          comments: terminal.comments,
          interactions: terminal.interactions,
          planLifecycleEvidence,
          questionLifecycleEvidence,
          matcherResults,
          invariantFailures,
          runEvents,
          runEventsCaptureError,
          runLogs,
        },
        secrets,
      );

      // The backend polling above can observe a terminal transition before a
      // websocket invalidation reaches the already-open task page. Reload the
      // canonical task route so the screenshot and UI assertions prove the
      // persisted final state, not a stale client cache.
      await page.goto(
        `/${encodeURIComponent(issuePrefix)}/issues/${encodeURIComponent(issue.identifier ?? issue.id)}`,
        { waitUntil: "domcontentloaded" },
      );
      await expect(
        page
          .getByTestId("task-chat-thread")
          .getByTestId("task-chat-agent-bubble")
          .filter({ hasText: renderedMarkerPattern(marker) })
          .last(),
      ).toBeVisible({ timeout: 30_000 });
      await expect(
        page.getByTestId("issue-detail-header").getByRole("button", {
          name: "Change status (current: Done)",
          exact: true,
        }),
      ).toBeVisible({ timeout: 30_000 });
      await captureScreenshot(
        "final-state",
        "Final visible task state",
        "final-state.png",
      );

      if (failedMatchers.length > 0) {
        throw new Error(
          `Matcher failure: ${failedMatchers.map((result) => result.detail).join("; ")}; run error: ${run.errorCode ?? "none"} ${run.error ?? "none"}`,
        );
      }
      if (invariantFailures.length > 0) {
        throw new Error(
          `Runtime invariant failure: ${invariantFailures.join("; ")}`,
        );
      }
    } catch (error) {
      primaryError = error;
      try {
        await captureFailureApiState();
      } catch (captureError) {
        const failureClass = classifyFailure(captureError);
        if (failureClass === "secret_leak") {
          failureClassOverride = "secret_leak";
          primaryError = new AggregateError(
            [primaryError, captureError],
            `Failure evidence contained unsafe data: ${captureError instanceof Error ? captureError.message : String(captureError)}`,
          );
        } else {
          networkDiagnostics.push({
            evidenceCaptureError:
              captureError instanceof Error
                ? captureError.message
                : String(captureError),
          });
        }
      }
      if (!page.isClosed()) {
        const failureScreenshot = path.join(privateDir, "failure.png");
        await page
          .screenshot({ path: failureScreenshot, fullPage: true })
          .catch(() => undefined);
        await testInfo
          .attach("failure", {
            path: failureScreenshot,
            contentType: "image/png",
          })
          .catch(() => undefined);
      }
    } finally {
      try {
        await writeSanitizedJson(
          snapshotsDir,
          "browser-diagnostics.json",
          {
            console: consoleDiagnostics,
            network: networkDiagnostics,
          },
          secrets,
        );
      } catch (error) {
        primaryError = new AggregateError(
          [primaryError, error].filter(Boolean),
          `Browser diagnostics contained unsafe data: ${error instanceof Error ? error.message : String(error)}`,
        );
        failureClassOverride = "secret_leak";
      }
      if (fixtures) {
        await captureRuntimeLeases().catch((error) => {
          networkDiagnostics.push({
            runtimeBillingCaptureError:
              error instanceof Error ? error.message : String(error),
          });
        });
        try {
          await fixtures.teardown();
          cleanup = "passed";
        } catch (error) {
          cleanup = "failed";
          const priorFailureClass = primaryError
            ? classifyFailure(primaryError)
            : undefined;
          const cleanupFailureClass = classifyFailure(error);
          failureClassOverride =
            priorFailureClass === "secret_leak"
              ? priorFailureClass
              : cleanupFailureClass === "cleanup_failure"
                ? cleanupFailureClass
                : (priorFailureClass ?? cleanupFailureClass);
          primaryError = new AggregateError(
            [primaryError, error].filter(Boolean),
            `Cleanup failed after ${primaryError ? "test failure" : "test execution"}: ${error instanceof Error ? error.message : String(error)}`,
          );
        }
        if (runtimeLeases.length > 0) {
          runtimeLeases = await Promise.all(
            runtimeLeases.map((lease) =>
              api
                .get<EnvironmentLeaseRecord>(
                  `/api/environment-leases/${lease.id}`,
                )
                .catch(() => lease),
            ),
          );
        }
      } else {
        cleanup =
          primaryError &&
          /(?:cleanup|teardown) failed/i.test(
            primaryError instanceof Error
              ? primaryError.message
              : String(primaryError),
          )
            ? "failed"
            : "passed";
      }

      const finishedAtMs = Date.now();
      const runtimeUsage = buildRuntimeUsage({
        environmentId: execution.environment.id,
        runs: selectedRuns,
        leases: runtimeLeases,
        fallbackFinishedAt: new Date(finishedAtMs),
      });
      const resultWithoutBilling: RunnerE2EResult = {
        schema: "paperclip.runner-e2e.result/v2",
        executionId: execution.id,
        suiteId: execution.suite.id,
        suiteDefinitionHash: execution.suiteDefinitionHash,
        source: {
          sha: process.env.GITHUB_SHA ?? null,
          ref: process.env.GITHUB_REF ?? null,
          workflowRunUrl:
            process.env.GITHUB_SERVER_URL &&
            process.env.GITHUB_REPOSITORY &&
            process.env.GITHUB_RUN_ID
              ? `${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
              : null,
        },
        ...(execution.profile.ranking
          ? { rankingSnapshot: execution.profile.ranking }
          : {}),
        attempt,
        status: primaryError ? "failed" : "passed",
        ...(primaryError
          ? {
              failureClass:
                failureClassOverride ?? classifyFailure(primaryError),
              error:
                primaryError instanceof Error
                  ? primaryError.message
                  : String(primaryError),
            }
          : {}),
        profileId: execution.profile.id,
        environmentId: execution.environment.id,
        caseId: execution.task.id,
        provider: execution.profile.provider,
        model: execution.profile.model,
        runtimeMode: execution.profile.expectedRuntimeMode,
        issueId: issue?.id,
        issueIdentifier: issue?.identifier ?? null,
        runIds: selectedRuns.map((run) => run.id),
        startedAt,
        finishedAt: new Date(finishedAtMs).toISOString(),
        durationMs: finishedAtMs - startedAtMs,
        usage:
          selectedRuns.length === 1
            ? (selectedRuns[0]?.usageJson ?? null)
            : {
                runs: selectedRuns.map((candidate) => ({
                  runId: candidate.id,
                  usage: candidate.usageJson ?? null,
                })),
              },
        runtimeUsage,
        matcherResults,
        screenshots,
        cleanup,
      };
      const result: RunnerE2EResult = {
        ...resultWithoutBilling,
        billing: summarizeExecutionBilling(resultWithoutBilling),
      };
      await mkdir(privateDir, { recursive: true });
      await writeFile(
        resultPath,
        `${JSON.stringify(sanitizeJson(result, secrets), null, 2)}\n`,
        "utf8",
      );
      await testInfo.attach("runner-e2e-result", {
        path: resultPath,
        contentType: "application/json",
      });
    }

    if (primaryError) throw primaryError;
  });
}