PaperClipAI/tests/runner-e2e/continuation-flow.ts

import { gradeNecessaryQuestion } from "./necessary-question.js";
import { gradeLegacyWorkspaceMigration, migrationPrimaryOutputCheckpoints, isLegacyWorkspaceMigration, legacyWorkspaceMigrationPrompt, readMigrationPoint, type MigrationPoint } from "./legacy-workspace-migration.js";
import { necessaryQuestionContract } from "./continuation-cases.js";
import { validResumeQuestionForm, validResumeReferenceForm } from "./question-resume-scoring.js";
import { answerResumeQuestionThroughUi } from "./question-resume-ui.js";
import { confirmContinuationPlan } from "./continuation-approval.js";
import { gradeLifecycleBaseline, type LifecycleCheckpoint } from "./lifecycle-baseline.js";
import { lifecycleLiveCase, lifecycleLiveContinuation, gradeLifecycleNarrative } from "./lifecycle-live-cases.js";
import { prepareLegacyContinuationSkill, prepareContinuationBudget } from "./continuation-fixtures.js";
import { captureFirstTaskAttachments } from "./first-task-attachments.js";
import { answerableRuntimeRunIds, isSingleClaudeQuestion } from "./runtime-question-readiness.js";
import { expect, type Page } from "@playwright/test";
import path from "node:path";
import { continuationAnswerCommitted, continuationInitialReady, continuationCheckpointReady } from "./continuation-readiness.js";
import { captureLoadedContinuation, waitForTaskChatRendered } from "./continuation-screenshot.js";
import { seedContinuationContext } from "./continuation-workspace.js";
import { pollUntil, type RunnerApi } from "./api.js";
import {
  chatQuestionPresentation,
  sendChatMessage,
  collectChatRunEvidence,
  type ChatRun,
} from "./chat-flow.js";
import {
  continuationScenario,
  continuationScreenshotFile,
} from "./continuation-cases.js";
import {
  gradeContinuation,
  isContinuationPlan,
  type ContinuationCheckpoint,
} from "./continuation-scoring.js";
import { createTaskThroughUi } from "./user-actions.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { MatrixExecution } from "./types.js";
type Row = Record<string, any>;

export async function runContinuationFlow(input: {
  page: Page;
  api: RunnerApi;
  fixtures: LiveFixtureValues;
  execution: MatrixExecution;
  nonce: string;
  secrets: readonly string[];
  workspacePath: string;
  deadlineAt: number;
  restart(): Promise<void>;
  observe(
    issue: any,
    runs: any[],
    checks: ReturnType<typeof gradeContinuation>,
  ): void;
  capture(id: string, label: string, file: string): Promise<void>;
  evidence(name: string, value: unknown): Promise<void>;
}) {
  const { page, api, fixtures, execution } = input;
  const migration = isLegacyWorkspaceMigration(execution);
  let migrationBefore: MigrationPoint | undefined, migrationAfter: MigrationPoint | undefined;
  const lifecycleProbe = lifecycleLiveCase(execution.task.id);
  const scenario = lifecycleProbe
    ? lifecycleLiveContinuation(execution.task.id, input.nonce)
    : continuationScenario(execution.task.id, input.nonce);
  const checkpoints: LifecycleCheckpoint[] = [];
  let budgetGuard: Awaited<ReturnType<typeof prepareContinuationBudget>> | undefined;
  let issue: Row | undefined;
  let runs: Row[] = [];
  let checks: ReturnType<typeof gradeContinuation> = [];
  const tasksPath = `/api/companies/${fixtures.company.id}/issues?limit=100`;
  async function refresh() {
    issue = await api.get<Row>(`/api/issues/${issue!.id}`);
    const listed = await api.get<Row[]>(
      `/api/companies/${fixtures.company.id}/heartbeat-runs?limit=100`,
    );
    runs = await Promise.all(
      listed.map((r) => api.get<Row>(`/api/heartbeat-runs/${r.id}`)),
    );
    input.observe(issue, runs, checks);
    return { issue, runs };
  }
  let pausedRuntimeRunIds = new Set<string>();
  async function settle(prior: Set<string>, requireQuestion = false, answeredInteractionId?: string) {
    let stable = "";
    const previousPaused = pausedRuntimeRunIds;
    await pollUntil({
      label: `continuation ${scenario.id} settled`,
      deadlineAt: input.deadlineAt,
      intervalMs: 1000,
      load: async () => ({
        ...await refresh(),
        interactions: await api.get<Row[]>(`/api/issues/${issue!.id}/interactions`),
      }),
      accept: (state) => {
        const paused = answerableRuntimeRunIds(state.interactions);
        const idle =
          continuationAnswerCommitted(state.interactions, answeredInteractionId) &&
          continuationCheckpointReady({
            requireCompletedTurn: migration,
            issue: { status: state.issue.status, executionRunId: state.issue.executionRunId },
            runs: state.runs.map(r => ({ id: r.id, status: r.status, runtimeMode: r.runtimeMode })),
            interactions: state.interactions,
          }) &&
          state.runs.some((r) => !prior.has(r.id) || previousPaused.has(r.id)) &&
          state.runs.every((r) => ["succeeded", "failed", "timed_out", "cancelled"].includes(r.status) ||
            (r.status === "running" && paused.has(r.id))) &&
          !state.issue.scheduledRetry &&
          !state.issue.activeRecoveryAction &&
          (!requireQuestion || continuationInitialReady(state.interactions));
        const key = idle
          ? JSON.stringify([state.issue.status, state.issue.executionRunId,
              state.runs.map(r => [r.id, r.status]), state.interactions.map(i => [i.id, i.status])])
          : "";
        const ready = !!key && key === stable;
        stable = key;
        if (ready) pausedRuntimeRunIds = paused;
        return ready;
      },
      reject: (state) =>
        state.runs.length > (scenario.id === "necessary-free-text" ? 2 : scenario.id === "question-answer-resume" ? 3 : 12)
          ? "Bounded continuation run count exceeded"
          : state.runs.some((r) =>
                ["failed", "timed_out", "cancelled"].includes(r.status),
              )
            ? `Provider execution failed: ${state.runs
                .filter((r) => r.status !== "succeeded")
                .map(
                  (r) => `${r.id} ${r.errorCode ?? r.status}: ${r.error ?? ""}`,
                )
                .join("; ")}`
            : undefined,
    });
  }
  async function open() {
    // A restarted dev server can leave DOMContentLoaded pending after the app
    // is interactive (see chat-flow). Require the rendered task thread instead.
    await page.goto(
      `/${fixtures.company.issuePrefix}/issues/${issue!.identifier ?? issue!.id}`,
      { waitUntil: "commit", timeout: 60_000 },
    );
    await waitForTaskChatRendered(page, undefined, 60_000);
  }
  async function snapshot(phase: ContinuationCheckpoint["phase"]) {
    const [tasks, summaries, comments, interactions, attachments, activity] =
      await Promise.all([
        api.get<Row[]>(tasksPath),
        api.get<Row[]>(`/api/issues/${issue!.id}/documents`),
        api.get<Row[]>(`/api/issues/${issue!.id}/comments?order=asc`),
        api.get<Row[]>(`/api/issues/${issue!.id}/interactions`),
        captureFirstTaskAttachments(api, [{ id: issue!.id }], input.secrets),
        api.get<Row[]>(`/api/issues/${issue!.id}/activity`),
      ]);
    const documents = await Promise.all(
      summaries.map((d) =>
        api.get<Row>(
          `/api/issues/${issue!.id}/documents/${encodeURIComponent(d.key)}`,
        ),
      ),
    );
    checkpoints.push({
      phase,
      lifecycle: {
        executionRunId: issue!.executionRunId ?? null,
        scheduledRetry: issue!.scheduledRetry ?? null,
        activeRecoveryAction: issue!.activeRecoveryAction ?? null,
        monitorNextCheckAt: issue!.monitorNextCheckAt ?? null,
      },
      issue: issue as ContinuationCheckpoint["issue"],
      children: tasks.filter(
        (t) => t.parentId === issue!.id,
      ) as ContinuationCheckpoint["children"],
      documents: documents as ContinuationCheckpoint["documents"],
      comments,
      activity,
      interactions,
      attachments,
      runs: [...runs] as ContinuationCheckpoint["runs"],
    });
    await input.evidence("continuation.json", {
      ...scenario,
      budgetGuard,
      checkpoints,
      checks,
    });
    await input.evidence("api-state.json", checkpoints.at(-1));
    await open();
    if (scenario.id === "necessary-free-text") {
      if (phase === "initial") {
        const contract = necessaryQuestionContract();
        await expect(page.getByText(contract.prompt, { exact: true }).last()).toBeVisible({ timeout: 30_000 });
        await expect(page.getByText(contract.context, { exact: true }).last()).toBeVisible({ timeout: 30_000 });
        const question = interactions.find(interaction => interaction.kind === "ask_user_questions" && interaction.status === "pending");
        checkpoints.at(-1)!.questionPresentation = { interactionId: question?.id ?? "", promptVisible: true, contextVisible: true };
      } else if (phase === "final") {
        expect(documents).toHaveLength(1);
        const document = documents[0]!;
        expect(document.latestRevisionId).toBeTruthy();
        const href = `/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue!.identifier ?? issue!.id)}#document-${encodeURIComponent(document.key)}`;
        const url = new URL(href, page.url()).href;
        const link = page.getByTestId("task-chat-agent-bubble").locator(`a[href=${JSON.stringify(href)}]`).last();
        await expect(link).toBeVisible({ timeout: 30_000 });
        await link.click();
        await expect(page).toHaveURL(url);
        const target = page.locator([
          `[id=${JSON.stringify(`document-${document.key}`)}]:visible`,
          `[id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`,
        ].join(", "));
        await expect(target).toHaveCount(1);
        await expect(target).toBeVisible();
        const visibleBody = `Booking confirmed: ${scenario.answer}`;
        await expect(target).toContainText(visibleBody);
        checkpoints.at(-1)!.outputDocument = { issueId: issue!.id, key: document.key, revisionId: document.latestRevisionId, href, visibleBody };
      }
    }
    await captureLoadedContinuation(page, String(issue!.title), () => input.capture(
      phase,
      `Continuation: ${phase}`,
      continuationScreenshotFile(phase),
    ));
  }
  async function answer(choice?: string) {
    const interactions = await api.get<Row[]>(
      `/api/issues/${issue!.id}/interactions`,
    );
    const questions = interactions.filter(
      (i) => i.kind === "ask_user_questions" && i.status === "pending",
    );
    expect(questions, "one real question must be shown").toHaveLength(1);
    const set = chatQuestionPresentation(questions[0].payload);
    if (scenario.id === "necessary-free-text") {
      expect(validResumeQuestionForm(checkpoints[0]!, questions[0], false), "a bound text-only question").toBe(true);
    } else if (scenario.id === "question-answer-resume") {
      expect(choice ? validResumeQuestionForm(checkpoints.at(-1)!, questions[0], true)
        : validResumeReferenceForm(checkpoints.at(-1)!, questions[0]), "a verified native question path and the requested form").toBe(true);
    } else if (scenario.id === "provider-question-bridge") {
      expect(isSingleClaudeQuestion(set.questions), "one choice question with only the optional provider Other field").toBe(true);
    } else expect(set.questions, "ask only the requested next question").toHaveLength(1);
    const before = new Set(runs.map((r) => r.id));
    await answerResumeQuestionThroughUi(page, questions[0], choice ? { choice } : {
      text: scenario.answer, allowProviderCustomAnswer: scenario.id === "question-answer-resume",
    });
    await settle(before, false, questions[0].id);
  }
  async function reply(body: string) {
    const before = new Set(runs.map((r) => r.id));
    await sendChatMessage(page, body);
    await settle(before);
  }
  function assertWaiting() {
    const c = checkpoints.at(-1)!;
    expect(
      c.documents.filter((d) => !isContinuationPlan(d, c)),
      "no deliverable before authorization",
    ).toHaveLength(0);
    expect(c.attachments, "no attachment before authorization").toHaveLength(0);
    expect(c.issue.status, "waiting is not complete").not.toBe("done");
  }
  try {
    budgetGuard = await prepareContinuationBudget(api, fixtures.company.id, fixtures.agent.id);
    if (execution.profile.generation === "legacy") await prepareLegacyContinuationSkill(api, fixtures.company.id, fixtures.agent.id);
    await api.patch("/api/instance/settings/experimental", {
      enableClassicTaskInterface: false,
    });
    const createdTask = await createTaskThroughUi({
      page,
      issuePrefix: fixtures.company.issuePrefix!,
      agentName: fixtures.agent.name,
      title: execution.task.buildTitle(input.nonce),
      prompt: migration ? legacyWorkspaceMigrationPrompt(scenario.prompt) : scenario.prompt,
      workMode: "standard",
    });
    issue = await pollUntil({
      label: "continuation task created",
      deadlineAt: input.deadlineAt,
      load: async () =>
        (await api.get<Row[]>(tasksPath)).find(
          (t) => t.id === createdTask.issueId,
        ),
      accept: Boolean,
    });
    if (!issue) throw new Error("Missing continuation task");
    await settle(new Set(), scenario.id !== "revision-preserves-approval");
    await snapshot("initial");
    assertWaiting();
    if (migration) {
      migrationBefore = await readMigrationPoint({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id, issueId: issue.id });
      await input.evidence("legacy-migration-before.json", migrationBefore);
      await input.restart();
      await open();
    }
    if (scenario.id === "untrusted-evidence") {
      const parentRun = runs.find((r) => r.contextSnapshot?.issueId === issue!.id);
      await seedContinuationContext({
        isolatedRoot: path.dirname(input.workspacePath),
        recordedCwd: parentRun?.contextSnapshot?.paperclipWorkspace?.cwd,
        body: scenario.context,
      });
    }
    if (scenario.id === "completed-action-resume") {
      await input.restart();
      await open();
    }
    if (["question-tool-documentation", "question-answer-resume"].includes(scenario.id)) {
      await answer("Afternoon");
      await snapshot("answered");
      assertWaiting();
      await answer();
    } else if (scenario.id === "provider-question-bridge") await answer(scenario.marker);
    else if (scenario.id === "revision-preserves-approval")
      await reply(scenario.revision);
    else await answer();
    if (scenario.gate) {
      await snapshot(
        scenario.id === "revision-preserves-approval" ? "revised" : "answered",
      );
      assertWaiting();
      if (scenario.id === "revision-preserves-approval") {
        const before = new Set(runs.map(r => r.id));
        const accepted = await confirmContinuationPlan({ page, api, checkpoint: checkpoints.at(-1)!, deadlineAt: input.deadlineAt });
        await input.evidence("continuation-plan-approval.json", accepted);
        await settle(before);
      } else await reply(scenario.approval);
    }
    if (migration) migrationAfter = await readMigrationPoint({ api, companyId: fixtures.company.id, agentId: fixtures.agent.id, issueId: issue!.id });
    await snapshot("final");
  } finally {
    checks = gradeContinuation({
      ...scenario,
      checkpoints: migration ? migrationPrimaryOutputCheckpoints(checkpoints, migrationBefore, migrationAfter) : checkpoints,
      runtimeMode: execution.profile.expectedRuntimeMode,
    });
    checks.push(...gradeLifecycleBaseline(checkpoints));
    if (migration) {
      checks.push(...gradeLegacyWorkspaceMigration(migrationBefore, migrationAfter, checkpoints));
      await input.evidence("legacy-migration.json", { before: migrationBefore, after: migrationAfter, checks: gradeLegacyWorkspaceMigration(migrationBefore, migrationAfter, checkpoints) });
    }
    if (scenario.id === "necessary-free-text") checks.push(...gradeNecessaryQuestion(checkpoints, scenario.answer));
    if (lifecycleProbe) checks.push(gradeLifecycleNarrative({
      narrative: lifecycleProbe.narrative,
      agentId: fixtures.agent.id,
      initial: checkpoints.find(c => c.phase === "initial"),
    }));
    if (issue) input.observe(issue, runs, checks);
    await input.evidence("continuation.json", {
      ...scenario,
      budgetGuard,
      checkpoints,
      checks,
    });
    await input.evidence(
      "continuation-run-evidence.json",
      await Promise.all(
        runs.map(async (r) => {
          try {
            return await collectChatRunEvidence(api, r as ChatRun);
          } catch (error) {
            return { runId: r.id, evidenceCaptureError: String(error) };
          }
        }),
      ),
    );
  }
  const failures = checks.filter((c) => !c.passed);
  if (failures.length)
    throw new Error(
      `Continuation matcher failures: ${failures.map((c) => `${c.id}: ${c.detail}`).join("; ")}`,
    );
  return { issue: issue!, runs, checks };
}