PaperClipAI/tests/runner-e2e/chat-flow.ts

import { runHiringTemplateFlow } from "./hiring-template-flow.js";
import { gradeHiringTemplateTurns } from "./hiring-template-turn-accounting.js";
import type { HiringTemplateEvidence } from "./hiring-template-scoring.js";
import { runAmbiguousConfirmationReply, runUnansweredQuestionReturn } from "./confirmation-replies.js";
import { expect, type Page } from "@playwright/test";
import { pollUntil, type RunnerApi } from "./api.js";
import type {
  AskUserQuestionsPayload,
  PaperclipQuestionSetPayload,
} from "../../packages/shared/src/types/issue.js";
import type { LiveFixtureValues } from "./live-fixtures.js";
import type { MatrixExecution } from "./types.js";
import { isBlockedUnstartedWake } from "./non-execution-wake.js";
import { collectRunEvents } from "./run-observations.js";
import { chatMarker } from "./chat-cases.js";
import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";
import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js";
import { runActiveReassignment, runWorkerCrash, runAnswerQuality } from "./chat-qualification.js";
import { matchesRunCount, minimumRunCount } from "./run-count.js";
import { runChatCompletionUpdate } from "./completion-update-flow.js";
import { CHAT_FEATURE_SUITE } from "./chat-feature-cases.js";
import { runChatFeatureFlow } from "./chat-feature-flow.js";

/** Hiring alone admits bounded, verified lifecycle notifications. Other suites keep their count contract. */
export function assertChatFlowRunCount(input: {
  suiteId: string; task: Parameters<typeof matchesRunCount>[0]; runs: ChatRun[];
  hiringEvidence?: HiringTemplateEvidence; hiringApiState?: unknown;
}) {
  expect(matchesRunCount(input.task, input.runs.length), "Declared total provider-run bounds").toBe(true);
  if (input.suiteId !== "hiring-templates") return;
  const accounting = gradeHiringTemplateTurns({
    evidence: input.hiringEvidence ? { ...input.hiringEvidence, runs: input.runs } : undefined,
    apiState: input.hiringApiState,
  });
  expect(accounting.passed, `Hiring lifecycle: ${accounting.predicates.filter(p => !p.passed).map(p => p.id).join(", ")}`).toBe(true);
  return accounting;
}

// Public API observations only: this driver never fabricates provider results or writes DB state.
export interface ChatIssue {
  id: string;
  companyId: string;
  title: string;
  status: string;
  identifier?: string;
  conversationState?: string;
  conversationSessionGeneration?: number;
  conversationBoundaryCommentId?: string;
  parentId?: string | null;
  projectId?: string | null;
  assigneeAgentId?: string | null;
  scheduledRetry?: unknown;
  activeRecoveryAction?: unknown;
}
export interface ChatRun {
  id: string;
  companyId: string;
  agentId: string;
  status: string;
  error?: string | null;
  errorCode?: string | null;
  runtimeMode?: string;
  runtimeModeResolvedAt?: string | null;
  logStore?: string | null;
  logRef?: string | null;
  contextSnapshot?: Record<string, unknown>;
  resultJson?: Record<string, unknown>;
  sessionIdBefore?: string | null;
  sessionIdAfter?: string | null;
  startedAt?: string | null;
}
type Comment = {
  id: string;
  body: string;
  authorAgentId?: string;
  createdByRunId?: string;
  conversationSessionGeneration?: number;
};
type Plan = { body: string; latestRevisionId: string; updatedAt: string };
type ChatOutputDocument = Plan & { id: string; issueId: string; key: string };

export function assertChatBacklogCreation(input: {
  tasks: ChatIssue[];
  runs: ChatRun[];
  ownerId: string;
  plan: Plan;
  marker: string;
  activity: Array<{ action: string; details?: Record<string, unknown> }>;
}) {
  expect(input.tasks).toHaveLength(1);
  expect(input.tasks[0]).toMatchObject({ status: "backlog", parentId: null, assigneeAgentId: input.ownerId });
  expect(input.runs.filter(run => run.contextSnapshot?.issueId === input.tasks[0]!.id)).toHaveLength(0);
  expect(input.plan.body).toContain(input.marker);
  expect(input.plan.latestRevisionId).toBeTruthy();
  const created = input.activity.filter(row => row.action === "issue.created");
  expect(created).toHaveLength(1);
  // Correcting todo after creation still permits an unauthorized start race.
  expect(created[0]!.details).toMatchObject({ status: "backlog", source: "paperclip_runner_protocol" });
}

/** Clarification may request information imperatively rather than end in a question mark. */
export function isChatClarificationReply(body: string): boolean {
  if (body.includes("?")) return true;
  // A present-tense request can introduce the required information directly,
  // without a noun such as "brief" or "details" before the list.
  const listRequest = body.match(/\b(?:I|we)(?:'ll|\s+will)?\s+need\s*:\s*([\s\S]*)/i);
  if (listRequest) {
    const items: string[] = [];
    let paragraphBoundary = false;
    // Only the directly introduced list belongs to this request. A later
    // paragraph can introduce a separate plan without invalidating it.
    for (const line of listRequest[1].split(/\r?\n/)) {
      if (!line.trim()) {
        paragraphBoundary = true;
        continue;
      }
      const item = line.match(/^\s*(?:[-*]|\d+[.)])\s+(.+)$/);
      if (!item) {
        // Markdown permits soft-wrapped item text without indentation. A
        // separate paragraph or heading ends the introduced information list.
        if (items.length && !paragraphBoundary && !/^\s*#|:\s*$/.test(line)) {
          items[items.length - 1] += ` ${line.trim()}`;
          continue;
        }
        break;
      }
      items.push(item[1].replace(/^\[[ xX]\]\s*/, "").replace(/^[*_`]+/, ""));
      paragraphBoundary = false;
    }
    // Positively identify questions or brief fields. Unknown bullets are not
    // evidence of clarification merely because their verbs are unlisted.
    const information = /^(?:(?:who|what|where|when|whether|how)\b|(?:(?:any|the|your|preferred|intended|required|target|desired|existing|delivery|must-include)\s+){0,3}(?:(?:club|company|project|team|event|organization)\s+name|audience|readers|recipients|tone|voice|format|length|deadline|date|purpose|context|details|requirements|constraints|examples|links|name|budget|location|venue|schedule|timezone|language|accessibility|contact|scope|goals|background|references|assets|brand|style)\b)/i;
    if (items.length >= 2 && items.every(item => information.test(item))) return true;
  }
  const request = body.match(
    /\b(?:please\s+(?:share|provide|clarify|confirm)|tell me|let me know)\b([\s\S]*)/i,
  ) ?? body.match(
    /\b(?:I|we)(?:'ll|\s+will)?\s+need\s+(?:(?:a|some|the|your|more|following|compact|short|few|additional)\s+){0,4}(?:brief|details|information|context|clarification)\b([\s\S]*)/i,
  );
  return Boolean(request && /[\p{L}\p{N}]/u.test(request[1]));
}

export function assertChatExecutionOutput(
  body: string,
  marker: string,
  supersededMarker?: string,
): void {
  expect(body).toContain(marker);
  if (supersededMarker) expect(body).not.toContain(supersededMarker);
}

/** A requested output document may have a descriptive key; a copied plan is not output. */
export async function readChatOutputDocument(
  api: Pick<RunnerApi, "get">,
  issueId: string,
  marker: string,
): Promise<ChatOutputDocument> {
  const summaries = await api.get<Array<{ key: string }>>(
    `/api/issues/${issueId}/documents`,
  );
  const documents = await Promise.all(
    summaries
      .filter((document) => document.key !== "plan")
      .map((document) =>
        api.get<ChatOutputDocument>(
          `/api/issues/${issueId}/documents/${encodeURIComponent(document.key)}`,
        ),
      ),
  );
  const output = documents.find(
    (document) => document.issueId === issueId && document.body.includes(marker),
  );
  if (!output)
    throw new Error(
      `Execution task ${issueId} has no non-plan output document containing ${marker}; document keys: ${summaries.map((document) => document.key).join(", ") || "none"}`,
    );
  expect(output.id).toBeTruthy();
  expect(output.latestRevisionId).toBeTruthy();
  return output;
}

export const isResetRun = (run: ChatRun) =>
  run.contextSnapshot?.conversationReset === true ||
  run.resultJson?.conversationReset === true;
export function chatRunFailure(
  runs: ChatRun[],
  allowCancelled = false,
): string | undefined {
  const failed = runs.find(
    (run) =>
      ["failed", "timed_out"].includes(run.status) ||
      (!allowCancelled && run.status === "cancelled"),
  );
  return failed
    ? `run ${failed.id} ${failed.status}${failed.errorCode ? ` (${failed.errorCode})` : ""}${failed.error ? `: ${failed.error}` : ""}`
    : undefined;
}

export function chatTaskCompletionFailure(
  task: ChatIssue,
  runs: ChatRun[],
): string | undefined {
  if (
    task.scheduledRetry ||
    task.activeRecoveryAction ||
    runs.some((run) => ["queued", "running"].includes(run.status))
  )
    return undefined;
  return chatRunFailure(runs);
}

/** Require two settled observations so an in-flight finalization is not a failure. */
export function createChatIdleFailureDetector(minimumProviderRuns: number) {
  let priorInconsistentState: string | undefined;
  return (state: {
    resolved: boolean;
    status?: string;
    conversationState?: string;
    providerRunCount: number;
    activeRuns: string[];
  }): string | undefined => {
    const inconsistentState =
      state.resolved &&
      state.providerRunCount >= minimumProviderRuns &&
      state.activeRuns.length === 0 &&
      state.conversationState === "waiting" &&
      ["blocked", "done", "cancelled"].includes(state.status ?? "")
        ? `${state.status}:${state.providerRunCount}`
        : undefined;
    const stable =
      inconsistentState !== undefined &&
      inconsistentState === priorInconsistentState;
    priorInconsistentState = inconsistentState;
    return stable
      ? `chat_idle_state_invariant: conversation is ${state.status} while waiting after ${state.providerRunCount} settled provider runs`
      : undefined;
  };
}

/** Match the shared question form's durable/native presentation, including custom labels. */
export function chatQuestionPresentation(
  payload: AskUserQuestionsPayload,
): PaperclipQuestionSetPayload {
  if (payload.questionSet) return payload.questionSet;
  return {
    schema: "paperclip.question_set.v1",
    ...(payload.submitLabel ? { submitLabel: payload.submitLabel } : {}),
    questions: payload.questions.map((question) => {
      const freeText = question.options.find((option) => option.freeText);
      return {
        id: question.id,
        prompt: question.prompt,
        required: question.required === true,
        answerMode:
          question.selectionMode === "multi" ? "multi_select" : "single_select",
        ...(freeText
          ? { customAnswer: { enabled: true as const, label: freeText.label } }
          : {}),
      };
    }),
  };
}

export function assertChatTaskHandoff(
  task: ChatIssue,
  runs: ChatRun[],
  source: ChatIssue,
) {
  expect(task.parentId).toBeNull();
  expect(task.projectId).toBeTruthy();
  expect(task.assigneeAgentId).toBe(source.assigneeAgentId);
  expect(runs.length).toBeGreaterThan(0);
}

/** A running row can precede creation of its log file. Only that expected 404 is retryable. */
export async function readRunningChatLog(
  api: Pick<RunnerApi, "request">,
  runId: string,
): Promise<string | undefined> {
  const response = await api.request.get(
    `/api/heartbeat-runs/${runId}/log?limitBytes=65536`,
  );
  if (response.status() === 404) return undefined;
  if (!response.ok())
    throw new Error(`Run ${runId} log returned ${response.status()}`);
  return ((await response.json()) as { content?: string }).content;
}

/** Reset/blocked runs and proven pre-provider failures may have no provider log.
 * Preserve their durable events; missing logs for real provider work still fail. */
export async function collectChatRunEvidence(
  api: Pick<RunnerApi, "get">,
  run: ChatRun,
) {
  const recovery = run.resultJson?.executionRecovery as Record<string, unknown> | undefined;
  const providerNeverStarted = run.runtimeMode === "legacy" && !run.runtimeModeResolvedAt &&
    run.logStore === null && run.logRef === null && ["failed", "interrupted"].includes(run.status) &&
    recovery?.kind === "bootstrap" && recovery.providerWorkStarted === false;
  return {
    runId: run.id,
    ...(providerNeverStarted ? { logOmissionReason: "provider_not_started" } : {}),
    log: isResetRun(run) || isBlockedUnstartedWake({ ...run }) || providerNeverStarted
      ? null
      : await api.get(`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`),
    events: await collectRunEvents((afterSeq, limit) =>
      api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`),
    ),
  };
}

export function assertChatHandoff(
  task: ChatIssue,
  plan: Plan,
  runs: ChatRun[],
  source: ChatIssue,
) {
  assertChatTaskHandoff(task, runs, source);
  expect(plan.body.trim()).not.toBe("");
  for (const run of runs) {
    expect(Date.parse(plan.updatedAt)).toBeLessThanOrEqual(
      Date.parse(run.startedAt!),
    );
  }
}
export async function sendChatMessage(page: Page, message: string) {
  const composer = page.getByTestId("task-chat-composer-input").last();
  const takeover = page.getByTestId("task-chat-composer-takeover").last();
  await expect(composer.or(takeover).first()).toBeVisible();
  if (await takeover.isVisible()) {
    // Close the card's composer view without answering, skipping, or approving it.
    await takeover.getByRole("button", { name: /^Dismiss / }).click();
  }
  await composer
    .locator('[contenteditable="true"], textarea')
    .first()
    .fill(message);
  await page.getByTestId("task-chat-composer-send").last().click();
}


/** Independent durable oracle: prose alone cannot make a reassignment pass. */
export function assertChatReassignment(input: {
  readyId: string; queuedId: string; teammateId: string; tasks: ChatIssue[]; runs: ChatRun[];
  audit: Array<{ action: string; details?: Record<string, unknown> }>; outputBody: string; marker: string;
}) {
  expect(input.tasks.map(task => task.id).sort()).toEqual([input.readyId, input.queuedId].sort());
  expect(input.tasks.find(task => task.id === input.readyId)).toMatchObject({ status: "done", assigneeAgentId: input.teammateId });
  expect(input.tasks.find(task => task.id === input.queuedId)).toMatchObject({ status: "backlog", assigneeAgentId: input.teammateId });
  const successor = input.runs.filter(run => run.contextSnapshot?.issueId === input.readyId);
  expect(successor).toHaveLength(1);
  expect(successor[0]).toMatchObject({ agentId: input.teammateId, status: "succeeded", runtimeMode: "native" });
  expect(input.runs.filter(run => run.contextSnapshot?.issueId === input.queuedId)).toHaveLength(0);
  expect(input.audit.filter(row => row.action === "issue.reassigned" && row.details?.source === "paperclip_runner_protocol")).toHaveLength(1);
  expect(input.outputBody).toContain(input.marker);
}

export interface ChatFlowInput {
  page: Page;
  api: RunnerApi;
  fixtures: LiveFixtureValues;
  execution: MatrixExecution;
  nonce: string;
  workspacePath: string;
  restart: () => Promise<void>;
  observe: (issue: ChatIssue, runs: ChatRun[]) => void;
  capture: (id: string, label: string, file: string) => Promise<void>;
  evidence: (name: string, data: unknown) => Promise<void>;
  check?: (id: string, passed: boolean, detail: string) => void;
}
export async function runChatFlow(input: ChatFlowInput) {
  const { page, api, fixtures: f, execution, nonce } = input;
  const chatPath = `/api/companies/${f.company.id}/chats/${f.agent.id}`;
  const route = `/${f.company.issuePrefix}/chats/${f.agent.id}`;
  const marker = execution.task.buildVisibleMarker(nonce);
  const draftMarker = chatMarker("DRAFT", nonce);
  const caseId = execution.task.id;
  const stopCase = ["stop-new-resume", "stop-startup-new-resume"].includes(caseId);
  const interruptionCase = ["followup-while-running", "revise-while-running"].includes(caseId);
  const expectedStops = new Map<string, string>();
  let issue: ChatIssue;
  let runs: ChatRun[] = [];
  const settings = await api.get<Record<string, unknown>>(
    "/api/instance/settings/experimental",
  );
  const allRuns = async () => {
    const rows = await api.get<ChatRun[]>(
      `/api/companies/${f.company.id}/heartbeat-runs?limit=100`,
    );
    return Promise.all(
      rows.map((row) => api.get<ChatRun>(`/api/heartbeat-runs/${row.id}`)),
    );
  };
  const tasks = () =>
    api.get<ChatIssue[]>(`/api/companies/${f.company.id}/issues`);
  const comments = async () =>
    (
      await api.get<Array<Comment & { createdAt: string }>>(
        `/api/issues/${issue.id}/comments?order=asc`,
      )
    ).sort(
      (a, b) =>
        a.createdAt.localeCompare(b.createdAt) || a.id.localeCompare(b.id),
    );
  const idle = async (minimumProviderRuns: number) => {
    const inconsistentIdle = createChatIdleFailureDetector(minimumProviderRuns);
    await pollUntil({
      label: "chat turn settles to waiting",
      deadlineAt: Date.now() + 240_000,
      intervalMs: 1000,
      load: async () => {
        const resolved = await api.get<ChatIssue | null>(chatPath);
        if (resolved) issue = resolved;
        runs = await allRuns();
        if (resolved) input.observe(resolved, runs);
        return {
          resolved: Boolean(resolved),
          status: resolved?.status,
          conversationState: resolved?.conversationState,
          providerRunCount: runs.filter((run) => !isResetRun(run)).length,
          activeRuns: runs
            .filter((run) => ["queued", "running"].includes(run.status))
            .map((run) => run.id),
          failure: chatRunFailure(runs.filter(run => expectedStops.get(run.id) !== run.status), stopCase),
        };
      },
      reject: (state) => state.failure ?? inconsistentIdle(state),
      accept: (state) =>
        !state.failure &&
        state.resolved &&
        state.providerRunCount >= minimumProviderRuns &&
        state.activeRuns.length === 0 &&
        state.status === "in_review" &&
        state.conversationState === "waiting",
    });
  };
  const turn = async (text: string, count: number) => {
    await sendChatMessage(page, text);
    await idle(count);
  };
  const noTasks = async () => expect(await tasks()).toHaveLength(0);
  let hiringEvidence: HiringTemplateEvidence | undefined;
  let refreshHiringEvidence: (() => Promise<HiringTemplateEvidence>) | undefined;
  try {
    if (caseId === "enable-disable-resume") await enableChatThroughSettings(input);
    else await api.patch("/api/instance/settings/experimental", {
      enableAgentChat: true,
      enableClassicTaskInterface: false,
    });
    expect(await api.get(chatPath)).toBeNull();
    // Cold Vite startup can keep unrelated assets loading after the chat is
    // interactive. The composer assertion below verifies actual UI readiness.
    await page.goto(route, { waitUntil: "domcontentloaded", timeout: 60_000 });
    await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();
    expect(await api.get(chatPath)).toBeNull();
    expect(await allRuns()).toHaveLength(0);

    if (execution.suite.id === CHAT_FEATURE_SUITE) {
      await runChatFeatureFlow({ input, issue: () => issue, allRuns,
        refreshIssue: async () => { issue = await api.get<ChatIssue>(chatPath); if (issue) input.observe(issue, await allRuns()); } });
    } else if (caseId === "confirmation-ambiguous") {
      await runAmbiguousConfirmationReply({ input, issue: () => issue!, idle, comments });
    } else if (caseId === "unanswered-question-return") {
      await runUnansweredQuestionReturn({ input, issue: () => issue!, idle, comments });
    } else if (caseId.startsWith("handoff-completion-")) {
      await runChatCompletionUpdate({ input, marker, allRuns, issue: () => issue!,
        refreshIssue: async () => { issue = await api.get<ChatIssue>(chatPath); if (issue) input.observe(issue, await allRuns()); } });
    } else if (execution.suite.id === "hiring-templates") {
      const hiring = await runHiringTemplateFlow({ input, issue: () => issue!, turn, tasks, allRuns });
      hiringEvidence = hiring.evidence;
      refreshHiringEvidence = hiring.refresh;
    } else if (execution.suite.id === "agent-chat-qualification") {
      const context = { input, marker, issue: () => issue!, idle, allRuns, comments, expectedStops,
        refreshIssue: async () => { issue = await api.get<ChatIssue>(chatPath); input.observe(issue, await allRuns()); } };
      if (caseId === "active-reassignment") await runActiveReassignment(context);
      else if (caseId === "worker-crash-retry") await runWorkerCrash(context);
      else await runAnswerQuality(context);
    } else if (caseId === "enable-disable-resume") {
      await runChatSettingsLifecycle({ input, marker, issue: () => issue!, idle, allRuns, comments });
    } else if (interruptionCase) {
      await runChatInterruption({ input, marker, issue: () => issue!, idle, allRuns, comments,
        refreshIssue: async () => { issue = await api.get<ChatIssue>(chatPath); input.observe(issue, await allRuns()); } });
    } else if (
      ["continuity-restart", "new-session", "stop-new-resume", "stop-startup-new-resume"].includes(caseId)
    ) {
      const secret = chatMarker("OLDCONTEXT", nonce);
      await turn(
        `For this conversation only, remember the phrase ${secret}. Just acknowledge briefly; no project or task is needed.`,
        1,
      );
      const initialId = issue!.id;
      const before = runs.filter((run) => !isResetRun(run))[0]!;
      await noTasks();
      await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
      if (caseId === "continuity-restart") {
        await turn(
          "What phrase did I just ask you to remember? Reply with the phrase only.",
          2,
        );
        expect(
          (await comments()).filter((c) => c.authorAgentId).at(-1)?.body,
        ).toContain(secret);
        const count = runs.length;
        await input.restart();
        // Re-enter the canonical route after the server replaces its browser
        // transport; reloading the stale document can target a detached page.
        // The restarted dev server can leave DOMContentLoaded pending after the
        // app is interactive. Require the actual chat composer after navigation.
        await page.goto(route, { waitUntil: "commit", timeout: 60_000 });
        await expect(page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: 60_000 });
        await idle(2);
        expect(runs).toHaveLength(count);
        await turn(
          `What phrase did I ask you to remember earlier? Reply with that remembered phrase followed by ${marker}; no further work.`,
          3,
        );
        assertChatRememberedAfterRestart(
          (await comments()).filter((comment) => comment.authorAgentId).at(-1)?.body ?? "",
          secret,
          marker,
        );
      } else {
        let cancelledId: string | undefined;
        if (stopCase) {
          await sendChatMessage(
            page,
            "Explain the history of gardening at length here, in 100 numbered paragraphs. This is discussion only; do not create work.",
          );
          await expect
            .poll(
              async () => {
                runs = await allRuns();
                const active = runs.find(
                  (run) => run.status === "running" && run.id !== before.id,
                );
                if (!active) return false;
                const events = await api.get<Array<Record<string, unknown>>>(
                  `/api/heartbeat-runs/${active.id}/events?limit=1000`,
                );
                if (execution.profile.generation === "native") {
                  if (!isChatStopReady(events, caseId === "stop-startup-new-resume" ? "startup" : "active")) return false;
                } else if (!(events.length || (await readRunningChatLog(api, active.id))?.length)) return false;
                cancelledId = active.id;
                await input.evidence("chat-stop-boundary.json", {
                  phase: caseId === "stop-startup-new-resume" ? "startup" : "active",
                  runId: active.id, events,
                });
                return true;
              },
              { timeout: 120_000 },
            )
            .toBe(true);
          await page.getByTestId("task-chat-composer-stop").click();
          await expect
            .poll(
              async () =>
                (await api.get<ChatRun>(`/api/heartbeat-runs/${cancelledId}`))
                  .status,
              { timeout: 30_000 },
            )
            .toBe("cancelled");
        }
        const oldComments = await comments();
        await sendChatMessage(page, "/new");
        await expect
          .poll(
            async () =>
              (await api.get<ChatIssue>(chatPath))
                .conversationSessionGeneration,
            { timeout: 30_000 },
          )
          .toBe(1);
        await expect
          .poll(async () => (await allRuns()).some(isResetRun))
          .toBe(true);
        await turn(
          `Without reading older history or files, if you have a remembered phrase in your current context return it; otherwise reply exactly ${marker}. Do not look it up.`,
          caseId === "new-session" ? 2 : 3,
        );
        const fresh = runs
          .filter((run) => !isResetRun(run) && run.status === "succeeded")
          .sort((a, b) => Date.parse(a.startedAt!) - Date.parse(b.startedAt!))
          .at(-1)!;
        expect(fresh.contextSnapshot?.conversationSessionGeneration).toBe(1);
        expect(fresh.sessionIdBefore).toBeFalsy();
        expect(
          String(fresh.contextSnapshot?.paperclipTaskMarkdown ?? ""),
        ).not.toContain(secret);
        if (before.sessionIdAfter && fresh.sessionIdAfter)
          expect(fresh.sessionIdAfter).not.toBe(before.sessionIdAfter);
        const replies = (await comments()).filter(
          (c) => c.createdByRunId === fresh.id && c.authorAgentId,
        );
        expect(replies.map((c) => c.body).join("\n")).toContain(marker);
        expect(replies.map((c) => c.body).join("\n")).not.toContain(secret);
        const reset = runs.filter(isResetRun);
        expect(reset).toHaveLength(1);
        expect(
          (await comments()).filter(
            (c) => c.authorAgentId && c.createdByRunId === reset[0]!.id,
          ),
        ).toHaveLength(0);
        expect(
          (await comments()).filter((c) =>
            oldComments.some((old) => old.id === c.id),
          ),
        ).toHaveLength(oldComments.length);
        if (cancelledId)
          expect(
            (await comments()).filter((c) => c.createdByRunId === cancelledId),
          ).toEqual(
            oldComments.filter((c) => c.createdByRunId === cancelledId),
          );
        if (caseId === "stop-startup-new-resume" && cancelledId) {
          const stopped = await api.get<ChatRun>(`/api/heartbeat-runs/${cancelledId}`);
          const events = await api.get<Array<{ eventType?: unknown; createdAt?: string }>>(`/api/heartbeat-runs/${cancelledId}/events?limit=1000`);
          await input.evidence("chat-startup-stop-result.json", { stopped, events });
          assertChatStartupStopped(stopped, events);
        }
        await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
        await expect(
          page.getByText("New session", { exact: true }),
        ).toHaveCount(1);
      }
      expect(issue!.id).toBe(initialId);
      await noTasks();
    } else if (["hire-delegate-reuse", "blocked-status-review", "committed-send-retry"].includes(caseId)) {
      await runChatHardeningFlow({ input, marker, issue: () => issue!, turn, idle, tasks, allRuns, comments });
    } else if (caseId === "create-backlog") {
      const project = await api.post<{ id: string }>(`/api/companies/${f.company.id}/projects`, {
        name: `Later planning ${nonce}`, description: "Repository-free plans to save for later.",
      });
      await turn(`Create exactly one task titled Later checklist ${nonce} in the existing Later planning ${nonce} project. Assign it to yourself but keep it in backlog: do not start or execute it. Save a concise three-step initial plan that contains ${marker}. Reply with its identifier and status. Do not create a replacement task or change other tasks.`, 1);
      const saved = (await tasks())[0]!;
      expect(saved).toBeTruthy();
      const plan = await api.get<Plan>(`/api/issues/${saved.id}/documents/plan`);
      await turn(`What are the current owner and status of ${saved.identifier}? Just report its saved state. Do not execute it, change its status, or create another task.`, 2);
      const observedTasks = await tasks();
      // Issue activity also links the conversation run that created the task.
      // Inspect actual run bindings to distinguish creation from execution.
      const observedRuns = await allRuns();
      const activity = await api.get<Array<{ action: string; details?: Record<string, unknown> }>>(`/api/issues/${saved.id}/activity`);
      const persistedPlan = await api.get<Plan>(`/api/issues/${saved.id}/documents/plan`);
      assertChatBacklogCreation({ tasks: observedTasks, runs: observedRuns, ownerId: f.agent.id, plan: persistedPlan, marker, activity });
      expect(observedTasks[0]).toMatchObject({ id: saved.id, projectId: project.id });
      expect(persistedPlan).toEqual(plan);
      expect((await comments()).filter(c => c.authorAgentId).at(-1)?.body).toMatch(/backlog/i);
      await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
      await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();
      await expect(page.getByRole("link", { name: saved.identifier! }).first()).toBeVisible();
      await input.capture("chat-backlog", "Planned backlog task without execution", "chat-backlog.png");
      await input.evidence("chat-backlog.json", { tasks: observedTasks, runs: observedRuns, activity, plan: persistedPlan });
    } else if (caseId === "reassign-task") {
      const config = execution.profile.buildAgent({
        environmentId: f.environment.id, environmentFixtureId: execution.environment.id,
        workspacePath: input.workspacePath, secretRefs: f.secretRefs, executionId: nonce,
      });
      const teammate = await api.post<{ id: string }>(`/api/companies/${f.company.id}/agents`, {
        ...config, name: "Riley Reassignment", role: "engineer", reportsTo: f.agent.id,
        instructionsBundle: { entryFile: "AGENTS.md", files: { "AGENTS.md": "Complete the assigned work and save the requested Paperclip document." } },
      });
      const queued = await api.post<ChatIssue>(`/api/companies/${f.company.id}/issues`, {
        title: `Later checklist ${nonce}`, description: `Preserve this deferred scope ${draftMarker}.`,
        status: "backlog", assigneeAgentId: f.agent.id,
      });
      const ready = await api.post<ChatIssue>(`/api/companies/${f.company.id}/issues`, {
        title: `Ready checklist ${nonce}`, status: "todo",
        description: `Write a short launch checklist as a Paperclip document attached to this task, including ${marker}. Complete this task after saving it.`,
      });
      await turn(`Assign the existing Ready checklist ${nonce} task to Riley Reassignment so Riley completes it. Also move the existing Later checklist ${nonce} task from you to Riley, keeping it in backlog. Preserve both tasks and their descriptions. Explain the handoff briefly here. Do not create replacement tasks or start the backlog work.`, 2);
      const observedTasks = await tasks();
      const readyAfter = await api.get<ChatIssue>(`/api/issues/${ready.id}`);
      const queuedAfter = await api.get<ChatIssue>(`/api/issues/${queued.id}`);
      const output = await readChatOutputDocument(api, ready.id, marker);
      const activity = await api.get<Array<{ action: string; details?: Record<string, unknown> }>>(`/api/issues/${ready.id}/activity`);
      assertChatReassignment({ readyId: ready.id, queuedId: queued.id, teammateId: teammate.id, tasks: observedTasks, runs,
        audit: activity, outputBody: output.body, marker });
      expect(queuedAfter).toMatchObject({ assigneeAgentId: teammate.id, status: "backlog", description: `Preserve this deferred scope ${draftMarker}.` });
      expect(readyAfter).toMatchObject({ assigneeAgentId: teammate.id, status: "done" });
      expect(issue!).toMatchObject({ assigneeAgentId: f.agent.id, conversationState: "waiting" });
      await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
      await expect(page.getByTestId("task-chat-composer-input")).toBeVisible();
      await expect(page.getByRole("link", { name: readyAfter.identifier! }).first()).toBeVisible();
      await input.capture("chat-reassignment", "Reassignment completed in Agent Chat", "chat-reassignment.png");
      await input.evidence("chat-reassignment.json", { tasks: observedTasks, runs, activity, output, teammateId: teammate.id });
    } else {
      let existingProject: { id: string; name: string } | undefined;
      let acceptedPlan: Plan | undefined;
      let repositoryCatalogBefore:
        { repositories: Array<{ id: string; url: string }> } | undefined;
      const expectedRepositoryUrls = [
        "https://github.com/octocat/Hello-World",
        "https://github.com/octocat/Spoon-Knife",
      ];
      if (caseId === "clarify-reuse") {
        existingProject = await api.post(
          `/api/companies/${f.company.id}/projects`,
          {
            name: `Garden ${nonce}`,
            description: "Garden club welcome notes and event announcements.",
          },
        );
        await turn(
          "I need a welcome note for our club. Help me clarify what information you need before assigning the work.",
          1,
        );
        await noTasks();
        const questions = await api.get<
          Array<{
            status: string;
            kind: string;
            payload: AskUserQuestionsPayload;
          }>
        >(`/api/issues/${issue!.id}/interactions`);
        const pendingInteraction = questions.find(
          (row) =>
            row.status === "pending" && row.kind === "ask_user_questions",
        );
        const questionSet = pendingInteraction
          ? chatQuestionPresentation(pendingInteraction.payload)
          : undefined;
        const pendingQuestions = questionSet?.questions;
        expect(
          Boolean(pendingQuestions?.length) ||
            (await comments()).some(
              (c) => c.authorAgentId && isChatClarificationReply(c.body),
            ),
        ).toBe(true);
        const clarification = `It is the garden club; use the existing Garden ${nonce} project. Make one assigned task for yourself to write a two-sentence welcome note. Include ${marker} in that note, save it as a Paperclip document attached to that execution task, and finish that execution task. Please get it started now.`;
        if (pendingQuestions?.length) {
          for (const [index, question] of pendingQuestions.entries()) {
            const textInput = page
              .getByTestId("question-text-answer-composer")
              .last();
            if (await textInput.isVisible()) {
              await textInput
                .locator('[contenteditable="true"],textarea')
                .first()
                .fill(clarification);
            } else {
              await page
                .getByRole(
                  question.answerMode === "multi_select" ? "checkbox" : "radio",
                  {
                    name: question.customAnswer?.label ?? "Other",
                    exact: true,
                  },
                )
                .last()
                .click();
              await page
                .getByTestId("question-other-answer-composer")
                .last()
                .locator('[contenteditable="true"],textarea')
                .first()
                .fill(clarification);
            }
            await page
              .getByRole("button", {
                name:
                  index === pendingQuestions.length - 1
                    ? (questionSet?.submitLabel ?? "Submit answers")
                    : "Next",
                exact: true,
              })
              .last()
              .click();
          }
          await idle(3);
        } else await turn(clarification, 3);
      } else if (caseId === "plan-handoff") {
        await page.getByTestId("task-chat-composer-add").click();
        await page.getByTestId("composer-add-plan").click();
        await turn(
          `Let's plan a two-sentence garden club welcome note. The finished welcome note itself must contain the exact phrase ${draftMarker}. Write a plan in the plan panel that includes this requirement, and present it for approval. When I approve the final revision, create a suitable repository-free project and an assigned task for yourself, copy the plan into that task, and have it save the note as a Paperclip document attached to that execution task and finish. Do not create the project or task before approval.`,
          1,
        );
        const draft = await api.get<Plan>(
          `/api/issues/${issue!.id}/documents/plan`,
        );
        expect(draft.body).toContain(draftMarker);
        await noTasks();
        await input.capture(
          "chat-plan-draft",
          "Draft plan in the conversation",
          "chat-plan-draft.png",
        );
        const initialInteractions = await api.get<
          Array<{
            status: string;
            kind: string;
            payload?: {
              target?: { revisionId?: string };
              rejectLabel?: string;
            };
          }>
        >(`/api/issues/${issue!.id}/interactions`);
        const initialApproval = initialInteractions.find(
          (row) =>
            row.status === "pending" &&
            row.kind === "request_confirmation" &&
            row.payload?.target?.revisionId === draft.latestRevisionId,
        );
        expect(
          initialApproval,
          "draft has a revision-bound approval",
        ).toBeTruthy();
        const reviseButton = page
          .getByRole("button", {
            name: initialApproval!.payload?.rejectLabel ?? "Reject",
            exact: true,
          })
          .last();
        await reviseButton.click();
        await page
          .getByTestId("plan-revision-composer")
          .last()
          .locator('[contenteditable="true"],textarea')
          .first()
          .fill(
            `Revise the plan: the finished welcome note itself must contain the exact phrase ${marker} instead of ${draftMarker}. Include that requirement in the revised plan. The execution task should save that welcome note as a Paperclip document attached to that task. Present this revised plan for approval; wait for that approval before handing it off as agreed.`,
          );
        await reviseButton.click();
        await idle(2);
        const revised = await api.get<Plan>(
          `/api/issues/${issue!.id}/documents/plan`,
        );
        expect(revised.body).toContain(marker);
        // A revision-history section may quote the superseded requirement.
        // The executed output below must use only the accepted requirement.
        expect(revised.latestRevisionId).not.toBe(draft.latestRevisionId);
        acceptedPlan = revised;
        await noTasks();
        const interactions = await api.get<
          Array<{
            id: string;
            status: string;
            kind: string;
            payload?: {
              target?: { revisionId?: string };
              acceptLabel?: string;
            };
          }>
        >(`/api/issues/${issue!.id}/interactions`);
        const approval = interactions.find(
          (row) =>
            row.status === "pending" &&
            row.kind === "request_confirmation" &&
            row.payload?.target?.revisionId === revised.latestRevisionId,
        );
        expect(approval, "approval targets the revised plan").toBeTruthy();
        await input.capture(
          "chat-plan-revised",
          "Revised plan before handoff",
          "chat-plan-revised.png",
        );
        await page
          .getByRole("button", {
            name: approval!.payload?.acceptLabel ?? "Approve",
            exact: true,
          })
          .last()
          .click();
        await idle(4);
        await input.evidence("chat-plan-revisions.json", {
          draft,
          revised,
          approval,
          source: await api.get(`/api/issues/${issue!.id}/documents/plan`),
        });
      } else {
        repositoryCatalogBefore = await api.get(
          `/api/companies/${f.company.id}/project-repositories`,
        );
        expect(
          repositoryCatalogBefore!.repositories.filter((repository) =>
            expectedRepositoryUrls.includes(repository.url),
          ),
        ).toHaveLength(0);
        await turn(
          `Create a project called Repository Discussion ${nonce} for work spanning https://github.com/octocat/Hello-World and https://github.com/octocat/Spoon-Knife. These existing public repositories are not in our catalog; register both URLs. Then make one assigned task for yourself to write a two-sentence description of the intended project as a Paperclip document attached to that execution task, containing ${marker}, and complete that task. No code changes or remote repository creation are needed.`,
          2,
        );
      }
      const children = await tasks();
      expect(children).toHaveLength(1);
      const child = children[0]!;
      await pollUntil({
        label: `execution task ${child.id} completes`,
        deadlineAt: Date.now() + 240_000,
        intervalMs: 1000,
        load: async () => ({
          task: await api.get<ChatIssue>(`/api/issues/${child.id}`),
          runs: await api.get<ChatRun[]>(`/api/issues/${child.id}/runs`),
        }),
        accept: (state) => state.task.status === "done",
        reject: (state) => chatTaskCompletionFailure(state.task, state.runs),
      });
      runs = await allRuns();
      input.observe(issue!, runs);
      const plan =
        caseId === "plan-handoff"
          ? await api.get<Plan>(`/api/issues/${child.id}/documents/plan`)
          : null;
      const taskRuns = runs.filter(
        (run) => run.contextSnapshot?.issueId === child.id,
      );
      if (plan) {
        assertChatHandoff(child, plan, taskRuns, issue!);
        expect(plan.body).toContain(marker);
        const sourcePlan = await api.get<Plan>(
          `/api/issues/${issue!.id}/documents/plan`,
        );
        expect(sourcePlan.body).toBe(acceptedPlan!.body);
        expect(sourcePlan.latestRevisionId).toBe(
          acceptedPlan!.latestRevisionId,
        );
      } else assertChatTaskHandoff(child, taskRuns, issue!);
      const output = await readChatOutputDocument(api, child.id, marker);
      assertChatExecutionOutput(
        output.body,
        marker,
        caseId === "plan-handoff" ? draftMarker : undefined,
      );
      await input.evidence("chat-execution-output.json", {
        taskId: child.id,
        document: output,
        revisions: await api.get(
          `/api/issues/${child.id}/documents/${encodeURIComponent(output.key)}/revisions`,
        ),
        executionRunIds: taskRuns.map((run) => run.id),
      });
      expect(
        (await comments())
          .filter((c) => c.authorAgentId)
          .map((c) => c.body)
          .join("\n"),
      ).toMatch(new RegExp(`${child.id}|${child.identifier}`));
      const projects = await api.get<
        Array<{
          id: string;
          name: string;
          workspaces: Array<{ id: string; name: string; repoUrl?: string }>;
        }>
      >(`/api/companies/${f.company.id}/projects`);
      expect(projects).toHaveLength(1);
      if (existingProject) {
        expect(child.projectId).toBe(existingProject.id);
        await expect(
          page.getByRole("article", { name: /Project created:/ }),
        ).toHaveCount(0);
      } else {
        await expect(
          page.getByRole("article", { name: /Project created:/ }),
        ).toHaveCount(1);
        if (caseId === "multi-repository") {
          expect(projects[0]!.workspaces.map((w) => w.repoUrl).sort()).toEqual(
            [...expectedRepositoryUrls].sort(),
          );
          // URL registration is persisted as project workspaces; the discovery
          // catalog continues to reflect authorized GitHub connections only.
          expect(
            projects[0]!.workspaces.every((workspace) => Boolean(workspace.id)),
          ).toBe(true);
          expect(
            new Set(projects[0]!.workspaces.map((workspace) => workspace.id))
              .size,
          ).toBe(2);
          const persistedProject = await api.get<{
            workspaces: Array<{ id: string; repoUrl?: string }>;
          }>(`/api/projects/${projects[0]!.id}`);
          expect(
            persistedProject.workspaces.map(({ id, repoUrl }) => ({
              id,
              repoUrl,
            })),
          ).toEqual(
            projects[0]!.workspaces.map(({ id, repoUrl }) => ({ id, repoUrl })),
          );
          await input.evidence("chat-repository-registration.json", {
            catalogBefore: repositoryCatalogBefore,
            projectId: projects[0]!.id,
            registeredWorkspaces: persistedProject.workspaces,
          });
          const projectCard = page.getByRole("article", {
            name: /Project created:/,
          });
          // Repository labels may be customized; verify the actual destinations.
          for (const repositoryUrl of expectedRepositoryUrls) {
            await expect(
              projectCard.locator(`a[href="${repositoryUrl}"]`),
            ).toBeVisible();
          }
        } else
          expect(projects[0]!.workspaces.filter((w) => w.repoUrl)).toHaveLength(
            0,
          );
        await page.reload({ waitUntil: "domcontentloaded", timeout: 60_000 });
        await expect(
          page.getByRole("article", { name: /Project created:/ }),
        ).toHaveCount(1);
      }
      await input.evidence("chat-handoff.json", {
        source: issue!,
        task: child,
        plan,
        output,
        projects,
      });
    }
    await idle(minimumRunCount(execution.task));
    if (execution.suite.id === "hiring-templates") {
      // Refresh the whole consistent evidence generation for the final guard.
      hiringEvidence = await refreshHiringEvidence!();
      runs = hiringEvidence.runs;
    }
    assertChatFlowRunCount({ suiteId: execution.suite.id, task: execution.task,
      // Hiring must account for every actual company run, including unexpected resets.
      runs: execution.suite.id === "hiring-templates" ? runs : runs.filter((run) => !isResetRun(run)),
      hiringEvidence, hiringApiState: execution.suite.id === "hiring-templates"
        ? hiringEvidence!.turnApiState : undefined,
    });
    for (const run of runs.filter((run) => !isResetRun(run))) {
      expect(run.runtimeMode).toBe(execution.profile.expectedRuntimeMode);
      expect(run.status).toBe(
        expectedStops.get(run.id) ?? (stopCase && run.status === "cancelled"
          ? "cancelled"
          : "succeeded"),
      );
    }
    await input.evidence("api-state.json", {
      issue: issue!,
      runs,
      runGroups: {
        resets: runs.filter(isResetRun).map((run) => run.id),
        cancelled: runs
          .filter((run) => run.status === "cancelled")
          .map((run) => run.id),
        conversation: runs
          .filter(
            (run) =>
              !isResetRun(run) && run.contextSnapshot?.issueId === issue!.id,
          )
          .map((run) => run.id),
        handoff: runs
          .filter((run) => run.contextSnapshot?.issueId !== issue!.id)
          .map((run) => run.id),
      },
      comments: await comments(),
      activity: await api.get(`/api/issues/${issue!.id}/activity`),
      runEvidence: await Promise.all(
        runs.map((run) => collectChatRunEvidence(api, run)),
      ),
    });
    await input.capture(
      "final-state",
      "Chat waiting after its verified workflow",
      "final-state.png",
    );
    return { issue: issue!, runs };
  } finally {
    await api.patch("/api/instance/settings/experimental", {
      enableAgentChat: settings.enableAgentChat,
      enableClassicTaskInterface: settings.enableClassicTaskInterface,
    });
  }
}