PaperClipAI/tests/runner-e2e/chat-flow.test.ts

import { describe, it, expect, vi } from "vitest";
import type { AskUserQuestionsPayload } from "../../packages/shared/src/types/issue.js";
import {
  assertChatBacklogCreation,
  assertChatHandoff,
  assertChatReassignment,
  assertChatTaskHandoff,
  chatQuestionPresentation,
  chatRunFailure,
  chatTaskCompletionFailure,
  createChatIdleFailureDetector,
  collectChatRunEvidence,
  readRunningChatLog,
  readChatOutputDocument,
  isChatClarificationReply,
  assertChatExecutionOutput,
  isResetRun,
  type ChatIssue,
  type ChatRun,
} from "./chat-flow.js";
import type { RunnerApi } from "./api.js";
import { chatMarker } from "./chat-cases.js";
import { runnerMatrix } from "./catalog.js";
import { isPublicRunnerScreenshotRoute } from "./screenshot-policy.js";
import { classifyFailure, shouldRetryFailure } from "./failure-classifier.js";

const source: ChatIssue = {
  id: "chat",
  companyId: "co",
  title: "Chat",
  status: "in_review",
  assigneeAgentId: "agent",
};
const task: ChatIssue = {
  ...source,
  id: "work",
  parentId: null,
  projectId: "project",
};
const plan = {
  body: "# Relevant plan",
  latestRevisionId: "revision",
  updatedAt: "2026-09-11T10:00:00Z",
};
const run: ChatRun = {
  id: "run",
  companyId: "co",
  agentId: "agent",
  status: "succeeded",
  startedAt: "2026-09-11T10:00:01Z",
};
describe("chat acceptance contracts", () => {
  it("accepts concrete information requests without requiring question punctuation", () => {
    expect(isChatClarificationReply("What is the club name?")).toBe(true);
    expect(
      isChatClarificationReply(
        "Before assigning the welcome-note work, please share:\n\n1. Club name and intended readers.\n2. Format, length, and tone.\n3. Required details, sender, and deadline.",
      ),
    ).toBe(true);
    expect(
      isChatClarificationReply("Tell me the intended audience and format."),
    ).toBe(true);
    expect(
      isChatClarificationReply(
        "Thanks — before assigning the welcome-note drafting work, I need a compact brief covering:\n\n- Club and audience: club name and intended readers.\n- Purpose: welcome or next steps.\n- Required content: dates, links, and contacts.\n- Voice: tone and sender.\n- Delivery constraints: format, length, and deadline.\n- Examples or policies: existing notes and approval requirements.",
      ),
    ).toBe(true);
    expect(isChatClarificationReply("I'll need your details about the audience and format.")).toBe(true);
    expect(isChatClarificationReply("We need some information about the club and intended readers.")).toBe(true);
    expect(isChatClarificationReply("I needed a compact brief before I assigned the work.")).toBe(false);
    expect(isChatClarificationReply("I need a compact brief:")).toBe(false);
    expect(isChatClarificationReply("I need information.")).toBe(false);
    expect(isChatClarificationReply("I need to create the task and write the note.")).toBe(false);
    expect(isChatClarificationReply("Please share:")).toBe(false);
    expect(isChatClarificationReply("Please share.")).toBe(false);
    expect(
      isChatClarificationReply("Asked the user clarifying questions about their club."),
    ).toBe(false);
    expect(
      isChatClarificationReply("I created the task and started writing the welcome note."),
    ).toBe(false);
  });

  it("recognizes the retained clarification list without treating empty requests or execution plans as clarification", () => {
    // Reduced public reply from gha-36307603818-1 / agent-chat.legacy-codex.local.clarify-reuse.
    expect(isChatClarificationReply(`Absolutely. Before I assign the writing, I need:

- Club name and what the club is about
- Who the note welcomes (new members, event guests, website visitors, etc.)
- Where the note will appear and your preferred length
- Tone (warm, formal, playful, energetic, or another style)
- Any must-include details: benefits, meeting information, values, next step, or signatory

If you only have the club name, audience, and tone, that is enough to begin; I can make sensible assumptions about the rest.`)).toBe(true);
    expect(isChatClarificationReply("We need:\n1. Intended readers\n2. Delivery date")).toBe(true);
    for (const body of ["I need:", "I need:\n- .\n- ...", "I needed:\n- Audience\n- Tone",
      "I need:\n- to create the task\n- to write the note", "I need to create the task and write the note.",
      "I need:\n- Create the task\n- Write the note", "We need:\n1. **Build** the page\n2. **Publish** the site",
      "I need:\n- Plan the note\n- Draft it", "I need:\n- [ ] Create task\n- [ ] Write note"]) {
      expect(isChatClarificationReply(body), body).toBe(false);
    }
  });

  it("checks the requested information list separately from a later plan", () => {
    expect(isChatClarificationReply("I need:\n- Audience\n- Tone\n\nOnce you answer, the plan is:\n- Create the task\n- Write the note")).toBe(true);
  });

  it("requires information fields or questions rather than arbitrary non-blacklisted bullets", () => {
    expect(isChatClarificationReply("I need:\n- Set up the project\n- Install dependencies")).toBe(false);
    expect(isChatClarificationReply("I need:\n- Gather the audience details\n- Decide the tone")).toBe(false);
    expect(isChatClarificationReply("I need:\n- Audience\n- Install dependencies")).toBe(false);
    expect(isChatClarificationReply("I need:\n- **Audience**: intended readers\n\n- [ ] Preferred format\n- When this is due")).toBe(true);
  });

  it("accepts logistical brief fields and wrapped information items", () => {
    expect(isChatClarificationReply("I need:\n- Budget\n- Location")).toBe(true);
    expect(isChatClarificationReply("I need:\n- Audience, including age range\nand whether these are new or existing members\n- Tone\n\nOnce you answer, I can:\n- Draft the note\n- Publish it")).toBe(true);
    expect(isChatClarificationReply("I need:\n- Audience, including age range\n  and whether these are new or existing members\n- Tone")).toBe(true);
  });

  it("rejects superseded plan requirements in executed output, independently of plan history", () => {
    expect(() => assertChatExecutionOutput("Welcome CHAT123.", "CHAT123", "DRAFT123")).not.toThrow();
    expect(() => assertChatExecutionOutput("Welcome DRAFT123 and CHAT123.", "CHAT123", "DRAFT123")).toThrow();
    expect(() => assertChatExecutionOutput("Welcome DRAFT123.", "CHAT123", "DRAFT123")).toThrow();
  });

  it("keeps chat markers literal across rich-text and Markdown boundaries", () => {
    for (const prefix of ["CHAT", "DRAFT", "OLDCONTEXT"] as const) {
      expect(chatMarker(prefix, "abc123-1")).toBe(`${prefix}abc1231`);
      expect(chatMarker(prefix, "abc_123-1")).toMatch(/^[a-zA-Z0-9]+$/);
    }
    expect(chatMarker("OLDCONTEXT", "one-1")).not.toBe(
      chatMarker("CHAT", "one-1"),
    );
    expect(chatMarker("CHAT", "one-1")).not.toBe(chatMarker("CHAT", "two-1"));
  });
  it("covers existing workflows and native reassignment on the chosen local profiles", () => {
    const matrix = runnerMatrix.filter(
      (cell) => cell.suite.id === "agent-chat",
    );
    expect(matrix).toHaveLength(28);
    expect(new Set(matrix.map((cell) => cell.profile.id))).toEqual(
      new Set([
        "legacy-codex",
        "legacy-claude",
        "runner-codex",
        "runner-acpx-claude",
      ]),
    );
    expect(new Set(matrix.map((cell) => cell.task.id)).size).toBe(8);
    expect(
      matrix.every(
        (cell) =>
          cell.environment.id === "local" &&
          cell.task.expectedTerminalState.issue === "in_review",
      ),
    ).toBe(true);
  });
  it("rejects missing plan, chat children, wrong assignments, and execution before the plan", () => {
    expect(() => assertChatHandoff(task, plan, [run], source)).not.toThrow();
    for (const invalid of [
      { ...task, parentId: "chat" },
      { ...task, projectId: null },
      { ...task, assigneeAgentId: "other" },
    ])
      expect(() => assertChatHandoff(invalid, plan, [run], source)).toThrow();
    expect(() =>
      assertChatHandoff(task, { ...plan, body: "" }, [run], source),
    ).toThrow();
    expect(() =>
      assertChatHandoff(
        task,
        { ...plan, updatedAt: "2026-09-11T10:00:02Z" },
        [run],
        source,
      ),
    ).toThrow();
    expect(() => assertChatHandoff(task, plan, [], source)).toThrow();
  });
  it("requires a plan for plan handoff, while direct requests need only normal task assignment", () => {
    expect(() => assertChatTaskHandoff(task, [run], source)).not.toThrow();
    expect(() =>
      assertChatHandoff(task, { ...plan, body: "" }, [run], source),
    ).toThrow();
    expect(() =>
      assertChatTaskHandoff({ ...task, projectId: null }, [run], source),
    ).toThrow();
  });
  it("still rejects completed chat work that has lost its assignee", () => {
    expect(() => assertChatTaskHandoff({ ...task, status: "done" }, [run], source)).not.toThrow();
    expect(() => assertChatTaskHandoff({ ...task, status: "done", assigneeAgentId: null }, [run], source)).toThrow();
  });
  it.each(["project-description", "welcome-note", "output"])("finds committed %s output without accepting a copied plan or a claim", async (key) => {
    const output = {
      ...plan,
      id: "description-doc",
      issueId: "work",
      key,
      body: "A completed description with CHAT123.",
      createdByAgentId: "agent",
    };
    const get = vi.fn(async (path: string) => {
      if (path === "/api/issues/work/documents")
        return [{ key: "plan" }, { key }];
      if (path === `/api/issues/work/documents/${key}`)
        return output;
      throw new Error(`Unexpected document read: ${path}`);
    });
    const api = { get } as Pick<RunnerApi, "get">;
    await expect(readChatOutputDocument(api, "work", "CHAT123")).resolves.toBe(
      output,
    );
    await expect(readChatOutputDocument(api, "work", "WRONG123")).rejects.toThrow(
      "no non-plan output document",
    );
    get.mockImplementation(async () => [{ key: "plan" }]);
    await expect(readChatOutputDocument(api, "work", "CHAT123")).rejects.toThrow(
      "document keys: plan",
    );
  });
  it("uses durable free-text labels, multi-selection, and the supplied submit label", () => {
    const payload: AskUserQuestionsPayload = {
      version: 1,
      submitLabel: "Send brief",
      questions: [
        {
          id: "audience",
          prompt: "Who is it for?",
          selectionMode: "multi",
          required: true,
          options: [
            { id: "members", label: "New members" },
            {
              id: "custom",
              label: "Another audience or occasion",
              freeText: true,
            },
          ],
        },
      ],
    };
    const presentation = chatQuestionPresentation(payload);
    expect(presentation.submitLabel).toBe("Send brief");
    expect(presentation.questions[0]).toMatchObject({
      answerMode: "multi_select",
      customAnswer: { enabled: true, label: "Another audience or occasion" },
    });
    const nativePayload: AskUserQuestionsPayload = {
      ...payload,
      questionSet: {
        schema: "paperclip.question_set.v1",
        submitLabel: "Continue",
        questions: [
          {
            id: "audience",
            prompt: "Who is it for?",
            required: true,
            answerMode: "text",
          },
        ],
      },
    };
    expect(chatQuestionPresentation(nativePayload)).toBe(
      nativePayload.questionSet,
    );
  });
  it("retains reset events without requesting a provider log, and does not hide missing real logs", async () => {
    const get = vi.fn().mockResolvedValue([{ seq: 1, type: "session_reset" }]);
    const reset = { ...run, resultJson: { conversationReset: true } };
    await expect(collectChatRunEvidence({ get }, reset)).resolves.toEqual({
      runId: run.id,
      log: null,
      events: [{ seq: 1, type: "session_reset" }],
    });
    expect(get.mock.calls).toEqual([
      [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],
    ]);
    get.mockRejectedValue(new Error("Run log not found"));
    await expect(collectChatRunEvidence({ get }, run)).rejects.toThrow(
      "Run log not found",
    );
  });
  it("retains the tail of a long chat run for invariant checks", async () => {
    const firstPage = Array.from({ length: 1000 }, (_, i) => ({ seq: i + 1, eventType: "item.delta" }));
    const tail = { seq: 1001, eventType: "run.terminal" };
    const get = vi.fn().mockResolvedValueOnce({ content: "fixture log" })
      .mockResolvedValueOnce(firstPage).mockResolvedValueOnce([tail]);
    const evidence = await collectChatRunEvidence({ get }, run);
    expect(evidence.events).toEqual([...firstPage, tail]);
    expect(get.mock.calls).toEqual([
      [`/api/heartbeat-runs/${run.id}/log?limitBytes=1048576`],
      [`/api/heartbeat-runs/${run.id}/events?afterSeq=0&limit=1000`],
      [`/api/heartbeat-runs/${run.id}/events?afterSeq=1000&limit=1000`],
    ]);
  });
  it("records legitimate pre-provider log absence only with explicit bootstrap and storage evidence", async () => {
    const stopped = { ...run, status: "interrupted", runtimeMode: "legacy", runtimeModeResolvedAt: null, logStore: null, logRef: null,
      resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: false } } };
    const get = vi.fn().mockResolvedValue([]);
    expect(await collectChatRunEvidence({ get }, stopped)).toMatchObject({ log: null, logOmissionReason: "provider_not_started", events: [] });
    expect(get).toHaveBeenCalledTimes(1);
    get.mockRejectedValue(new Error("Run log not found"));
    for (const change of [{ resultJson: {} }, { runtimeMode: "native" }, { logRef: "recorded-log" }, { logStore: undefined },
      { runtimeModeResolvedAt: "2026-09-28T00:00:00Z" }, { resultJson: { executionRecovery: { kind: "bootstrap", providerWorkStarted: true } } }]) {
      await expect(collectChatRunEvidence({ get }, { ...stopped, ...change })).rejects.toThrow("Run log not found");
    }
  });
  it("retains events for an unstarted dependency-blocked wake without asking for a nonexistent log", async () => {
    const get = vi.fn().mockResolvedValue([]);
    const suppressed = { ...run, status: "cancelled", errorCode: "issue_dependencies_blocked", startedAt: null };
    expect((await collectChatRunEvidence({ get }, suppressed)).log).toBeNull();
    expect(get).toHaveBeenCalledTimes(1);
    get.mockRejectedValue(new Error("Run log not found"));
    await expect(collectChatRunEvidence({ get }, { ...suppressed, startedAt: "2026-09-18T00:00:00Z" })).rejects.toThrow("Run log not found");
    await expect(collectChatRunEvidence({ get }, { ...suppressed, errorCode: "provider_transport_failed" })).rejects.toThrow("Run log not found");
  });
  it("waits for a newly running provider's log file without swallowing server failures", async () => {
    const get = vi.fn().mockResolvedValue({ status: () => 404 });
    const api = { request: { get } } as unknown as Pick<RunnerApi, "request">;
    await expect(readRunningChatLog(api, "starting")).resolves.toBeUndefined();
    get.mockResolvedValue({
      status: () => 200,
      ok: () => true,
      json: async () => ({ content: "streamed reply" }),
    });
    await expect(readRunningChatLog(api, "running")).resolves.toBe(
      "streamed reply",
    );
    get.mockResolvedValue({ status: () => 500, ok: () => false });
    await expect(readRunningChatLog(api, "broken")).rejects.toThrow(
      "log returned 500",
    );
  });
  it("fails terminal execution errors without preempting active retries or recovery", () => {
    const failed = {
      ...run,
      status: "failed",
      error: "provider rejected request",
    };
    expect(chatTaskCompletionFailure(task, [failed])).toContain(
      "provider rejected request",
    );
    expect(
      chatTaskCompletionFailure(task, [failed, { ...run, status: "queued" }]),
    ).toBeUndefined();
    expect(
      chatTaskCompletionFailure({ ...task, scheduledRetry: { id: "retry" } }, [
        failed,
      ]),
    ).toBeUndefined();
    expect(
      chatTaskCompletionFailure(
        { ...task, activeRecoveryAction: { id: "recovery" } },
        [failed],
      ),
    ).toBeUndefined();
  });
  it("fails stable contradictory idle states promptly without paid retries or transient false alarms", () => {
    const detect = createChatIdleFailureDetector(3);
    const settled = {
      resolved: true,
      status: "blocked",
      conversationState: "waiting",
      providerRunCount: 3,
      activeRuns: [] as string[],
    };
    expect(detect(settled)).toBeUndefined();
    expect(detect({ ...settled, activeRuns: ["running"] })).toBeUndefined();
    expect(detect(settled)).toBeUndefined();
    const failure = detect(settled);
    expect(failure).toContain("chat_idle_state_invariant");
    expect(classifyFailure(failure)).toBe("candidate_failure");
    expect(shouldRetryFailure(classifyFailure(failure))).toBe(false);
    expect(detect({ ...settled, status: "in_review" })).toBeUndefined();
    expect(detect(settled)).toBeUndefined();
    expect(detect({ ...settled, providerRunCount: 2 })).toBeUndefined();
    expect(detect({ ...settled, status: "in_progress" })).toBeUndefined();
    expect(detect(settled)).toBeUndefined();
  });
  it("fails promptly on terminal provider failures while permitting only expected cancellations", () => {
    expect(chatRunFailure([run])).toBeUndefined();
    expect(chatRunFailure([{ ...run, status: "running" }])).toBeUndefined();
    expect(
      chatRunFailure([
        {
          ...run,
          status: "failed",
          errorCode: "permission_denied",
          error: "sandbox unavailable",
        },
      ]),
    ).toContain("run run failed (permission_denied): sandbox unavailable");
    expect(chatRunFailure([{ ...run, status: "cancelled" }])).toContain(
      "cancelled",
    );
    expect(
      chatRunFailure([{ ...run, status: "cancelled" }], true),
    ).toBeUndefined();
  });
  it("separates reset control runs from provider runs without treating failures as resets", () => {
    expect(isResetRun(run)).toBe(false);
    expect(isResetRun({ ...run, status: "failed" })).toBe(false);
    expect(
      isResetRun({ ...run, contextSnapshot: { conversationReset: true } }),
    ).toBe(true);
  });
  it("only publishes screenshots of the exact disposable chat", () => {
    const target = {
      issuePrefix: "E2E",
      issueId: "chat",
      issueIdentifier: null,
      chatAgentId: "fixture-agent",
    };
    expect(
      isPublicRunnerScreenshotRoute(
        "http://127.0.0.1:3199/E2E/chats/fixture-agent",
        target,
      ),
    ).toBe(true);
    expect(
      isPublicRunnerScreenshotRoute(
        "http://127.0.0.1:3199/E2E/chats/another-agent",
        target,
      ),
    ).toBe(false);
    expect(
      isPublicRunnerScreenshotRoute(
        "https://example.com/E2E/chats/fixture-agent",
        target,
      ),
    ).toBe(false);
  });
});


describe("reassignment outcome oracle", () => {
  const evidence = () => ({ readyId: "ready", queuedId: "queued", teammateId: "riley",
    tasks: [{ id: "ready", companyId: "co", title: "Ready", status: "done", assigneeAgentId: "riley" }, { id: "queued", companyId: "co", title: "Later", status: "backlog", assigneeAgentId: "riley" }],
    runs: [{ id: "successor", companyId: "co", agentId: "riley", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "ready" } }],
    audit: [{ action: "issue.reassigned", details: { source: "paperclip_runner_protocol" } }], outputBody: "Launch CHECK123", marker: "CHECK123",
  });
  it("accepts persisted ownership and exactly one successful successor", () => {
    expect(() => assertChatReassignment(evidence())).not.toThrow();
  });
  it.each(["owner", "duplicate", "missing-run", "missing-audit", "backlog-started", "missing-output"])("rejects %s evidence", defect => {
    const data = evidence();
    if (defect === "owner") data.tasks[0]!.assigneeAgentId = "old";
    if (defect === "duplicate") data.tasks.push({ ...data.tasks[0]!, id: "replacement" });
    if (defect === "missing-run") data.runs = [];
    if (defect === "missing-audit") data.audit = [];
    if (defect === "backlog-started") data.runs.push({ ...data.runs[0]!, id: "early", contextSnapshot: { issueId: "queued" } });
    if (defect === "missing-output") data.outputBody = "I reassigned it";
    expect(() => assertChatReassignment(data)).toThrow();
  });
});

describe("backlog creation outcome oracle", () => {
  const evidence = () => ({
    tasks: [{ id: "held", companyId: "co", title: "Later", status: "backlog", parentId: null, assigneeAgentId: "planner" }],
    runs: [] as ChatRun[], ownerId: "planner", marker: "PLAN123",
    plan: { body: "Three steps PLAN123", latestRevisionId: "revision-1", updatedAt: "2026-09-19T00:00:00Z" },
    activity: [{ action: "issue.created", details: { status: "backlog", source: "paperclip_runner_protocol" } }],
  });
  it("accepts one planned backlog task with no execution", () => {
    expect(() => assertChatBacklogCreation(evidence())).not.toThrow();
  });
  it("does not mistake the creating conversation for task execution", () => {
    const data = evidence();
    data.runs.push({ id: "creator", companyId: "co", agentId: "planner", status: "succeeded", contextSnapshot: { issueId: "conversation" } });
    expect(() => assertChatBacklogCreation(data)).not.toThrow();
  });
  it.each(["duplicate", "status", "started-then-stopped", "corrected-after-creation", "owner", "plan"])("rejects %s", defect => {
    const data = evidence();
    if (defect === "duplicate") data.tasks.push({ ...data.tasks[0]!, id: "duplicate" });
    if (defect === "status") data.tasks[0]!.status = "todo";
    if (defect === "started-then-stopped") data.runs.push({ id: "early", companyId: "co", agentId: "planner", status: "cancelled", contextSnapshot: { issueId: "held" } });
    if (defect === "corrected-after-creation") data.activity[0]!.details.status = "todo";
    if (defect === "owner") data.tasks[0]!.assigneeAgentId = "other";
    if (defect === "plan") data.plan.body = "I saved a plan";
    expect(() => assertChatBacklogCreation(data)).toThrow();
  });
});