test: cover chat settings and active-turn follow-ups

Co-Authored-By: Paperclip <noreply@paperclip.ing>

doc/evals.md

@@ -66,6 +66,12 @@ Daytona paths. Its [fixture contract](../tests/runner-e2e/README.md) distinguish
 startup cancellation from active response cancellation and HTTP send replay
 from ambiguous provider action recovery. Select it explicitly; `--all` excludes it.
 
+The explicit-only `agent-chat-stories` suite covers the experimental settings
+lifecycle for a configured native agent and follow-ups during active work. Its
+fixture-driven file wait and persisted-plan oracle are documented in the
+[Product E2E guide](../tests/runner-e2e/README.md). It does not qualify the native
+onboarding wizard or change the native API-tool rollout defaults.
+
 ## Validation ladder
 
 Start with credential-free checks and a catalog listing. For Product E2E:

tests/runner-e2e/FIXTURES.md

@@ -195,3 +195,13 @@ The lost-acknowledgement probe may interrupt only the fixture browser's own
 comment request after the real server has committed it. Retain its request ID
 and replay that same request through the public API after restarting the server.
 Never fabricate tool results or repair task state after a failed assertion.
+
+`chat-stories.ts` uses an ordinary local file wait as a deterministic interruption
+boundary. The real provider command writes the readiness file and waits at most
+two minutes. The harness must persist the next browser message while the same
+run is active before supplying the brief. Always release the wait in `finally`.
+Save boundary observations independently of the final outcome. The final answer
+must recover a brief reference absent from both prompts; the revision oracle
+also reads the actual conversation plan. Fixture setup never enables native API
+tools for this suite. Do not describe its prepared-agent settings case as a
+production onboarding qualification.

tests/runner-e2e/README.md

@@ -188,6 +188,26 @@ requires the original comment, task, plan, and single consuming run. This proves
 HTTP request idempotency across restart, not replay safety for an ambiguous
 provider tool response. Existing native tool-receipt tests cover that boundary.
 
+`agent-chat-stories` adds six explicit-only local cells across native Codex and
+Claude. `enable-disable-resume` uses the Experimental settings UI to enable
+Agent Chat, starts a conversation, disables new messages, verifies the public
+write endpoint rejects a send without creating work, and re-enables the same
+conversation with its remembered context. The company, credential, and native
+agent are fixture-provisioned. This qualifies the experimental-settings path,
+not native first-run onboarding: the current production wizard offers legacy
+adapters, and native API tools remain an independent opt-in.
+
+`followup-while-running` and `revise-while-running` send a second browser message
+while the provider runs a bounded command waiting for a fixture brief file.
+The command publishes its own readiness file; the harness verifies the original
+run is still active after the follow-up is saved, then supplies the brief.
+The final reply must contain the previously undisclosed brief reference and the
+new request's marker. The revision case also checks the saved plan uses Friday
+instead of the original Monday. The oracle permits either steering the active
+run or one queued successor, but rejects missing/duplicate comments, failed or
+unfinished runs, stale plan contents, and unintended tasks/projects. This does
+not qualify active-task reassignment or worker-crash recovery.
+
 ```sh
 pnpm test:e2e:runner -- --list --suite agent-chat-hardening
 pnpm test:e2e:runner -- --id agent-chat-hardening.runner-codex.local.stop-startup-new-resume

tests/runner-e2e/catalog.test.ts

@@ -71,10 +71,10 @@ describe("runner E2E catalog", () => {
     expect(localIntegrityTasks).toHaveLength(2);
     expect(openRouterBreadthTasks).toHaveLength(3);
     expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([
-      23, 38, 52, 28, 18, 42, 14, 10, 2,
+      23, 38, 52, 28, 18, 6, 42, 14, 10, 2,
     ]);
-    expect(validateRunnerCatalog()).toHaveLength(227);
-    expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(227);
+    expect(validateRunnerCatalog()).toHaveLength(233);
+    expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(233);
     expect(
       runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"),
     ).toHaveLength(42);

tests/runner-e2e/catalog.ts

@@ -2,7 +2,7 @@ import { continuationTasks } from "./continuation-cases.js";
 import { everydayTasks, productionStoryProfile } from "./everyday-cases.js";
 
 import { firstTaskTasks } from "./first-task-cases.js";
-import { chatTasks, chatHardeningTasks } from "./chat-cases.js";
+import { chatTasks, chatHardeningTasks, chatStoryTasks } from "./chat-cases.js";
 import { createHash } from "node:crypto";
 import { createAgentSchema } from "../../packages/shared/src/validators/agent.js";
 import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js";
@@ -959,6 +959,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [
       ["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)),
     definitionMetadata: { version: 5, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count", readOnlyState: "public-mutation-contract-and-relations", hiringReference: "neutral-document-reference-line" },
   },
+  {
+    id: "agent-chat-stories", label: "Agent Chat Setup and Interruptions", manualOnly: true,
+    description: "Experimental settings lifecycle and user follow-ups during active native work.",
+    groups: ["chat", "native"],
+    profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id))
+      .map(profile => productionStoryProfile(defaultPermissionProfile(profile))),
+    environments: [localEnvironment], tasks: chatStoryTasks, expectedMatrixSize: 6,
+    definitionMetadata: { version: 1, setup: "configured-native-agent", permissions: "production-defaults", interruptionBoundary: "provider-file-wait-active", grading: "persisted-comments-and-plan", scheduling: "explicit-only" },
+  },
   ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []),
   {
     id: "core-compatibility",

tests/runner-e2e/chat-cases.ts

@@ -47,6 +47,11 @@ function buildChatTasks(definitions: readonly (readonly [string, string, number]
 }
 export const chatTasks = buildChatTasks(CHAT_CASES);
 export const chatHardeningTasks = buildChatTasks(HARDENING_CASES);
+export const chatStoryTasks = buildChatTasks([
+  ["enable-disable-resume", "Enable Agent Chat, pause access, and resume preserved history", 2],
+  ["followup-while-running", "Deliver a follow-up while a provider turn is running", 2],
+  ["revise-while-running", "Change instructions during active work and save the updated plan", 2],
+]);
 
 export function chatNeedsApiTools(suiteId: string, caseId: string): boolean {
   return suiteId === "agent-chat-hardening" && ["hire-delegate-reuse", "blocked-status-review"].includes(caseId);

tests/runner-e2e/chat-flow.ts

@@ -9,6 +9,7 @@ import type { MatrixExecution } from "./types.js";
 import { isBlockedUnstartedWake } from "./non-execution-wake.js";
 import { chatMarker } from "./chat-cases.js";
 import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js";
+import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js";
 
 // Public API observations only: this driver never fabricates provider results or writes DB state.
 export interface ChatIssue {
@@ -368,7 +369,8 @@ export async function runChatFlow(input: ChatFlowInput) {
   };
   const noTasks = async () => expect(await tasks()).toHaveLength(0);
   try {
-    await api.patch("/api/instance/settings/experimental", {
+    if (caseId === "enable-disable-resume") await enableChatThroughSettings(input);
+    else await api.patch("/api/instance/settings/experimental", {
       enableAgentChat: true,
       enableClassicTaskInterface: false,
     });
@@ -380,7 +382,12 @@ export async function runChatFlow(input: ChatFlowInput) {
     expect(await api.get(chatPath)).toBeNull();
     expect(await allRuns()).toHaveLength(0);
 
-    if (
+    if (caseId === "enable-disable-resume") {
+      await runChatSettingsLifecycle({ input, marker, issue: () => issue!, idle, allRuns, comments });
+    } else if (["followup-while-running", "revise-while-running"].includes(caseId)) {
+      await runChatInterruption({ input, marker, issue: () => issue!, idle, allRuns, comments,
+        refreshIssue: async () => { issue = await api.get<ChatIssue>(chatPath); input.observe(issue, await allRuns()); } });
+    } else if (
       ["continuity-restart", "new-session", "stop-new-resume", "stop-startup-new-resume"].includes(caseId)
     ) {
       const secret = chatMarker("OLDCONTEXT", nonce);

tests/runner-e2e/chat-stories.test.ts

@@ -0,0 +1,51 @@
+import { describe, expect, it } from "vitest";
+import { assertInterruptedChat } from "./chat-stories.js";
+import { runnerMatrix } from "./catalog.js";
+import { buildRunnerE2EProcessEnvironment } from "./harness-env.js";
+
+const run = { id: "run", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "chat" } };
+const valid = {
+  first: "Read the brief", followup: "Change the launch day", reference: "BRIEF123", marker: "UPDATED123", issueId: "chat",
+  boundaryRun: { ...run, status: "running" }, activeAtFollowup: { ...run, status: "running" },
+  comments: [{ id: "first", body: "Read the brief" }, { id: "followup", body: "Change the launch day" },
+    { id: "answer", authorAgentId: "agent", body: "BRIEF123 UPDATED123" }],
+  runs: [run], revisedPlan: JSON.stringify({ launchDay: "Friday", reference: "BRIEF123", revision: "UPDATED123" }),
+};
+
+describe("active chat follow-up oracle", () => {
+  it("accepts either steering or one queued successor, with the saved correction", () => {
+    expect(() => assertInterruptedChat(valid)).not.toThrow();
+    expect(() => assertInterruptedChat({ ...valid, runs: [run, { ...run, id: "successor" }] })).not.toThrow();
+  });
+  it("rejects follow-ups sent after the active boundary", () => {
+    expect(() => assertInterruptedChat({ ...valid, boundaryRun: run })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, activeAtFollowup: run })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, activeAtFollowup: { ...valid.activeAtFollowup, id: "other" } })).toThrow();
+  });
+  it("rejects lost or duplicated input, missing file evidence, and stale plans", () => {
+    expect(() => assertInterruptedChat({ ...valid, comments: valid.comments.filter(c => c.id !== "followup") })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, comments: [...valid.comments, { id: "duplicate", body: valid.followup }] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, comments: [...valid.comments.slice(0, 2), { id: "answer", authorAgentId: "agent", body: "UPDATED123" }] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, revisedPlan: valid.revisedPlan.replace("Friday", "Monday") })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, revisedPlan: "The updated plan is saved." })).toThrow();
+  });
+  it("rejects failed, unfinished, duplicated, or unrelated execution", () => {
+    for (const status of ["running", "queued", "failed", "cancelled"])
+      expect(() => assertInterruptedChat({ ...valid, runs: [{ ...run, status }] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, runs: [] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, runs: [run, run, run] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, runs: [run, run] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, runs: [{ ...run, id: "other" }] })).toThrow();
+    expect(() => assertInterruptedChat({ ...valid, runs: [{ ...run, contextSnapshot: { issueId: "other" } }] })).toThrow();
+  });
+  it("keeps setup and interruption stories native, local, opt-in, and outside API-tool overrides", () => {
+    const cells = runnerMatrix.filter(cell => cell.suite.id === "agent-chat-stories");
+    expect(cells).toHaveLength(6);
+    for (const cell of cells) {
+      expect(cell.suite.manualOnly).toBe(true);
+      expect(cell.profile.generation).toBe("native");
+      expect(cell.environment.id).toBe("local");
+      expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_API_TOOLS_ENABLED).toBeUndefined();
+    }
+  });
+});

tests/runner-e2e/chat-stories.ts

@@ -0,0 +1,125 @@
+import { expect } from "@playwright/test";
+import { readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { randomUUID } from "node:crypto";
+import { sendChatMessage, type ChatFlowInput, type ChatIssue, type ChatRun } from "./chat-flow.js";
+
+type Comment = { id: string; body: string; authorAgentId?: string; createdByRunId?: string };
+type Context = {
+  input: ChatFlowInput; marker: string; issue(): ChatIssue;
+  idle(count: number): Promise<void>; allRuns(): Promise<ChatRun[]>; comments(): Promise<Comment[]>;
+};
+
+/** Seed only the disabled setting; the browser performs the user's opt-in. */
+export async function enableChatThroughSettings(input: ChatFlowInput) {
+  await input.api.patch("/api/instance/settings/experimental", { enableAgentChat: false, enableClassicTaskInterface: false });
+  await setChatEnabled(input, true);
+}
+
+async function setChatEnabled(input: ChatFlowInput, enabled: boolean) {
+  await input.page.goto(`/${input.fixtures.company.issuePrefix}/company/settings/instance/experimental`, { waitUntil: "domcontentloaded" });
+  const toggle = input.page.getByRole("switch", { name: "Toggle agent chat experimental setting" });
+  await expect(toggle).toBeVisible();
+  await toggle.setChecked(enabled);
+  await expect.poll(async () => (await input.api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental")).enableAgentChat).toBe(enabled);
+}
+
+export function assertInterruptedChat(input: {
+  first: string; followup: string; reference: string; marker: string; issueId: string;
+  boundaryRun: ChatRun; activeAtFollowup: ChatRun; comments: Comment[]; runs: ChatRun[];
+  revisedPlan?: string;
+}) {
+  expect(input.boundaryRun.status).toBe("running");
+  expect(input.activeAtFollowup).toMatchObject({ id: input.boundaryRun.id, status: "running" });
+  for (const message of [input.first, input.followup]) {
+    expect(input.comments.filter(comment => !comment.authorAgentId && comment.body === message)).toHaveLength(1);
+  }
+  const replies = input.comments.filter(comment => comment.authorAgentId);
+  const final = replies.at(-1)!;
+  expect(final).toBeTruthy();
+  expect(final.body).toContain(input.reference);
+  expect(final.body).toContain(input.marker);
+  expect(input.runs.length).toBeGreaterThanOrEqual(1);
+  expect(input.runs.length).toBeLessThanOrEqual(2);
+  expect(input.runs.every(run => run.status === "succeeded" && run.runtimeMode === "native")).toBe(true);
+  expect(new Set(input.runs.map(run => run.id)).size).toBe(input.runs.length);
+  expect(input.runs.every(run => run.agentId === input.boundaryRun.agentId && run.contextSnapshot?.issueId === input.issueId)).toBe(true);
+  expect(final.authorAgentId).toBe(input.boundaryRun.agentId);
+  expect(input.runs.some(run => run.id === input.boundaryRun.id)).toBe(true);
+  if (input.revisedPlan !== undefined) {
+    const body = input.revisedPlan.trim().replace(/^```(?:json)?\s*/, "").replace(/\s*```$/, "");
+    expect(JSON.parse(body)).toEqual({ launchDay: "Friday", reference: input.reference, revision: input.marker });
+  }
+}
+
+export async function runChatInterruption(context: Context & { refreshIssue(): Promise<void> }) {
+  const { input, marker, idle, allRuns, comments } = context;
+  const gate = path.join(input.workspacePath, `chat-brief-${input.nonce}.txt`);
+  const ready = `${gate}.waiting`;
+  const reference = `BRIEF${randomUUID().replaceAll("-", "")}`;
+  // Real provider tool execution waits on an ordinary fixture file. No runner
+  // hooks, provider results, database records, or task outcomes are fabricated.
+  const script = `const fs=require("node:fs");fs.writeFileSync(${JSON.stringify(ready)},"waiting");const end=Date.now()+120000;const timer=setInterval(()=>{if(fs.existsSync(${JSON.stringify(gate)})){console.log(fs.readFileSync(${JSON.stringify(gate)},"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`;
+  const command = `node -e '${script.replaceAll("'", "'\\''")}'`;
+  const first = `Our launch is planned for Monday. First run this bounded command to wait for the brief reference file I am supplying: ${command}\nAfter it returns, acknowledge the reference from the file here. This is discussion only; do not create projects or tasks.`;
+  const revise = input.execution.task.id === "revise-while-running";
+  const followup = revise
+    ? `Change the launch day to Friday. Save the plan on this conversation as JSON with exactly launchDay, reference (from the brief file), and revision ("${marker}"). Then reply here with the brief reference and ${marker}. Do not create projects or execution tasks.`
+    : `After reading the brief, reply here with its reference and ${marker}. Keep this in the current conversation; do not create projects or tasks.`;
+  let boundaryRun: ChatRun | undefined;
+  let activeAtFollowup: ChatRun | undefined;
+  try {
+    await sendChatMessage(input.page, first);
+    await expect.poll(async () => {
+      if (await readFile(ready, "utf8").catch(() => "") !== "waiting") return false;
+      boundaryRun = (await allRuns()).find(run => run.status === "running");
+      return Boolean(boundaryRun);
+    }, { timeout: 120_000 }).toBe(true);
+    await context.refreshIssue();
+    await sendChatMessage(input.page, followup);
+    activeAtFollowup = await input.api.get<ChatRun>(`/api/heartbeat-runs/${boundaryRun!.id}`);
+    await input.evidence("chat-interruption-boundary.json", { first, followup, boundaryRun, activeAtFollowup, comments: await comments() });
+    expect(activeAtFollowup.status, "Follow-up must persist while the first provider run is active").toBe("running");
+    await writeFile(gate, reference, "utf8");
+    await expect.poll(async () => (await comments()).filter(comment => comment.authorAgentId).at(-1)?.body ?? "", { timeout: 240_000 }).toContain(marker);
+    await idle(1); // Providers may queue a new run or steer the current run.
+    const plan = revise ? await input.api.get<{ body: string }>(`/api/issues/${context.issue().id}/documents/plan`) : undefined;
+    const evidence = { first, followup, reference, marker, issueId: context.issue().id, boundaryRun: boundaryRun!, activeAtFollowup,
+      comments: await comments(), runs: await allRuns(), ...(plan ? { revisedPlan: plan.body } : {}) };
+    await input.evidence("chat-interruption.json", evidence);
+    assertInterruptedChat(evidence);
+    expect(await input.api.get(`/api/companies/${input.fixtures.company.id}/issues`)).toEqual([]);
+    expect(await input.api.get(`/api/companies/${input.fixtures.company.id}/projects`)).toEqual([]);
+  } finally {
+    // Always release a provider that is still waiting, including failed UI runs.
+    await writeFile(gate, reference, "utf8");
+  }
+}
+
+export async function runChatSettingsLifecycle(context: Context) {
+  const { input, idle, comments, allRuns, marker } = context;
+  const remembered = `REMEMBER${marker}`;
+  const route = `/${input.fixtures.company.issuePrefix}/chats/${input.fixtures.agent.id}`;
+  await sendChatMessage(input.page, `Remember ${remembered} in this conversation. Acknowledge only; no tasks or projects.`);
+  await idle(1);
+  const before = { issue: context.issue(), comments: await comments(), runs: await allRuns() };
+  await setChatEnabled(input, false);
+  await input.page.goto(route, { waitUntil: "domcontentloaded" });
+  await expect(input.page.getByText(/Agent Chat is disabled/)).toBeVisible();
+  const blocked = await input.api.request.post(`/api/issues/${before.issue.id}/comments`, { data: { body: `DISABLED${marker}` } });
+  expect(blocked.status()).toBe(404);
+  expect(await comments()).toEqual(before.comments);
+  expect((await allRuns()).map(run => ({ id: run.id, status: run.status })))
+    .toEqual(before.runs.map(run => ({ id: run.id, status: run.status })));
+  await setChatEnabled(input, true);
+  await input.page.goto(route, { waitUntil: "domcontentloaded" });
+  await sendChatMessage(input.page, `What reference did I ask you to remember? Reply with it and ${marker}. No new work.`);
+  await idle(2);
+  expect(context.issue().id).toBe(before.issue.id);
+  expect(context.issue().conversationSessionGeneration).toBe(before.issue.conversationSessionGeneration);
+  const reply = (await comments()).filter(comment => comment.authorAgentId).at(-1)?.body ?? "";
+  expect(reply).toContain(remembered);
+  expect(reply).toContain(marker);
+  expect(await input.api.get(`/api/companies/${input.fixtures.company.id}/issues`)).toEqual([]);
+  await input.evidence("chat-settings-lifecycle.json", { before, after: { issue: context.issue(), comments: await comments(), runs: await allRuns() } });
+}