From d28750acf4dcbc2c06957d77ee29db80f17b8777 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 21 Sep 2026 14:17:25 -0500 Subject: [PATCH] test: cover chat settings and active-turn follow-ups Co-Authored-By: Paperclip --- doc/evals.md | 6 ++ tests/runner-e2e/FIXTURES.md | 10 +++ tests/runner-e2e/README.md | 20 +++++ tests/runner-e2e/catalog.test.ts | 6 +- tests/runner-e2e/catalog.ts | 11 ++- tests/runner-e2e/chat-cases.ts | 5 ++ tests/runner-e2e/chat-flow.ts | 11 ++- tests/runner-e2e/chat-stories.test.ts | 51 +++++++++++ tests/runner-e2e/chat-stories.ts | 125 ++++++++++++++++++++++++++ 9 files changed, 239 insertions(+), 6 deletions(-) create mode 100644 tests/runner-e2e/chat-stories.test.ts create mode 100644 tests/runner-e2e/chat-stories.ts diff --git a/doc/evals.md b/doc/evals.md index 842422bf1c7..45a196b3461 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -66,6 +66,12 @@ Daytona paths. Its [fixture contract](../tests/runner-e2e/README.md) distinguish startup cancellation from active response cancellation and HTTP send replay from ambiguous provider action recovery. Select it explicitly; `--all` excludes it. +The explicit-only `agent-chat-stories` suite covers the experimental settings +lifecycle for a configured native agent and follow-ups during active work. Its +fixture-driven file wait and persisted-plan oracle are documented in the +[Product E2E guide](../tests/runner-e2e/README.md). It does not qualify the native +onboarding wizard or change the native API-tool rollout defaults. + ## Validation ladder Start with credential-free checks and a catalog listing. For Product E2E: diff --git a/tests/runner-e2e/FIXTURES.md b/tests/runner-e2e/FIXTURES.md index 9b1acf83510..b1c6d96254d 100644 --- a/tests/runner-e2e/FIXTURES.md +++ b/tests/runner-e2e/FIXTURES.md @@ -195,3 +195,13 @@ The lost-acknowledgement probe may interrupt only the fixture browser's own comment request after the real server has committed it. Retain its request ID and replay that same request through the public API after restarting the server. Never fabricate tool results or repair task state after a failed assertion. + +`chat-stories.ts` uses an ordinary local file wait as a deterministic interruption +boundary. The real provider command writes the readiness file and waits at most +two minutes. The harness must persist the next browser message while the same +run is active before supplying the brief. Always release the wait in `finally`. +Save boundary observations independently of the final outcome. The final answer +must recover a brief reference absent from both prompts; the revision oracle +also reads the actual conversation plan. Fixture setup never enables native API +tools for this suite. Do not describe its prepared-agent settings case as a +production onboarding qualification. diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index c8006b22e25..e37229a11ed 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -188,6 +188,26 @@ requires the original comment, task, plan, and single consuming run. This proves HTTP request idempotency across restart, not replay safety for an ambiguous provider tool response. Existing native tool-receipt tests cover that boundary. +`agent-chat-stories` adds six explicit-only local cells across native Codex and +Claude. `enable-disable-resume` uses the Experimental settings UI to enable +Agent Chat, starts a conversation, disables new messages, verifies the public +write endpoint rejects a send without creating work, and re-enables the same +conversation with its remembered context. The company, credential, and native +agent are fixture-provisioned. This qualifies the experimental-settings path, +not native first-run onboarding: the current production wizard offers legacy +adapters, and native API tools remain an independent opt-in. + +`followup-while-running` and `revise-while-running` send a second browser message +while the provider runs a bounded command waiting for a fixture brief file. +The command publishes its own readiness file; the harness verifies the original +run is still active after the follow-up is saved, then supplies the brief. +The final reply must contain the previously undisclosed brief reference and the +new request's marker. The revision case also checks the saved plan uses Friday +instead of the original Monday. The oracle permits either steering the active +run or one queued successor, but rejects missing/duplicate comments, failed or +unfinished runs, stale plan contents, and unintended tasks/projects. This does +not qualify active-task reassignment or worker-crash recovery. + ```sh pnpm test:e2e:runner -- --list --suite agent-chat-hardening pnpm test:e2e:runner -- --id agent-chat-hardening.runner-codex.local.stop-startup-new-resume diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 027ab20201d..1d5e571af9f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -71,10 +71,10 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 23, 38, 52, 28, 18, 42, 14, 10, 2, + 23, 38, 52, 28, 18, 6, 42, 14, 10, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(227); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(227); + expect(validateRunnerCatalog()).toHaveLength(233); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(233); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(42); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 11833435c62..3d1613e3cab 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -2,7 +2,7 @@ import { continuationTasks } from "./continuation-cases.js"; import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; import { firstTaskTasks } from "./first-task-cases.js"; -import { chatTasks, chatHardeningTasks } from "./chat-cases.js"; +import { chatTasks, chatHardeningTasks, chatStoryTasks } from "./chat-cases.js"; import { createHash } from "node:crypto"; import { createAgentSchema } from "../../packages/shared/src/validators/agent.js"; import { createEnvironmentSchema } from "../../packages/shared/src/validators/environment.js"; @@ -959,6 +959,15 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ ["stop-startup-new-resume", "hire-delegate-reuse", "blocked-status-review"].map(task => `agent-chat-hardening.${profile}.daytona.${task}`)), definitionMetadata: { version: 5, permissions: "production-defaults", instructions: "production", grading: "durable-state-and-source-evidence", scheduling: "explicit-only", restartMemory: "required-after-restart", statusEvidence: "structured-current-blocker-and-active-run-count", readOnlyState: "public-mutation-contract-and-relations", hiringReference: "neutral-document-reference-line" }, }, + { + id: "agent-chat-stories", label: "Agent Chat Setup and Interruptions", manualOnly: true, + description: "Experimental settings lifecycle and user follow-ups during active native work.", + groups: ["chat", "native"], + profiles: runnerProfiles.filter(profile => ["runner-codex", "runner-acpx-claude"].includes(profile.id)) + .map(profile => productionStoryProfile(defaultPermissionProfile(profile))), + environments: [localEnvironment], tasks: chatStoryTasks, expectedMatrixSize: 6, + definitionMetadata: { version: 1, setup: "configured-native-agent", permissions: "production-defaults", interruptionBoundary: "provider-file-wait-active", grading: "persisted-comments-and-plan", scheduling: "explicit-only" }, + }, ...(process.env.PAPERCLIP_RUNNER_E2E_CONNECTION_REVIEWS === "1" ? [connectionReviewSuite] : []), { id: "core-compatibility", diff --git a/tests/runner-e2e/chat-cases.ts b/tests/runner-e2e/chat-cases.ts index c0e59620e08..edead566804 100644 --- a/tests/runner-e2e/chat-cases.ts +++ b/tests/runner-e2e/chat-cases.ts @@ -47,6 +47,11 @@ function buildChatTasks(definitions: readonly (readonly [string, string, number] } export const chatTasks = buildChatTasks(CHAT_CASES); export const chatHardeningTasks = buildChatTasks(HARDENING_CASES); +export const chatStoryTasks = buildChatTasks([ + ["enable-disable-resume", "Enable Agent Chat, pause access, and resume preserved history", 2], + ["followup-while-running", "Deliver a follow-up while a provider turn is running", 2], + ["revise-while-running", "Change instructions during active work and save the updated plan", 2], +]); export function chatNeedsApiTools(suiteId: string, caseId: string): boolean { return suiteId === "agent-chat-hardening" && ["hire-delegate-reuse", "blocked-status-review"].includes(caseId); diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index 99fd8dfa606..0fda3d862cc 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -9,6 +9,7 @@ import type { MatrixExecution } from "./types.js"; import { isBlockedUnstartedWake } from "./non-execution-wake.js"; import { chatMarker } from "./chat-cases.js"; import { assertChatRememberedAfterRestart, assertChatStartupStopped, isChatStopReady, runChatHardeningFlow } from "./chat-hardening.js"; +import { enableChatThroughSettings, runChatInterruption, runChatSettingsLifecycle } from "./chat-stories.js"; // Public API observations only: this driver never fabricates provider results or writes DB state. export interface ChatIssue { @@ -368,7 +369,8 @@ export async function runChatFlow(input: ChatFlowInput) { }; const noTasks = async () => expect(await tasks()).toHaveLength(0); try { - await api.patch("/api/instance/settings/experimental", { + if (caseId === "enable-disable-resume") await enableChatThroughSettings(input); + else await api.patch("/api/instance/settings/experimental", { enableAgentChat: true, enableClassicTaskInterface: false, }); @@ -380,7 +382,12 @@ export async function runChatFlow(input: ChatFlowInput) { expect(await api.get(chatPath)).toBeNull(); expect(await allRuns()).toHaveLength(0); - if ( + if (caseId === "enable-disable-resume") { + await runChatSettingsLifecycle({ input, marker, issue: () => issue!, idle, allRuns, comments }); + } else if (["followup-while-running", "revise-while-running"].includes(caseId)) { + await runChatInterruption({ input, marker, issue: () => issue!, idle, allRuns, comments, + refreshIssue: async () => { issue = await api.get(chatPath); input.observe(issue, await allRuns()); } }); + } else if ( ["continuity-restart", "new-session", "stop-new-resume", "stop-startup-new-resume"].includes(caseId) ) { const secret = chatMarker("OLDCONTEXT", nonce); diff --git a/tests/runner-e2e/chat-stories.test.ts b/tests/runner-e2e/chat-stories.test.ts new file mode 100644 index 00000000000..4386ffea39e --- /dev/null +++ b/tests/runner-e2e/chat-stories.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, it } from "vitest"; +import { assertInterruptedChat } from "./chat-stories.js"; +import { runnerMatrix } from "./catalog.js"; +import { buildRunnerE2EProcessEnvironment } from "./harness-env.js"; + +const run = { id: "run", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native", contextSnapshot: { issueId: "chat" } }; +const valid = { + first: "Read the brief", followup: "Change the launch day", reference: "BRIEF123", marker: "UPDATED123", issueId: "chat", + boundaryRun: { ...run, status: "running" }, activeAtFollowup: { ...run, status: "running" }, + comments: [{ id: "first", body: "Read the brief" }, { id: "followup", body: "Change the launch day" }, + { id: "answer", authorAgentId: "agent", body: "BRIEF123 UPDATED123" }], + runs: [run], revisedPlan: JSON.stringify({ launchDay: "Friday", reference: "BRIEF123", revision: "UPDATED123" }), +}; + +describe("active chat follow-up oracle", () => { + it("accepts either steering or one queued successor, with the saved correction", () => { + expect(() => assertInterruptedChat(valid)).not.toThrow(); + expect(() => assertInterruptedChat({ ...valid, runs: [run, { ...run, id: "successor" }] })).not.toThrow(); + }); + it("rejects follow-ups sent after the active boundary", () => { + expect(() => assertInterruptedChat({ ...valid, boundaryRun: run })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, activeAtFollowup: run })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, activeAtFollowup: { ...valid.activeAtFollowup, id: "other" } })).toThrow(); + }); + it("rejects lost or duplicated input, missing file evidence, and stale plans", () => { + expect(() => assertInterruptedChat({ ...valid, comments: valid.comments.filter(c => c.id !== "followup") })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, comments: [...valid.comments, { id: "duplicate", body: valid.followup }] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, comments: [...valid.comments.slice(0, 2), { id: "answer", authorAgentId: "agent", body: "UPDATED123" }] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, revisedPlan: valid.revisedPlan.replace("Friday", "Monday") })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, revisedPlan: "The updated plan is saved." })).toThrow(); + }); + it("rejects failed, unfinished, duplicated, or unrelated execution", () => { + for (const status of ["running", "queued", "failed", "cancelled"]) + expect(() => assertInterruptedChat({ ...valid, runs: [{ ...run, status }] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, runs: [] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, runs: [run, run, run] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, runs: [run, run] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, runs: [{ ...run, id: "other" }] })).toThrow(); + expect(() => assertInterruptedChat({ ...valid, runs: [{ ...run, contextSnapshot: { issueId: "other" } }] })).toThrow(); + }); + it("keeps setup and interruption stories native, local, opt-in, and outside API-tool overrides", () => { + const cells = runnerMatrix.filter(cell => cell.suite.id === "agent-chat-stories"); + expect(cells).toHaveLength(6); + for (const cell of cells) { + expect(cell.suite.manualOnly).toBe(true); + expect(cell.profile.generation).toBe("native"); + expect(cell.environment.id).toBe("local"); + expect(buildRunnerE2EProcessEnvironment({}, [cell]).PAPERCLIP_RUNNER_API_TOOLS_ENABLED).toBeUndefined(); + } + }); +}); diff --git a/tests/runner-e2e/chat-stories.ts b/tests/runner-e2e/chat-stories.ts new file mode 100644 index 00000000000..6b88389b858 --- /dev/null +++ b/tests/runner-e2e/chat-stories.ts @@ -0,0 +1,125 @@ +import { expect } from "@playwright/test"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { sendChatMessage, type ChatFlowInput, type ChatIssue, type ChatRun } from "./chat-flow.js"; + +type Comment = { id: string; body: string; authorAgentId?: string; createdByRunId?: string }; +type Context = { + input: ChatFlowInput; marker: string; issue(): ChatIssue; + idle(count: number): Promise; allRuns(): Promise; comments(): Promise; +}; + +/** Seed only the disabled setting; the browser performs the user's opt-in. */ +export async function enableChatThroughSettings(input: ChatFlowInput) { + await input.api.patch("/api/instance/settings/experimental", { enableAgentChat: false, enableClassicTaskInterface: false }); + await setChatEnabled(input, true); +} + +async function setChatEnabled(input: ChatFlowInput, enabled: boolean) { + await input.page.goto(`/${input.fixtures.company.issuePrefix}/company/settings/instance/experimental`, { waitUntil: "domcontentloaded" }); + const toggle = input.page.getByRole("switch", { name: "Toggle agent chat experimental setting" }); + await expect(toggle).toBeVisible(); + await toggle.setChecked(enabled); + await expect.poll(async () => (await input.api.get<{ enableAgentChat: boolean }>("/api/instance/settings/experimental")).enableAgentChat).toBe(enabled); +} + +export function assertInterruptedChat(input: { + first: string; followup: string; reference: string; marker: string; issueId: string; + boundaryRun: ChatRun; activeAtFollowup: ChatRun; comments: Comment[]; runs: ChatRun[]; + revisedPlan?: string; +}) { + expect(input.boundaryRun.status).toBe("running"); + expect(input.activeAtFollowup).toMatchObject({ id: input.boundaryRun.id, status: "running" }); + for (const message of [input.first, input.followup]) { + expect(input.comments.filter(comment => !comment.authorAgentId && comment.body === message)).toHaveLength(1); + } + const replies = input.comments.filter(comment => comment.authorAgentId); + const final = replies.at(-1)!; + expect(final).toBeTruthy(); + expect(final.body).toContain(input.reference); + expect(final.body).toContain(input.marker); + expect(input.runs.length).toBeGreaterThanOrEqual(1); + expect(input.runs.length).toBeLessThanOrEqual(2); + expect(input.runs.every(run => run.status === "succeeded" && run.runtimeMode === "native")).toBe(true); + expect(new Set(input.runs.map(run => run.id)).size).toBe(input.runs.length); + expect(input.runs.every(run => run.agentId === input.boundaryRun.agentId && run.contextSnapshot?.issueId === input.issueId)).toBe(true); + expect(final.authorAgentId).toBe(input.boundaryRun.agentId); + expect(input.runs.some(run => run.id === input.boundaryRun.id)).toBe(true); + if (input.revisedPlan !== undefined) { + const body = input.revisedPlan.trim().replace(/^```(?:json)?\s*/, "").replace(/\s*```$/, ""); + expect(JSON.parse(body)).toEqual({ launchDay: "Friday", reference: input.reference, revision: input.marker }); + } +} + +export async function runChatInterruption(context: Context & { refreshIssue(): Promise }) { + const { input, marker, idle, allRuns, comments } = context; + const gate = path.join(input.workspacePath, `chat-brief-${input.nonce}.txt`); + const ready = `${gate}.waiting`; + const reference = `BRIEF${randomUUID().replaceAll("-", "")}`; + // Real provider tool execution waits on an ordinary fixture file. No runner + // hooks, provider results, database records, or task outcomes are fabricated. + const script = `const fs=require("node:fs");fs.writeFileSync(${JSON.stringify(ready)},"waiting");const end=Date.now()+120000;const timer=setInterval(()=>{if(fs.existsSync(${JSON.stringify(gate)})){console.log(fs.readFileSync(${JSON.stringify(gate)},"utf8"));clearInterval(timer)}else if(Date.now()>end){clearInterval(timer);process.exitCode=1}},100);`; + const command = `node -e '${script.replaceAll("'", "'\\''")}'`; + const first = `Our launch is planned for Monday. First run this bounded command to wait for the brief reference file I am supplying: ${command}\nAfter it returns, acknowledge the reference from the file here. This is discussion only; do not create projects or tasks.`; + const revise = input.execution.task.id === "revise-while-running"; + const followup = revise + ? `Change the launch day to Friday. Save the plan on this conversation as JSON with exactly launchDay, reference (from the brief file), and revision ("${marker}"). Then reply here with the brief reference and ${marker}. Do not create projects or execution tasks.` + : `After reading the brief, reply here with its reference and ${marker}. Keep this in the current conversation; do not create projects or tasks.`; + let boundaryRun: ChatRun | undefined; + let activeAtFollowup: ChatRun | undefined; + try { + await sendChatMessage(input.page, first); + await expect.poll(async () => { + if (await readFile(ready, "utf8").catch(() => "") !== "waiting") return false; + boundaryRun = (await allRuns()).find(run => run.status === "running"); + return Boolean(boundaryRun); + }, { timeout: 120_000 }).toBe(true); + await context.refreshIssue(); + await sendChatMessage(input.page, followup); + activeAtFollowup = await input.api.get(`/api/heartbeat-runs/${boundaryRun!.id}`); + await input.evidence("chat-interruption-boundary.json", { first, followup, boundaryRun, activeAtFollowup, comments: await comments() }); + expect(activeAtFollowup.status, "Follow-up must persist while the first provider run is active").toBe("running"); + await writeFile(gate, reference, "utf8"); + await expect.poll(async () => (await comments()).filter(comment => comment.authorAgentId).at(-1)?.body ?? "", { timeout: 240_000 }).toContain(marker); + await idle(1); // Providers may queue a new run or steer the current run. + const plan = revise ? await input.api.get<{ body: string }>(`/api/issues/${context.issue().id}/documents/plan`) : undefined; + const evidence = { first, followup, reference, marker, issueId: context.issue().id, boundaryRun: boundaryRun!, activeAtFollowup, + comments: await comments(), runs: await allRuns(), ...(plan ? { revisedPlan: plan.body } : {}) }; + await input.evidence("chat-interruption.json", evidence); + assertInterruptedChat(evidence); + expect(await input.api.get(`/api/companies/${input.fixtures.company.id}/issues`)).toEqual([]); + expect(await input.api.get(`/api/companies/${input.fixtures.company.id}/projects`)).toEqual([]); + } finally { + // Always release a provider that is still waiting, including failed UI runs. + await writeFile(gate, reference, "utf8"); + } +} + +export async function runChatSettingsLifecycle(context: Context) { + const { input, idle, comments, allRuns, marker } = context; + const remembered = `REMEMBER${marker}`; + const route = `/${input.fixtures.company.issuePrefix}/chats/${input.fixtures.agent.id}`; + await sendChatMessage(input.page, `Remember ${remembered} in this conversation. Acknowledge only; no tasks or projects.`); + await idle(1); + const before = { issue: context.issue(), comments: await comments(), runs: await allRuns() }; + await setChatEnabled(input, false); + await input.page.goto(route, { waitUntil: "domcontentloaded" }); + await expect(input.page.getByText(/Agent Chat is disabled/)).toBeVisible(); + const blocked = await input.api.request.post(`/api/issues/${before.issue.id}/comments`, { data: { body: `DISABLED${marker}` } }); + expect(blocked.status()).toBe(404); + expect(await comments()).toEqual(before.comments); + expect((await allRuns()).map(run => ({ id: run.id, status: run.status }))) + .toEqual(before.runs.map(run => ({ id: run.id, status: run.status }))); + await setChatEnabled(input, true); + await input.page.goto(route, { waitUntil: "domcontentloaded" }); + await sendChatMessage(input.page, `What reference did I ask you to remember? Reply with it and ${marker}. No new work.`); + await idle(2); + expect(context.issue().id).toBe(before.issue.id); + expect(context.issue().conversationSessionGeneration).toBe(before.issue.conversationSessionGeneration); + const reply = (await comments()).filter(comment => comment.authorAgentId).at(-1)?.body ?? ""; + expect(reply).toContain(remembered); + expect(reply).toContain(marker); + expect(await input.api.get(`/api/companies/${input.fixtures.company.id}/issues`)).toEqual([]); + await input.evidence("chat-settings-lifecycle.json", { before, after: { issue: context.issue(), comments: await comments(), runs: await allRuns() } }); +}