From 5f45ed9fd65d4b6bb18a2e78081a433927c82c09 Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 26 Oct 2026 17:43:52 -0500 Subject: [PATCH] test(e2e): prove native pause and controller continuity boundaries --- .../runtime-agent-lifecycle.test.ts | 21 +++++ .../runtime-agent-lifecycle.ts | 63 +++++++++++++ .../runtime-busy-worker.ts | 33 +++++++ .../runtime-capability-audit.json | 4 +- .../runtime-recovery.test.ts | 26 ++++++ .../chat-catalog-domains/runtime-recovery.ts | 91 +++++++++++++++++++ 6 files changed, 236 insertions(+), 2 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-recovery.test.ts create mode 100644 tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts new file mode 100644 index 00000000000..66e23e98074 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.test.ts @@ -0,0 +1,21 @@ +import { it, expect } from "vitest"; +import { gradePauseAgent } from "./runtime-agent-lifecycle.js"; +function valid() { + return { scenarioId: "agents.pause-agent", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { worker: { id: "morgan" }, workerRunId: "worker-run", workerWasReallyWaiting: true, beforeHumanStatus: "running", agentPauseStatus: 403, humanPaused: true }, + agents: [{ id: "morgan", status: "paused", lifecycleState: "paused" }], tasks: [], links: ["/ACORN/agents/morgan"], + reply: "Open Morgan's profile and use the Pause control. Existing running work stops after cleanup; the pause remains until resumed.", + activity: [{ id: "pause", action: "agent.paused", entityId: "morgan", actorType: "user" }], + runs: [{ id: "worker-run", status: "cancelled" }], humanQueries: [] }; +} +it("accepts an actual human pause with stopped worker evidence", () => expect(gradePauseAgent(valid()).passed).toBe(true)); +it("rejects missing pause evidence", () => expect(gradePauseAgent(undefined).passed).toBe(false)); +it("rejects acknowledgement without a saved human pause", () => { + const o = valid(); o.state.humanPaused = false; expect(gradePauseAgent(o).passed).toBe(false); +}); +it("rejects a paused label while the old process still runs", () => { + const o = valid(); o.runs[0]!.status = "running"; expect(gradePauseAgent(o).passed).toBe(false); +}); +it("rejects an agent-authored pause attribution", () => { + const o = valid(); o.activity[0]!.actorType = "agent"; expect(gradePauseAgent(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts new file mode 100644 index 00000000000..dfca55d5712 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-agent-lifecycle.ts @@ -0,0 +1,63 @@ +import { expect } from "@playwright/test"; +import { pollUntil } from "../api.js"; +import { checks } from "./internal-support.js"; +import { probeAgentPermission } from "./identity-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime, agentLink, type RuntimeAgent } from "./runtime-support.js"; +import { busyRuntimeWorker, releaseRuntimeWorker } from "./runtime-busy-worker.js"; +export function gradePauseAgent(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("actual-human-pause-evidence", !!o, "Actual active work, board control, saved pause and terminal cancellation are independently required."); if (!o) return c.result(); + const worker = o.state.worker as RuntimeAgent; const current = o.agents.find(row => row.id === worker.id); + const stopped = o.runs.find(run => run.id === o.state.workerRunId); + c.add("real-active-work-before-human-pause", o.state.workerWasReallyWaiting === true && o.state.beforeHumanStatus !== "paused" + && o.state.agentPauseStatus === 403, "The agent's acknowledgement and bearer cannot complete the board-only pause while Morgan is really working."); + c.add("human-pause-persisted-and-settled", o.state.humanPaused === true && current?.status === "paused" + && current.lifecycleState === "paused" && stopped?.status === "cancelled", + "The real profile Pause control persists suspension and confirms the original process stopped."); + c.add("human-attribution", o.activity.some(row => row.action === "agent.paused" && row.entityId === worker.id + && row.actorType === "user" && !row.agentId), "The pause audit belongs to the authorized human."); + c.add("pending-human-step-honest", /profile|pause button|pause control|you can pause|you.*pause/i.test(o.reply) + && /stop|settle|current|existing|running/i.test(o.reply) && !/I(?:'ve| have)? (?:now )?paused/i.test(o.reply), + "The reply identifies the exact human step and explains how existing execution settles without falsely claiming the pause."); + c.add("exact-agent-result-link", !!current && !!agentLink(o, current), "The guidance links Morgan's real saved profile."); + return c.result(); +} +export const runtimeLifecycleCases = [runtimeDefinition("agents.pause-agent", busyRuntimeWorker, gradePauseAgent, { + sourceFiles: ["chat-catalog-domains/runtime-agent-lifecycle.ts", "chat-catalog-domains/runtime-busy-worker.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + expectedRunCount: 2, attemptTimeoutMs: 600_000, + setupPolicy: { readiness: "preserve-scenario", invariantCheckIds: ["lead-ready", "real-busy-worker", "worker-task-scope"] }, + async verifySetup(ctx, state) { + const c = checks(); const worker = state.worker as RuntimeAgent; + const lead = await ctx.input.api.get(`/api/agents/${ctx.input.fixtures.agent.id}`); + const saved = await ctx.input.api.get(`/api/agents/${worker.id}`); + const runs = await ctx.allRuns(); const run = runs.find(row => row.id === state.workerRunId); + c.add("lead-ready", lead.lifecycleState === "ready" && ["idle", "active"].includes(lead.status ?? ""), "The actual CEO is ready for the canonical query."); + c.add("real-busy-worker", saved.lifecycleState === "ready" && ["running", "active"].includes(saved.status ?? "") + && run?.status === "running" && state.workerWasReallyWaiting === true, "Morgan remains genuinely active rather than being forced idle by shared setup."); + const task = state.task as { id: string }; + c.add("worker-task-scope", run?.agentId === worker.id && run.companyId === ctx.input.fixtures.company.id + && [run.contextSnapshot?.issueId, (run as typeof run & { nativeIssueId?: string })?.nativeIssueId].includes(task.id), "The actual native worker run belongs to this exact company and Website task."); + return c.result(); + }, + runPolicy: { kind: "terminal-counts", terminalStatusCounts: { succeeded: 1, cancelled: 1 } }, + async act(ctx, state) { + await pollUntil({ label: "Chat guidance while Morgan is still working", deadlineAt: Date.now() + 150_000, + load: async () => { await ctx.refreshIssue(); return { source: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: value => value.source?.conversationState === "waiting" && value.runs.some(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded") + && value.runs.find(run => run.id === state.workerRunId)?.status === "running" }); + const worker = state.worker as RuntimeAgent; + state.beforeHumanStatus = (await ctx.input.api.get(`/api/agents/${worker.id}`)).status; + state.agentPauseStatus = await probeAgentPermission(ctx, `/api/agents/${worker.id}/pause`, "POST", {}); + const o = await observeRuntime(ctx, state); const href = agentLink(o, worker); if (!href) throw new Error("Pause guidance lacks Morgan's actual profile"); + const source = ctx.input.page.url(); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); + await ctx.input.page.getByRole("button", { name: "Pause", exact: true }).click(); + await pollUntil({ label: "Morgan pause confirmed after native process cleanup", deadlineAt: Date.now() + 90_000, + load: async () => ({ worker: await ctx.input.api.get(`/api/agents/${worker.id}`), runs: await ctx.allRuns() }), + accept: value => value.worker.status === "paused" && value.worker.lifecycleState === "paused" && value.runs.find(run => run.id === state.workerRunId)?.status === "cancelled" }); + state.humanPaused = true; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); + }, + cleanup: async (_ctx, state) => releaseRuntimeWorker(state), +})]; diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts new file mode 100644 index 00000000000..0eb02619b65 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-busy-worker.ts @@ -0,0 +1,33 @@ +import path from "node:path"; +import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js"; +import { prepareChatBrief } from "../chat-stories.js"; +import { pollUntil } from "../api.js"; +import { runtimeWorker } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { Entity } from "./internal-support.js"; +export async function busyRuntimeWorker(ctx: ChatCatalogContext) { + if (ctx.input.execution.environment.id !== "local") throw new Error("Owned file-gate fixture requires a real local environment"); + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Busy fixture requires its isolated attempt root"); + const worker = await runtimeWorker(ctx); const candidate = path.resolve(resolveDefaultAgentWorkspaceDir(worker.id)); + const isolated = await realpath(root); + if (!candidate.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture escaped its isolated company home"); + await mkdir(candidate, { recursive: true }); + const home = await realpath(candidate); + if (!home.startsWith(`${isolated}${path.sep}`)) throw new Error("Busy fixture resolved outside its isolated company home"); + const gate = await prepareChatBrief(home, `runtime-${ctx.input.nonce}`, 300_000); + const company = `/api/companies/${ctx.input.fixtures.company.id}`; + for (const scope of [{ scopeType: "company", scopeId: ctx.input.fixtures.company.id }, { scopeType: "agent", scopeId: worker.id }]) + await ctx.input.api.post(`${company}/budgets/policies`, { ...scope, amount: 500, hardStopEnabled: true, isActive: true, unpricedUsagePolicy: "allow" }); + const task = await ctx.input.api.post(`${company}/issues`, { title: "Website task", status: "todo", assigneeAgentId: worker.id, + description: `The website brief is being supplied by the human. Run node ${JSON.stringify(gate.scriptPath)} once and wait for its real output. Do not invent a brief, replace the script, or mark work complete before it returns. Once the brief arrives, save it as a task document and finish normally.` }); + const running = await pollUntil({ label: "Real Morgan process waiting for its human brief", deadlineAt: Date.now() + 150_000, + load: async () => ({ runs: await ctx.allRuns(), waiting: await readFile(gate.ready, "utf8").catch(() => "") }), + accept: value => value.waiting === "waiting" && value.runs.length === 1 && value.runs[0]?.agentId === worker.id && value.runs[0].status === "running", + reject: value => value.runs.some(run => ["failed", "cancelled", "timed_out", "succeeded"].includes(run.status)) ? "Busy fixture ended before its required active boundary" : undefined }); + return { worker, task, gate, workerRunId: running.runs[0]!.id, workerWasReallyWaiting: true }; +} +export async function releaseRuntimeWorker(state: Record) { + const gate = state.gate as { gate?: string } | undefined; + if (gate?.gate) await writeFile(gate.gate, "The human ended this disposable fixture. Preserve any saved work and finish normally.", "utf8"); +} diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json index a44a91ad774..7f93c23949d 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-audit.json @@ -51,7 +51,7 @@ }, { "scenarioId": "agents.pause-agent", - "readinessClass": "local", + "readinessClass": "implemented", "needs": "Genuine active worker; board profile Pause; observe pause state and actual old-run settlement without acknowledgement credit." }, { @@ -171,7 +171,7 @@ }, { "scenarioId": "recovery.controller-restart", - "readinessClass": "setup-turn", + "readinessClass": "implemented", "needs": "Existing input.restart supports action; native saved file/instruction before canonical query requires declared setup-human boundary." }, { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-recovery.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-recovery.test.ts new file mode 100644 index 00000000000..1d066091e6b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-recovery.test.ts @@ -0,0 +1,26 @@ +import { it, expect } from "vitest"; +import { gradeControllerRestart } from "./runtime-recovery.js"; +function valid() { + const initial = { contentVerified: true, content: "

Unpredictable Orchid header

", attachmentId: "initial" }; + const final = { contentVerified: true, content: "

Unpredictable Orchid header

Contact hello@acorn.example
", attachmentId: "final" }; + return { scenarioId: "recovery.controller-restart", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", + state: { initialArtifact: initial, finalArtifact: final, fileBeforeRestart: initial.content, fileAfterRestart: initial.content, restarted: true, + headerMarker: "Unpredictable Orchid header", finalFile: final.content, initialRunId: "first", sourceIssueBefore: "chat", sourceIssueAfter: "chat", priorHistoryRetained: true }, + agents: [], tasks: [], links: ["/api/attachments/final/content?download=1"], reply: "The website is ready.", activity: [], humanQueries: [], + runs: [{ id: "first", agentId: "ceo", status: "succeeded" }, { id: "second", agentId: "ceo", status: "succeeded" }] }; +} +it("accepts real persisted bytes and one attributed continuation", () => expect(gradeControllerRestart(valid()).passed).toBe(true)); +it("rejects missing restart evidence", () => expect(gradeControllerRestart(undefined).passed).toBe(false)); +it("rejects claiming continuation when prior actual files were lost", () => { + const o = valid(); o.state.fileAfterRestart = ""; expect(gradeControllerRestart(o).passed).toBe(false); +}); +it("rejects final work that forgot the persisted contact instruction", () => { + const o = valid(); o.state.finalArtifact.content = o.state.initialArtifact.content; o.state.finalFile = o.state.finalArtifact.content; + expect(gradeControllerRestart(o).passed).toBe(false); +}); +it("rejects source-history replacement", () => { + const o = valid(); o.state.sourceIssueAfter = "new-chat"; expect(gradeControllerRestart(o).passed).toBe(false); +}); +it("rejects a duplicate delegated execution task", () => { + const o = valid(); (o.tasks as Array<{ id: string }>).push({ id: "duplicate-work" }); expect(gradeControllerRestart(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts b/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts new file mode 100644 index 00000000000..7f3e0c6627b --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-recovery.ts @@ -0,0 +1,91 @@ +import { expect } from "@playwright/test"; +import { readFile, realpath } from "node:fs/promises"; +import path from "node:path"; +import { resolveDefaultAgentWorkspaceDir } from "../../../server/src/home-paths.js"; +import { pollUntil } from "../api.js"; +import { sendChatMessage } from "../chat-flow.js"; +import { readRegisteredArtifacts } from "../registered-artifact.js"; +import { checks, type Entity } from "./internal-support.js"; +import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +type Artifact = Awaited>[number]; +async function settled(ctx: ChatCatalogContext, count: number) { + return pollUntil({ label: `Controller continuity phase ${count}`, deadlineAt: Date.now() + 180_000, + load: async () => { await ctx.refreshIssue(); return { issue: ctx.issue(), runs: await ctx.allRuns() }; }, + accept: ({ issue, runs }) => issue?.conversationState === "waiting" && runs.length === count && runs.every(run => run.status === "succeeded"), + reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "A continuity provider turn failed" : undefined }); +} +async function currentFile(ctx: ChatCatalogContext, name: string) { + const root = process.env.PAPERCLIP_RUNNER_E2E_TEMP_ROOT; if (!root) throw new Error("Continuity inspection requires the isolated attempt root"); + const home = await realpath(resolveDefaultAgentWorkspaceDir(ctx.input.fixtures.agent.id)); + const isolated = await realpath(root); + if (!home.startsWith(`${isolated}${path.sep}`) || path.basename(name) !== name) throw new Error("Continuity file escapes its owned fixture directory"); + return readFile(path.join(home, name), "utf8"); +} +export function gradeControllerRestart(value: unknown) { + const c = checks(); const o = runtimeObservation(value); + c.add("actual-restart-continuity-evidence", !!o, "Native prior work, real controller restart, retained bytes and run-attributed final artifact are required."); if (!o) return c.result(); + const before = o.state.initialArtifact as Artifact | undefined; const final = o.state.finalArtifact as Artifact | undefined; + c.add("real-prior-file-survives-restart", !!before?.contentVerified && typeof o.state.fileBeforeRestart === "string" + && o.state.fileBeforeRestart === before.content && o.state.fileAfterRestart === o.state.fileBeforeRestart + && o.state.restarted === true, "The actual provider-written file bytes survive the supported controller restart unchanged."); + c.add("prior-instruction-and-content-retained", !!final?.contentVerified && final.content.includes(String(o.state.headerMarker)) + && final.content.includes("hello@acorn.example") && /contact/i.test(final.content) + && !before?.content.includes("hello@acorn.example") && o.state.finalFile === final.content, + "The final saved website retains the unpredictable original header and follows the prior persisted contact-section instruction."); + c.add("one-eligible-continuation", o.tasks.length === 0 && o.runs.length === 2 && o.runs.every(run => run.agentId === o.leadId && run.status === "succeeded") + && o.runs.filter(run => run.id !== o.state.initialRunId).length === 1, + "One ordinary continuation follows the explicit user request without duplicate execution tasks or replayed side effects."); + c.add("source-and-history-retained", o.state.sourceIssueBefore === o.state.sourceIssueAfter && o.state.priorHistoryRetained === true, + "The same conversation and genuine prior messages survive the controller restart."); + c.add("actual-final-download-link", !!final && o.links.some(href => { + try { const url = new URL(href, o.origin); return url.origin === new URL(o.origin).origin + && url.pathname === `/api/attachments/${final.attachmentId}/content`; } catch { return false; } + }), "The actual final file is available through a rendered production attachment link."); + return c.result(); +} +export const runtimeRecoveryCases = [runtimeDefinition("recovery.controller-restart", async ctx => { + if (ctx.input.execution.environment.id !== "local") throw new Error("Owned-file controller continuity fixture requires the actual local runtime"); + const headerMarker = `Retained Acorn header ${ctx.input.nonce}`; const filename = `website-${ctx.input.nonce}.html`; + await sendChatMessage(ctx.input.page, + `Start a small website draft in ${filename} in your current working directory, with a single header containing exactly ${JSON.stringify(headerMarker)}. Register that real file as a downloadable artifact work product titled ${filename}. This is a draft: do not add the contact section yet. Remember this instruction for when I ask to continue: keep the existing header and file, then add a contact section containing hello@acorn.example and register the final file as a new artifact from that continuation. Do this within our current conversation; no project, delegated task or external action is needed.`); + const first = await settled(ctx, 1); const source = ctx.issue()!; + const [initialArtifact] = await readRegisteredArtifacts(ctx.input.api, source.id, first.runs[0]!.id, [filename]); + const fileBeforeRestart = await currentFile(ctx, filename); + if (fileBeforeRestart !== initialArtifact!.content) throw new Error("Initial downloadable artifact does not match the actual saved draft file"); + const priorHistory = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + const url = ctx.input.page.url(); + await ctx.input.restart(); + await ctx.input.page.goto(url, { waitUntil: "commit", timeout: 60_000 }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: 60_000 }); + await ctx.refreshIssue(); + const afterHistory = await ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`); + return { headerMarker, filename, initialArtifact, initialRunId: first.runs[0]!.id, fileBeforeRestart, + fileAfterRestart: await currentFile(ctx, filename), restarted: true, sourceIssueBefore: source.id, sourceIssueAfter: ctx.issue()?.id, + priorHistoryRetained: priorHistory.every(prior => afterHistory.some(row => row.id === prior.id && row.body === prior.body && row.createdByRunId === prior.createdByRunId)) }; +}, gradeControllerRestart, { + setupHumanMessageCount: 1, expectedHumanMessageCount: 2, expectedRunCount: 2, attemptTimeoutMs: 600_000, + async observe(ctx, state) { + const runs = await ctx.allRuns(); const source = ctx.issue(); + const final = runs.find(run => run.id !== state.initialRunId && run.status === "succeeded"); + if (source && final) { + const [artifact] = await readRegisteredArtifacts(ctx.input.api, source.id, final.id, [String(state.filename)]); + state.finalArtifact = artifact; state.finalFile = await currentFile(ctx, String(state.filename)); + } + return observeRuntime(ctx, state); + }, + async verifyResult(ctx, state, value) { + const o = runtimeObservation(value); const artifact = state.finalArtifact as Artifact; + if (!o || !artifact) throw new Error("No actual final continuation artifact"); + const href = o.links.find(link => { const url = new URL(link, o.origin); return url.origin === new URL(o.origin).origin + && url.pathname === `/api/attachments/${artifact.attachmentId}/content`; }); + if (!href) throw new Error("Final artifact link is absent"); + // HTML attachments use the production download policy. Verify the actual + // human result control and download, not merely an API registration row. + const downloadPromise = ctx.input.page.waitForEvent("download"); + await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).last().click(); + const download = await downloadPromise; + if (download.suggestedFilename() !== state.filename) throw new Error("The visible result downloaded a different file"); + await ctx.input.evidence("runtime-controller-final-download.json", { attachmentId: artifact.attachmentId, filename: download.suggestedFilename(), sha256: artifact.sha256 }); + }, +})];