diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index a70b2d4a65f..ddc2881ae27 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -102,6 +102,21 @@ Neither G1 nor H1 is in the default workloads. The board's actions and what the observer saw are in the timeline (`governance_*`) and in each task's grade (`governance`). +## Stop while running, then continue (S1, `--triggers stop`) + +```sh +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads S1 --faults K1,K2,K4 --triggers stop --concurrency 4 --tasks 6 --seed 23 +``` + +S1 is the board's Stop button. While the task's run is running, the harness cancels it through +`POST /api/heartbeat-runs/:id/cancel`. The `stop` trigger fires the fault 0–2 s later. At least +20 s after the stop, once no run is running or queued and no fault is in progress, the harness +comments that it stopped the run on purpose and asks the agent to continue with an extra marker +line. The oracle checks that the stopped run ended `cancelled`, that nothing started between the +stop and the continue comment (recovery must not restart work the operator stopped), and that the +finished brief has both markers. S1 is not a default workload. + ## Code change in a real repository (R1, `--workspace-repo`) ```sh diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 5c24f708ae4..8852fd00693 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -140,7 +140,7 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC env, adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), // The governance scenarios (G1, H1) are opt-in: H1 needs a wave of its own. - workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1")), + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1" && id !== "R1" && id !== "S1")), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. faults: list(values.faults, FAULT_KINDS, "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), diff --git a/tests/restart-chaos/governance.test.ts b/tests/restart-chaos/governance.test.ts index bc6ef406610..0ce5a2d189b 100644 --- a/tests/restart-chaos/governance.test.ts +++ b/tests/restart-chaos/governance.test.ts @@ -260,6 +260,8 @@ describe("governance wave options", () => { expect(command.options.workloads).not.toContain("G1"); expect(command.options.workloads).not.toContain("H1"); expect(triggerFitsWorkload("approval", "G1")).toBe(true); + expect(triggerFitsWorkload("stop", "S1")).toBe(true); + expect(triggerFitsWorkload("stop", "C1")).toBe(false); expect(triggerFitsWorkload("approval", "A1")).toBe(false); expect(triggerFitsWorkload("paused", "H1")).toBe(true); expect(() => planTasks({ seed: 1, tasks: 2, workloads: ["A1"], adapters: ["codex"], faults: ["K1"], triggers: ["approval"] })) diff --git a/tests/restart-chaos/grader.test.ts b/tests/restart-chaos/grader.test.ts index 1f133a67178..79f0043516b 100644 --- a/tests/restart-chaos/grader.test.ts +++ b/tests/restart-chaos/grader.test.ts @@ -238,6 +238,17 @@ describe("T-oracles", () => { expect(stallWakeChecks({ ...stalled, childStalls: [] } as never)).toEqual([]); }); + it("grades S1: stopped run cancelled, nothing ran before the continue comment, brief has both markers", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: { brief: "CHAOS-abcd1234\nCONTINUE-abcd1234\n- one\n- two\n- three\n" }, + agentComments: [], files: {}, commands: {}, workspacePath: null, + stop: { runId: "r1", runStatus: "cancelled", runsStartedBeforeContinue: [], continuePosted: true } }; + expect(workloadDefinitions.S1.grade("abcd1234", base).verdict).toBe("pass"); + const restarted = workloadDefinitions.S1.grade("abcd1234", { ...base, stop: { ...base.stop!, runsStartedBeforeContinue: ["r2"] } }); + expect(restarted.checks.find(check => check.name === "nothing_ran_after_stop")).toMatchObject({ ok: false, detail: "continue=true runs=r2" }); + expect(workloadDefinitions.S1.grade("abcd1234", { ...base, stop: { ...base.stop!, runStatus: "succeeded" } }).verdict).toBe("fail"); + expect(workloadDefinitions.S1.grade("abcd1234", { ...base, stop: undefined }).verdict).toBe("fail"); + }); + it("grades B1 by running the tests and the CLI", () => { const base: OracleArtifacts = { issueStatus: "done", documents: {}, agentComments: [], workspacePath: "/tmp/ws", files: { "NOTES-abcd1234.md": "B1-abcd1234\n" }, diff --git a/tests/restart-chaos/observer.test.ts b/tests/restart-chaos/observer.test.ts index 1ea8d6b37da..4e6510fe111 100644 --- a/tests/restart-chaos/observer.test.ts +++ b/tests/restart-chaos/observer.test.ts @@ -4,7 +4,7 @@ import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import type { ChaosApi } from "./api.js"; -import { CHILD_WAKE_GRACE_MS, REST_CONFIRM_MS, STALL_GIVE_UP_MS, TaskObserver, childrenPending, listWorkspaceFiles } from "./observer.js"; +import { CHILD_WAKE_GRACE_MS, REST_CONFIRM_MS, STALL_GIVE_UP_MS, STOP_CONTINUE_AFTER_MS, TaskObserver, childrenPending, listWorkspaceFiles } from "./observer.js"; import { gradeTask } from "./grader.js"; import { createFixtureRepository, type ChaosLab } from "./seed.js"; import { Timeline } from "./timeline.js"; @@ -138,6 +138,82 @@ describe("TaskObserver", () => { expect(childRetries).toHaveLength(1); }); + it("S1: stops the running run as the board would, then posts the continue comment only after it settles", async () => { + let runStatus = "running"; + const base = fakeApi({ "/runs": () => [{ id: "r1", status: runStatus, createdAt: new Date(0).toISOString(), finishedAt: null }] }, "in_progress"); + const posted: string[] = []; + const api = { ...base.api, get: base.api.get.bind(base.api), async post(route: string) { + posted.push(route); + if (route === "/api/heartbeat-runs/r1/cancel") { runStatus = "cancelled"; return { id: "r1", status: "cancelled" }; } + return route.endsWith("/issues") ? { id: "i1", identifier: "CHA-1" } : { id: "comment-1" }; + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "S1", followUpAfterMs: 1_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 2_000); + expect(posted).toContain("/api/heartbeat-runs/r1/cancel"); + expect(observer.record.stop).toMatchObject({ runId: "r1", httpStatus: 200, error: null }); + await observer.poll(start + 2_000 + STOP_CONTINUE_AFTER_MS - 1); + expect(observer.record.followUp).toBeNull(); + await observer.poll(start + 2_000 + STOP_CONTINUE_AFTER_MS + 1); + expect(observer.record.followUp).toMatchObject({ commentId: "comment-1", duringRun: false }); + expect(posted.filter(route => route.endsWith("/cancel"))).toHaveLength(1); + }); + + it("S1: does not count the continue comment's own run as a run after the stop, even when it starts before the comment returns", async () => { + let runs: Array> = [{ id: "r1", status: "running", createdAt: new Date(0).toISOString(), finishedAt: null }]; + const base = fakeApi({ "/runs": () => runs, "/comments": () => [], "/documents/": () => null, "/workspace-operations": () => [] }, "in_progress"); + let clock = 0; + const api = { ...base.api, get: base.api.get.bind(base.api), async post(route: string) { + if (route === "/api/heartbeat-runs/r1/cancel") { runs = [{ ...runs[0]!, status: "cancelled" }]; return { id: "r1" }; } + if (route.endsWith("/comments")) { + runs = [...runs, { id: "r2", status: "running", createdAt: new Date(clock - 10).toISOString(), finishedAt: null, wakeCommentIds: ["continue-1"] }]; + return { id: "continue-1" }; + } + return { id: "i1", identifier: "CHA-1" }; + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "S1", followUpAfterMs: 1_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 2_000); + clock = start + 2_000 + STOP_CONTINUE_AFTER_MS + 1; + await observer.poll(clock); + await observer.poll(clock + 3_000); + const artifacts = await observer.collectArtifacts(); + expect(artifacts.stop).toMatchObject({ runId: "r1", runStatus: "cancelled", runsStartedBeforeContinue: [], continuePosted: true }); + }); + + it("S1: counts a run recovery started between the poll's reads and the continue request", async () => { + let runs: Array> = [{ id: "r1", status: "running", createdAt: new Date(0).toISOString(), finishedAt: null }]; + let stopped = false; + const base = fakeApi({ "/runs": () => runs, "/comments": () => [], "/documents/": () => null, "/workspace-operations": () => [] }, "in_progress"); + let sentAt = 0; + const api = { ...base.api, async get(route: string) { + const value = await base.api.get(route); + if (stopped && route.endsWith("/runs") && runs.length === 1) { + runs = [...runs, { id: "r2", status: "queued", createdAt: new Date(sentAt - 5).toISOString(), finishedAt: null }]; + } + return value; + }, async post(route: string) { + if (route === "/api/heartbeat-runs/r1/cancel") { stopped = true; runs = [{ ...runs[0]!, status: "cancelled" }]; return { id: "r1" }; } + if (route.endsWith("/comments")) return { id: "continue-1" }; + return { id: "i1", identifier: "CHA-1" }; + } } as unknown as ChaosApi; + const observer = new TaskObserver({ ...plan, workload: "S1", followUpAfterMs: 1_000 }, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 2_000); + const at = start + 2_000 + STOP_CONTINUE_AFTER_MS + 1; + sentAt = at + 100; + runs = [runs[0]!]; + observer.record.followUp = { at, sentAt, postedAt: sentAt + 50, commentId: "continue-1", error: null, duringRun: false }; + observer.record.samples.push({ at: sentAt + 200, reachable: true, serverReady: true, issueStatus: "in_progress", interruption: null, + activeRecoveryAction: false, recoveryOwner: null, executionBlocker: false, scheduledRetry: false, + runs: [runs[0], { id: "r2", status: "queued", createdAt: new Date(sentAt - 5).toISOString(), finishedAt: null }] } as never); + const artifacts = await observer.collectArtifacts(); + expect(artifacts.stop?.runsStartedBeforeContinue).toEqual(["r2"]); + }); + it("closes an open task as unfinished at interrupt, so the wave grades it as a failure", async () => { const { api } = fakeApi({}, "in_progress"); const timeline = new Timeline(null); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index efd5dd47e70..251a2b9bf90 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -50,6 +50,7 @@ const RESULT_TRIGGER_EVENTS = new Set(["run.result.proposed", "run.result.accept export const REST_CONFIRM_MS = 12_000; export const CHILD_WAKE_GRACE_MS = 60_000; export const CHILD_STALL_CHECK_MS = 10_000; +export const STOP_CONTINUE_AFTER_MS = 20_000; export function childrenPending(children: ReadonlyArray<{ status: string | null }> | null): boolean { return !children || children.some(child => !["done", "cancelled", "blocked"].includes(child.status ?? "")); @@ -344,6 +345,11 @@ export class TaskObserver { case "tool": return this.record.benchFirstSeenAt ?? null; case "approval": return this.record.governance?.approvalSeenAt ?? null; case "paused": return this.record.governance?.budgetPausedAt ?? null; + case "stop": { + const stop = this.record.stop; + if (!stop) return null; + return stop.error ? "missed" : stop.at; + } case "result": { // B8: the run that received the follow-up has accepted its result but // has not settled yet. A follow-up posted at rest cannot cross it. @@ -415,6 +421,10 @@ export class TaskObserver { this.restCandidateSince = null; this.childrenSettledAt = null; } + if (await this.maybeStop(sample, now)) { + this.restCandidateSince = null; + return; + } if (await this.maybeFollowUp(sample, now)) { // The comment response is not the agent's answer: require fresh samples. this.restCandidateSince = null; @@ -490,18 +500,42 @@ export class TaskObserver { } } + private async maybeStop(sample: TaskSample, now: number): Promise { + if (!this.definition.stopBeforeFollowUp || this.record.stop || this.plan.followUpAfterMs === null || !sample.reachable) return false; + if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; + const running = (sample.runs ?? []).find(run => run.status === "running"); + if (!running) return false; + const stop: NonNullable = { at: now, runId: running.id, httpStatus: null, error: null }; + try { + await this.api.post(`/api/heartbeat-runs/${running.id}/cancel`, { cancellationRequestId: randomUUID() }); + stop.httpStatus = 200; + } catch (error) { + stop.httpStatus = error instanceof ChaosApiError ? error.status : null; + stop.error = error instanceof Error ? error.message : String(error); + } + this.record.stop = stop; + this.timeline.add("task_stop_clicked", { taskId: this.plan.taskId, ...stop }); + return true; + } + private async maybeFollowUp(sample: TaskSample, now: number): Promise { const definition = this.definition; if (!definition.followUp || this.record.followUp || this.plan.followUpAfterMs === null || !sample.reachable) return false; if (now < this.record.createdAt! + this.plan.followUpAfterMs) return false; + if (definition.stopBeforeFollowUp) { + const stop = this.record.stop; + if (!stop || now - stop.at < STOP_CONTINUE_AFTER_MS || this.supervisor.faultInProgress) return false; + if ((sample.runs ?? []).some(run => run.status === "running" || run.status === "queued")) return false; + } // C1 targets a live run (B8); wait for one unless the task already rests. const running = (sample.runs ?? []).some(run => run.status === "running"); - if (!running && !isResting(sample)) return false; + if (!running && !definition.stopBeforeFollowUp && !isResting(sample)) return false; + const sentAt = Math.max(now, Date.now()); try { const comment = await this.api.post(`/api/issues/${this.record.issueId}/comments`, { body: definition.followUp(this.plan.nonce) }); - this.record.followUp = { at: now, postedAt: Math.max(now, Date.now()), commentId: comment?.id ?? null, error: null, duringRun: running }; + this.record.followUp = { at: now, sentAt, postedAt: Math.max(now, Date.now()), commentId: comment?.id ?? null, error: null, duringRun: running }; } catch (error) { - this.record.followUp = { at: now, commentId: null, error: error instanceof Error ? error.message : String(error), duringRun: running }; + this.record.followUp = { at: now, sentAt, commentId: null, error: error instanceof Error ? error.message : String(error), duringRun: running }; } this.timeline.add("task_follow_up", { taskId: this.plan.taskId, ...this.record.followUp }); return true; @@ -567,6 +601,18 @@ export class TaskObserver { return this.retryBlockedChildren(blocked, now); } + private stopArtifacts(stop: NonNullable): NonNullable { + const runs = new Map[number]>(); + for (const sample of this.record.samples) for (const run of sample.runs ?? []) runs.set(run.id, run); + const continueAt = this.record.followUp?.sentAt ?? this.record.followUp?.at ?? null; + const continueCommentId = this.record.followUp?.commentId ?? null; + const startedBeforeContinue = [...runs.values()].filter(run => run.id !== stop.runId && run.createdAt + && !(continueCommentId && (run.wakeCommentIds ?? []).includes(continueCommentId)) + && Date.parse(run.createdAt) > stop.at && (continueAt === null || Date.parse(run.createdAt) < continueAt)).map(run => run.id); + return { runId: stop.runId, runStatus: runs.get(stop.runId)?.status ?? null, runsStartedBeforeContinue: startedBeforeContinue, + continuePosted: Boolean(this.record.followUp?.commentId) }; + } + private async retryBlockedChildren(childIds: readonly string[], now: number): Promise { let clicked = false; for (const childIssueId of childIds) { @@ -831,6 +877,7 @@ export class TaskObserver { agentComments: (comments ?? []).filter(comment => comment.authorAgentId).map(comment => String(comment.body ?? "")), files, commands, workspacePath, ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), + ...(this.record.stop ? { stop: this.stopArtifacts(this.record.stop) } : {}), ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId), childStalls: this.record.childStalls ?? [], stallWakes: await this.collectStallWakes(issueId) } : {}), ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 63b5131408a..696c4799ead 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -44,6 +44,7 @@ export const TRIGGER_DELAY_RANGE_MS: Record, readonly result: [0, 3_000], approval: [0, 3_000], paused: [0, 3_000], + stop: [0, 2_000], }; export function triggerFitsWorkload(trigger: TriggerId, workload: WorkloadId): boolean { @@ -51,11 +52,12 @@ export function triggerFitsWorkload(trigger: TriggerId, workload: WorkloadId): b if (trigger === "tool") return workload === "B5"; if (trigger === "approval") return workload === "G1"; if (trigger === "paused") return workload === "H1"; + if (trigger === "stop") return workload === "S1"; return true; } /** The workload a trigger needs, for the error when none is selected. */ -const TRIGGER_WORKLOAD: Partial> = { comment: "C1", result: "C1", tool: "B5", approval: "G1", paused: "H1" }; +const TRIGGER_WORKLOAD: Partial> = { comment: "C1", result: "C1", tool: "B5", approval: "G1", paused: "H1", stop: "S1" }; /** Faults a `result` task may fire inside a storm: a fast restart that lands on the boundary. */ const STORM_RESULT_FAULTS: readonly FaultKind[] = ["K1", "K2"]; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index fb02f8a617f..93676713716 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -7,13 +7,13 @@ export type FaultKind = (typeof FAULT_KINDS)[number]; export type TaskFaultKind = FaultKind | "K13"; /** G1: an approval-gated action across a restart. H1: a budget hard stop crossed mid-run with a restart (its own wave). */ -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1"] as const; +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1", "R1", "S1"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; export type AdapterId = (typeof ADAPTER_IDS)[number]; -export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool", "result", "approval", "paused"] as const; +export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool", "result", "approval", "paused", "stop"] as const; export type TriggerId = (typeof TRIGGER_IDS)[number]; export const ENVIRONMENT_KINDS = ["local", "daytona"] as const; @@ -221,6 +221,7 @@ export interface OracleArtifacts { workspacePath: string | null; /** B5: distinct `.bench/progress` values the harness itself saw while the task was live. */ benchProgressObserved?: number; + stop?: { runId: string; runStatus: string | null; runsStartedBeforeContinue: string[]; continuePosted: boolean }; children?: Array<{ id: string; title?: string | null; status: string | null; assigneeAgentId: string | null; documents: Record }>; childStalls?: Array<{ childIssueId: string; recoveryActionId: string | null }>; stallWakes?: Array<{ stalledIssueId: string | null; recoveryActionId: string | null }>; @@ -281,7 +282,8 @@ export interface TaskRecord { finalStatus: string | null; deadlineAt: number | null; timedOut: boolean; - followUp: { at: number; postedAt?: number; commentId: string | null; error: string | null; duringRun?: boolean } | null; + followUp: { at: number; sentAt?: number; postedAt?: number; commentId: string | null; error: string | null; duringRun?: boolean } | null; + stop?: { at: number; runId: string; httpStatus: number | null; error: string | null }; faultIds: string[]; sweep: RestSweep | null; /** Workspace files present just before the first fault that hit this task (R2). */ diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index 46693c100d6..3bd8ebc73da 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -16,6 +16,7 @@ export interface WorkloadDefinition { title(nonce: string): string; description(nonce: string, agents?: WorkloadAgents): string; followUp?(nonce: string): string; + stopBeforeFollowUp?: boolean; /** Files and commands the observer must collect for the T-oracle. */ collect: { documents: string[]; files(nonce: string): string[]; commands: Array<{ name: string; argv: string[] | ((nonce: string) => string[]) }>; childDocuments?: string[] }; @@ -35,6 +36,7 @@ const partMarker = (nonce: string, part: number) => `PART${part}-${nonce}`; const briefMarker = (nonce: string) => `CHAOS-${nonce}`; const followUpMarker = (nonce: string) => `FOLLOWUP-${nonce}`; +const continueMarker = (nonce: string) => `CONTINUE-${nonce}`; const notesFile = (nonce: string) => `NOTES-${nonce}.md`; const benchFile = (nonce: string) => `bench-result-${nonce}.txt`; const benchRecord = (nonce: string) => `.bench/run-${nonce}.txt`; @@ -251,6 +253,29 @@ export const workloadDefinitions: Record = { return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce), followUpMarker(nonce)])]); }, }, + S1: { + id: "S1", + label: "Stop while running, then continue", + usesRepository: false, + fireWindowMs: [20_000, 60_000], + followUpWindowMs: [8_000, 25_000], + budgetMs: 12 * 60_000, + title: nonce => `Chaos S1 brief stopped and continued ${nonce}`, + description: briefDescription, + stopBeforeFollowUp: true, + followUp: nonce => `I stopped your previous run on purpose. Please continue the task now: finish the \`brief\` document and also include the line \`${continueMarker(nonce)}\` on its own line before you mark the task done.`, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade(nonce, artifacts) { + const stop = artifacts.stop; + return result([ + statusCheck(artifacts), + ...briefChecks(artifacts.documents.brief, [briefMarker(nonce), continueMarker(nonce)]), + { name: "stop_took_effect", ok: stop?.runStatus === "cancelled", detail: `run=${stop?.runId ?? "none"} status=${stop?.runStatus ?? "none"}` }, + { name: "nothing_ran_after_stop", ok: Boolean(stop) && stop!.continuePosted && stop!.runsStartedBeforeContinue.length === 0, + detail: stop ? `continue=${stop.continuePosted} runs=${stop.runsStartedBeforeContinue.join(",") || "none"}` : "not stopped" }, + ]); + }, + }, B1: { id: "B1", label: "Python CLI with tests",