From c3c226d165ed978c9ecd5a8034c951c20bc213c8 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 13:40:39 -0600 Subject: [PATCH] test(restart-chaos): governance (approval, budget hard stop) and upgrade mid-run scenarios (#20089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness kills and restarts the control plane during real agent tasks and grades every task > - It covers run lifecycle, comments, delegation and long tools, but not two areas that must also survive a restart: governance (approvals and budget hard stops) and an upgrade in the middle of a run > - An approval that disappears, runs twice or runs after a rejection, or a budget pause that dispatches work, breaks governance. An upgrade that cannot resume saved state breaks the upgrade-safe-saved-state rule > - This pull request adds the harness support for both scenarios, using only the paths a real board user and agent use > - The benefit is that the next chaos waves can test these boundaries and report failures with evidence ## Linked Issues or Issue Description Refs #20084, #20067 **What happened?** The restart chaos harness had no scenario for an approval-gated action pending across a restart, for a budget hard stop crossed mid-run with a restart, or for a server upgrade in the middle of a run. These boundaries were untested. **Expected behavior** - **G1:** a pending approval survives K1, K2 or K4. Approving it after the restart runs the gated action exactly once; rejecting it never runs it. - **H1:** a budget hard stop crossed mid-run pauses the company, nothing dispatches while paused (also across a restart), and resuming continues the work. - **K14:** a wave started on an older build and upgraded mid-turn resumes old-saved state on the new build with no Retry, no lost comments and no stuck records. **Steps to reproduce** 1. Run the harness commands in `tests/restart-chaos/README.md` ("Governance across restarts" and "Upgrade mid-run"). 2. Each task is graded R1–R6 plus its oracle. **Paperclip version or commit** master at 4f7eee10c2. ## What Changed - **G1 workload: approval across a restart.** - The agent requests board approval through its `call_api` tool (`POST /api/companies/{companyId}/approvals`, `request_board_approval`, linked to its task), blocks, and ends its turn. - A new `approval` trigger fires the fault once the observer sees the approval pending. - After the restart, with the server ready, the observer acts as the board through `POST /api/approvals/:id/approve` or `/reject` (rejection also cancels the task). The decision is fixed by the nonce. - The oracle checks one request, still pending when decided after the restart, the decision, and the gated subtask exactly once on approval and never after rejection. - R1 and R3 grade a rejected task against `cancelled` (`expectedFinalStatus`). - **H1 workload: budget hard stop across a restart.** - Before launch the observer sets the company budget to observed spend plus one cent through `PATCH /api/companies/:id/budgets`. - A new `paused` trigger fires the fault once the company shows `paused` / `budget`. - After the restart the pause must hold 30 s with the server ready. The observer records runs that started while paused, then resolves the incident with `raise_budget_and_resume`. - The oracle checks the brief, the pause, no dispatch while paused, the resume after the fault, and an active company at rest. - H1 runs alone, at concurrency 1, and not with `--rc06-masked` (config errors). - **K14 fault: upgrade mid-run, with `--upgrade-from `.** - The server starts from the older checkout. - The first K14 stops it with SIGTERM and starts this checkout on the same database, instance directory, port and secrets. The runner binary, pinned Codex and source dependencies are prepared per checkout. - The receipt names both commits. - K14 is local-only and needs `--upgrade-from` (and the reverse). It is excluded from storm defaults. Preflight refuses an older checkout that is not prepared. - G1 and H1 are opt-in: they are not in the default workloads. - README sections for both scenarios. ## Verification - `pnpm exec vitest run --config tests/restart-chaos/vitest.config.ts`: every new and changed test passes; 188 of 189 pass in total. The one failure, `wave-speed.test.ts` "judges time to first agent activity…", also fails on master: #20077 gave the local TTFT a budget after #20071's test expected none. This PR does not touch `wave-speed` or the speed budgets. The new `governance.test.ts` covers: - the board decides only after the fault the pending approval armed, with the server ready, for approve and reject; - a blocked task waiting on its approval is not counted as resting; - G1 and H1 oracle pass and fail cases (duplicate request, action twice or after rejection, decision or resume that crossed no restart, dispatch while paused); - H1 budget set, pause hold and incident resolve, against a fake board API; - trigger gating, opt-in workloads, H1 exclusivity, and K14 option pairing. - `node scripts/typecheck-test-projects.mjs tests/restart-chaos`: passed. - Not run yet: a live wave of G1, H1 or K14. Phase 3 holds the harness slot, and the smoke runs are requested there. Their results will be added here. ## Risks - Test harness only; no product code changes. - G1 relies on the agent following the instructions (request, block, act only when approved). A task where the agent does not follow them fails the oracle visibly. - K14 seeds the company with this checkout's harness code while the older server serves. This works for recent builds; a much older build could reject a seed call. - Live behavior is not proven until the smoke runs. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use (shell, file edits, test execution, a read-only research subagent). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/restart-chaos/README.md | 74 ++++++- tests/restart-chaos/config.ts | 30 ++- tests/restart-chaos/governance.test.ts | 277 +++++++++++++++++++++++++ tests/restart-chaos/grader.ts | 15 +- tests/restart-chaos/main.ts | 45 +++- tests/restart-chaos/observer.ts | 200 +++++++++++++++++- tests/restart-chaos/scheduler.test.ts | 16 ++ tests/restart-chaos/scheduler.ts | 23 +- tests/restart-chaos/supervisor.ts | 95 +++++++-- tests/restart-chaos/types.ts | 53 ++++- tests/restart-chaos/workloads.ts | 85 +++++++- 11 files changed, 864 insertions(+), 49 deletions(-) create mode 100644 tests/restart-chaos/governance.test.ts diff --git a/tests/restart-chaos/README.md b/tests/restart-chaos/README.md index 8a860881fb1..fb5b348a458 100644 --- a/tests/restart-chaos/README.md +++ b/tests/restart-chaos/README.md @@ -55,6 +55,78 @@ and only launched tasks are graded. Storm tasks show fault `K13` in the ledger; receipts are in `faults.json`. A replay of a storm task fires the faults that crossed it, without a new storm. `--triggers time,result` may join a storm: a `result` task (C1) also fires its own K1 or K2 (from `--faults`) just after its run accepts a result, the next storm gap runs from its settle, and no triggered fault fires during the drain. Other triggers do not combine with a storm. +## Governance across restarts (G1, H1) + +```sh +# Approval-gated action: the fault fires 0-3 s after the approval appears pending. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads G1 --faults K1,K2,K4 --triggers approval --concurrency 4 --tasks 6 --seed 31 +# Budget hard stop: its own wave, one task at a time. +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads H1 --faults K1,K2,K4 --triggers paused --concurrency 1 --tasks 3 --seed 32 +``` + +**G1.** The agent asks for board approval through its `call_api` tool +(`POST /api/companies/{companyId}/approvals`, type `request_board_approval`, +linked to its task), blocks the task and ends its turn. The `approval` trigger +fires the planned fault once the observer sees the approval pending. + +After the restart, with the server ready, the harness acts as the board through +the board's own routes. It either approves (`POST /api/approvals/:id/approve`), +or rejects (`POST /api/approvals/:id/reject`) and cancels the task. The +decision is fixed by the task's nonce, so about half approve and half reject. + +On approval, the woken agent creates exactly one backlog subtask named after the +nonce and finishes. The oracle checks these points: + +- exactly one approval was requested (a restart that made the agent ask twice fails); +- it was still pending when the board decided after the restart; +- it reached the board's decision; +- the subtask exists exactly once on approval, and never after a rejection. + +A rejected task rests `cancelled`; R1 and R3 grade against that status. + +**H1.** Before launch, the harness sets the company budget to its observed spend +plus one cent through the board's budget route. The task's first priced usage +then crosses the hard stop mid-run, which pauses the company. The `paused` +trigger fires the fault once the observer sees `paused` with `pauseReason` +`budget`. + +After the restart, the pause must hold for 30 s with the server ready. The +harness records every run that started while paused (there must be none), then +resolves the incident with `raise_budget_and_resume` (spend plus 3000 cents). +The task must then finish its brief on its own. H1 pauses the whole company, so +it runs alone (`--workloads H1 --concurrency 1`) and never with `--rc06-masked`. + +Neither G1 nor H1 is in the default workloads. The board's actions and what the +observer saw are in the timeline (`governance_*`) and in each task's grade +(`governance`). + +## Upgrade mid-run (K14, `--upgrade-from`) + +```sh +git worktree add ../paperclip-838e9bf3 838e9bf3d2 # an older master build +(cd ../paperclip-838e9bf3 && node scripts/agent-preflight.mjs) +node cli/node_modules/tsx/dist/cli.mjs tests/restart-chaos/main.ts --env local \ + --adapters codex,claude --workloads A1,C1,M1 --faults K14 --upgrade-from ../paperclip-838e9bf3 \ + --concurrency 4 --tasks 8 --seed 21 +``` + +The wave's server starts from the older checkout. The first K14 stops it +gracefully (SIGTERM, as a deploy does) and starts this checkout against the same +database, instance directory, port and secrets, so the new build applies its +migrations to the old build's data and resumes what the old build saved. Each +task is graded as usual. Pass means: + +- no task needed Retry (R3); +- C1's follow-up comment reached the work (T); +- the wave-end sweep finds no stuck records (R6). + +A later K14 in the same wave restarts the current build, like K2. Every K14 +receipt names both commits (`upgrade` in `faults.json`, `upgrade_switched` in the +timeline). K14 is local-only, because a Daytona runner must match its image, and +it is never part of a storm's default faults. + ## Daytona (`--env daytona`) The control plane stays local; agent runs execute in Daytona sandboxes from an @@ -168,7 +240,7 @@ re-arms the task's own triggered faults instead of using their offsets. | `scheduler.ts` | Seeded plan: workload × adapter × fault cells and a random fire time per task. Saved to `results//plan.json`. `--replay` reruns one task alone with the same choices and fires the recorded sequence of faults that crossed it (other tasks' faults included) at the same offsets from launch. | | `observer.ts` | Polls each issue and its runs. When a task rests needing Retry it calls `POST /api/issues/:id/retry` once (once more only if a later fault started after that click), with the body `useTaskRetry.ts` sends. Collects T-oracle artifacts and the R6 sweep. | | `grader.ts` | Pure R1–R6 grading over the samples. | -| `workloads.ts` | A1 brief document, B1 python CLI with tests, B5 long bench, C1 comment while running, with their T-oracles. | +| `workloads.ts` | A1 brief document, B1 python CLI with tests, B5 long bench, C1 comment while running, M1 delegation, G1 approval across a restart, H1 budget hard stop across a restart, with their T-oracles. | | `evidence.ts`, `ledger.ts` | Evidence bundle for each failure and the JSONL ledger plus `SUMMARY.md`. | | `daytona-image.ts` | Docker-free extraction of the image's runnerd and provider pack, cached per digest, and the controller/image content-ID check. | | `daytona-sweep.ts` | Daytona R6: read-only reconciliation of this wave's labelled sandboxes with leases; cleanup only through Paperclip. | diff --git a/tests/restart-chaos/config.ts b/tests/restart-chaos/config.ts index 1395a16a8fa..963b966c6ed 100644 --- a/tests/restart-chaos/config.ts +++ b/tests/restart-chaos/config.ts @@ -1,9 +1,13 @@ +import path from "node:path"; import { parseArgs } from "node:util"; import { ADAPTER_IDS, ENVIRONMENT_KINDS, FAULT_KINDS, TRIGGER_IDS, WORKLOAD_IDS, type AdapterId, type EnvironmentKind, type FaultKind, type TriggerId, type WaveOptions, type WorkloadId, } from "./types.js"; +/** A storm's default faults: every restart kind except K14, which needs an older build (`--upgrade-from`). */ +export const STORM_DEFAULT_FAULTS: FaultKind[] = ["K1", "K2", "K4", "K6"]; + export const CREDENTIAL_FOR_ADAPTER: Record = { codex: "OPENAI_API_KEY", claude: "ANTHROPIC_API_KEY", @@ -91,13 +95,14 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC "k4-hold-max-seconds": { type: "string" }, "crash-point": { type: "string" }, "crash-point-hold-ms": { type: "string" }, + "upgrade-from": { type: "string" }, }, }); const keep = values.keep === true; const pollMs = integer(values["poll-ms"], "poll-ms", 3_000, 2_000, 5_000); const budgetCents = integer(values["budget-cents"], "budget-cents", 3_000, 1, 100_000); if (values.replay !== undefined) { - for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms"] as const) { + for (const key of ["env", "adapters", "workloads", "faults", "triggers", "concurrency", "tasks", "seed", "daytona-image", "daytona-cpu", "daytona-reuse-lease", "storm-minutes", "lifecycle", "k4-hold-max-seconds", "crash-point", "crash-point-hold-ms", "upgrade-from"] as const) { if (values[key] !== undefined) throw new Error(`--replay reproduces the saved plan; do not combine it with --${key}`); } return { mode: "replay", ref: parseReplayRef(values.replay), keep, @@ -132,9 +137,10 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC const options: WaveOptions = { env, adapters: list(values.adapters, ADAPTER_IDS, "adapters", ["codex", "claude"]), - workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices), + // The governance scenarios (G1, H1) are opt-in: H1 needs a wave of its own. + workloads: list(values.workloads, workloadChoices, "workloads", workloadChoices.filter(id => id !== "G1" && id !== "H1")), // A crash-point wave schedules no fault; K1 is the restart its self-kill gets. - faults: list(values.faults, FAULT_KINDS, "faults", stormMinutes ? FAULT_KINDS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]), + faults: list(values.faults, FAULT_KINDS, "faults", stormMinutes ? STORM_DEFAULT_FAULTS : crashPoint ? ["K1"] : ["K1", "K2", "K4"]), triggers: list(values.triggers, TRIGGER_IDS, "triggers", ["time"]), concurrency: integer(values.concurrency, "concurrency", 4, 1, 20), tasks: integer(values.tasks, "tasks", stormMinutes ? 200 : 8, 1, 200), @@ -154,6 +160,24 @@ export function parseCommand(argv: readonly string[], now = Date.now()): ParsedC ? { crashPointHoldMs: integer(values["crash-point-hold-ms"], "crash-point-hold-ms", 0, 0, 600_000) } : {}), }; if (!crashPoint && values["crash-point-hold-ms"] !== undefined) throw new Error("--crash-point-hold-ms applies only to --crash-point"); + const upgradeFrom = values["upgrade-from"]; + if (options.faults.includes("K14") !== (upgradeFrom !== undefined)) { + throw new Error("K14 (upgrade mid-run) and --upgrade-from go together: pass both or neither"); + } + if (upgradeFrom !== undefined) { + // The remote runner must match its Daytona image, so an older build cannot run there. + if (env !== "local") throw new Error("--upgrade-from runs only with --env local"); + if (crashPoint) throw new Error("--upgrade-from does not combine with --crash-point"); + options.upgradeFrom = path.resolve(upgradeFrom); + } + // H1 pauses the whole company when its budget hard stop is crossed: no other task can share its + // wave, its tasks run one at a time, and the hard stop must be on. + if (options.workloads.includes("H1")) { + if (options.workloads.length !== 1) throw new Error("--workloads H1 pauses the whole company; run it in its own wave (--workloads H1)"); + if (options.concurrency !== 1) throw new Error("--workloads H1 runs one task at a time; pass --concurrency 1"); + if (options.masks?.includes("rc06")) throw new Error("--workloads H1 needs the budget hard stop; do not combine it with --rc06-masked"); + if (stormMinutes) throw new Error("--workloads H1 does not combine with --storm-minutes"); + } if (options.workloads.includes("M1") && options.adapters.length < 2) { throw new Error("--workloads M1 delegates between two agents; pass at least two --adapters"); } diff --git a/tests/restart-chaos/governance.test.ts b/tests/restart-chaos/governance.test.ts new file mode 100644 index 00000000000..bc6ef406610 --- /dev/null +++ b/tests/restart-chaos/governance.test.ts @@ -0,0 +1,277 @@ +import { describe, expect, it } from "vitest"; +import type { ChaosApi } from "./api.js"; +import { parseCommand } from "./config.js"; +import { gradeR1 } from "./grader.js"; +import { BUDGET_PAUSE_HOLD_MS, BUDGET_RESUME_HEADROOM_CENTS, REST_CONFIRM_MS, TaskObserver, approvalTitleMatches, runsStartedAfter } from "./observer.js"; +import { planTasks, replayOnCurrentBuild, triggerFitsWorkload } from "./scheduler.js"; +import type { ChaosLab } from "./seed.js"; +import { Timeline } from "./timeline.js"; +import type { OracleArtifacts, TaskPlan, TaskRecord, TaskSample, WavePlan } from "./types.js"; +import { g1ApprovalTitle, g1ChildTitle, g1Decision, workloadDefinitions } from "./workloads.js"; + +process.env.RESTART_CHAOS_QUIET = "1"; +const lab: ChaosLab = { companyId: "c1", environmentId: "e1", environmentKind: "local", projectId: "p1", fixtureRepositoryPath: "/tmp/none", + agents: { codex: { id: "a1", name: "Chaos codex" }, claude: { id: "a2", name: "Chaos claude" }, opencode: { id: "a3", name: "Chaos opencode" } } }; +const ready = { serverReady: true, faultInProgress: false }; +const APPROVE = "0bcd1234"; +const REJECT = "1bcd1234"; + +/** Fake board API: routes answered by prefix, every write recorded. */ +function fakeApi(routes: Record unknown>) { + const writes: Array<{ method: string; route: string; body: unknown }> = []; + const answer = (route: string) => { + for (const [prefix, handler] of Object.entries(routes)) if (route.includes(prefix)) return handler(route); + if (route.endsWith("/runs")) return []; + return { id: "i1", status: "blocked", interruption: null }; + }; + const api = { + async get(route: string) { return answer(route); }, + async post(route: string, body: unknown) { writes.push({ method: "POST", route, body }); return route.endsWith("/issues") ? { id: "i1", identifier: "CHA-1" } : {}; }, + async patch(route: string, body: unknown) { writes.push({ method: "PATCH", route, body }); return {}; }, + }; + return { api: api as unknown as ChaosApi, writes }; +} + +const g1Plan = (nonce: string): TaskPlan => ({ taskId: "t01", index: 0, workload: "G1", adapter: "codex", fault: "K1", fireAfterMs: 30_000, + holdMs: 0, followUpAfterMs: null, nonce, trigger: "approval", triggerDelayMs: 0 }); + +describe("G1: approval-gated action across a restart", () => { + it("splits the board's decisions by nonce", () => { + expect(g1Decision(APPROVE)).toBe("approve"); + expect(g1Decision(REJECT)).toBe("reject"); + }); + + it.each([[APPROVE, "approve"], [REJECT, "reject"]] as const)("decides (%s) only after the fault the pending approval armed, with the server ready", + async (nonce, kind) => { + const pending = { id: "ap1", status: "pending", payload: { title: g1ApprovalTitle(nonce) } }; + const { api, writes } = fakeApi({ "/approvals": () => [pending], "view=index": () => [] }); + const observer = new TaskObserver(g1Plan(nonce), api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + expect(observer.record.governance?.approvalSeenAt).toBe(start + 1_000); + // The board does not act before the fault the pending approval armed. + expect(observer.record.governance?.decision).toBeUndefined(); + expect(observer.faultDue(start + 1_001)).toMatchObject({ kind: "K1", trigger: "approval" }); + observer.noteFault("f01", start + 1_001); + await observer.poll(start + 60_000); + expect(observer.record.governance?.decision).toMatchObject({ kind, error: null, pendingWhenDecided: true, afterFault: true }); + expect(observer.record.governance?.childrenBeforeDecision).toBe(0); + expect(writes.filter(write => write.route === `/api/approvals/ap1/${kind}`)).toHaveLength(1); + expect(writes.some(write => write.method === "PATCH" && (write.body as { status?: string }).status === "cancelled")).toBe(kind === "reject"); + }); + + it("holds a task blocked on its approval as waiting, not resting, while its fault is still to come", async () => { + const pending = { id: "ap1", status: "pending", payload: { title: g1ApprovalTitle(APPROVE) } }; + const { api, writes } = fakeApi({ "/approvals": () => [pending] }); + const observer = new TaskObserver({ ...g1Plan(APPROVE), trigger: undefined, triggerDelayMs: undefined, fireAfterMs: 999_999 }, + api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + await observer.poll(start + 1_000 + REST_CONFIRM_MS + 1); + await observer.poll(start + 1_000 + 2 * REST_CONFIRM_MS + 2); + expect(observer.done).toBe(false); + expect(observer.record.governance?.decision).toBeUndefined(); + expect(writes.some(write => write.route.startsWith("/api/approvals/"))).toBe(false); + }); + + const artifacts = (nonce: string, overrides: Partial = {}): OracleArtifacts => { + const approve = g1Decision(nonce) === "approve"; + return { issueStatus: approve ? "done" : "cancelled", documents: {}, agentComments: [], files: {}, commands: {}, workspacePath: null, + approvals: [{ id: "ap1", status: approve ? "approved" : "rejected", title: g1ApprovalTitle(nonce) }], + children: approve ? [{ id: "c1", title: g1ChildTitle(nonce), status: "backlog", assigneeAgentId: null, documents: {} }] : [], + governance: { approvalSeenAt: 1, childrenBeforeDecision: 0, + decision: { kind: g1Decision(nonce), at: 2, httpStatus: 200, error: null, pendingWhenDecided: true, afterFault: true } }, + ...overrides }; + }; + + it("passes an approved action that ran once and a rejected one that never ran", () => { + expect(workloadDefinitions.G1.grade(APPROVE, artifacts(APPROVE)).verdict).toBe("pass"); + expect(workloadDefinitions.G1.grade(REJECT, artifacts(REJECT)).verdict).toBe("pass"); + }); + + it("fails a duplicate request, an action that ran twice or after a rejection, and a decision that crossed no restart", () => { + const failed = (nonce: string, overrides: Partial) => + workloadDefinitions.G1.grade(nonce, artifacts(nonce, overrides)).checks.filter(check => !check.ok).map(check => check.name); + expect(failed(APPROVE, { approvals: [{ id: "ap1", status: "approved", title: g1ApprovalTitle(APPROVE) }, + { id: "ap2", status: "pending", title: g1ApprovalTitle(APPROVE) }] })).toContain("one_approval_requested"); + const child = { id: "c", title: g1ChildTitle(APPROVE), status: "backlog", assigneeAgentId: null, documents: {} }; + expect(failed(APPROVE, { children: [child, { ...child, id: "d" }] })).toEqual(["approved_action_ran_once"]); + expect(failed(REJECT, { children: [{ ...child, title: g1ChildTitle(REJECT) }] })).toEqual(["rejected_action_never_ran"]); + expect(failed(APPROVE, { governance: { approvalSeenAt: 1, childrenBeforeDecision: 0, decision: { kind: "approve", at: 2, httpStatus: 200, + error: null, pendingWhenDecided: true, afterFault: false } } })).toEqual(["board_decided_after_restart"]); + // The agent created the subtask while the approval was still pending: the approval did not gate it. + expect(failed(APPROVE, { governance: { approvalSeenAt: 1, childrenBeforeDecision: 1, decision: { kind: "approve", at: 2, httpStatus: 200, + error: null, pendingWhenDecided: true, afterFault: true } } })).toEqual(["action_gated_by_approval"]); + }); + + it("ends a task whose approval a restart lost at its deadline instead of waiting forever", async () => { + let approvals: unknown[] = [{ id: "ap1", status: "pending", payload: { title: g1ApprovalTitle(APPROVE) } }]; + const { api } = fakeApi({ "/approvals": () => approvals, "view=index": () => [] }); + const observer = new TaskObserver({ ...g1Plan(APPROVE), trigger: undefined, triggerDelayMs: undefined, fireAfterMs: 999_999 }, + api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + approvals = []; + const late = observer.record.deadlineAt! + 1; + await observer.poll(late - REST_CONFIRM_MS - 1); + await observer.poll(late); + expect(observer.done).toBe(true); + expect(observer.record.timedOut).toBe(true); + }); + + it("grades a rejected task that rests cancelled as continuing correctly (R1)", () => { + const record = { plan: g1Plan(REJECT), launchError: null, retry: null, finalStatus: "cancelled", timedOut: false, + createdAt: 0, restAt: 60_000, samples: [] } as unknown as TaskRecord; + expect(gradeR1(record, 15 * 60_000, []).verdict).toBe("pass"); + expect(gradeR1({ ...record, finalStatus: "done" }, 15 * 60_000, []).verdict).toBe("fail"); + }); + + it("counts a redacted approval title as the task's own", () => { + expect(approvalTitleMatches({ payload: { title: g1ApprovalTitle(APPROVE) } }, APPROVE)).toBe(true); + expect(approvalTitleMatches({ payload: {} }, APPROVE)).toBe(true); + expect(approvalTitleMatches({ payload: { title: "Approve something else" } }, APPROVE)).toBe(false); + }); +}); + +describe("H1: budget hard stop across a restart", () => { + const h1Plan: TaskPlan = { taskId: "t01", index: 0, workload: "H1", adapter: "codex", fault: "K2", fireAfterMs: 30_000, + holdMs: 0, followUpAfterMs: null, nonce: "abcd1234", trigger: "paused", triggerDelayMs: 0 }; + + it("sets the budget to spend plus one cent, then resumes through the incident after the pause held across the fault", async () => { + const pausedAt = new Date(Date.now() + 2_000).toISOString(); + let paused = false; + const { api, writes } = fakeApi({ + "/budgets/overview": () => ({ policies: [{ scopeType: "company", observedAmount: 40.25 }], + activeIncidents: paused ? [{ id: "inc1", scopeType: "company", amountObserved: 55.5 }] : [] }), + "/api/companies/c1": () => paused ? { id: "c1", status: "paused", pauseReason: "budget", pausedAt } : { id: "c1", status: "active" }, + }); + const observer = new TaskObserver(h1Plan, api, lab, new Timeline(null), ready); + await observer.launch(); + // Whole cents: observed spend can be fractional. + expect(writes[0]).toMatchObject({ method: "PATCH", route: "/api/companies/c1/budgets", body: { budgetMonthlyCents: 42 } }); + const start = observer.record.createdAt!; + paused = true; + await observer.poll(start + 3_000); + expect(observer.record.governance?.budgetPausedAt).toBeDefined(); + expect(observer.faultDue(start + 3_001)).toMatchObject({ kind: "K2", trigger: "paused" }); + observer.noteFault("f01", start + 3_001); + // The pause must hold with the server ready before the board resumes. + await observer.poll(start + 10_000); + expect(observer.record.governance?.resume).toBeUndefined(); + await observer.poll(start + 10_000 + BUDGET_PAUSE_HOLD_MS + 1); + expect(observer.record.governance?.resume).toMatchObject({ incidentId: "inc1", amount: 56 + BUDGET_RESUME_HEADROOM_CENTS, + error: null, afterFault: true }); + expect(observer.record.governance?.runsStartedWhilePaused).toEqual([]); + expect(writes.at(-1)).toMatchObject({ method: "POST", route: "/api/companies/c1/budget-incidents/inc1/resolve", + body: { action: "raise_budget_and_resume", amount: 56 + BUDGET_RESUME_HEADROOM_CENTS } }); + }); + + it("catches, at grading, a run that started after the last poll but before the resume returned", async () => { + let paused = true; + let runs: unknown[] = []; + const { api } = fakeApi({ + "/budgets/overview": () => ({ policies: [{ scopeType: "company", observedAmount: 0 }], + activeIncidents: [{ id: "inc1", scopeType: "company", amountObserved: 1 }] }), + "/api/companies/c1": () => paused ? { id: "c1", status: "paused", pauseReason: "budget" } : { id: "c1", status: "active" }, + "/runs": () => runs, + "/documents/": () => null, + "/comments": () => [], + }); + const observer = new TaskObserver(h1Plan, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + observer.faultDue(start + 1_001); + observer.noteFault("f01", start + 1_001); + await observer.poll(start + 5_000); + await observer.poll(start + 5_000 + BUDGET_PAUSE_HOLD_MS + 1); + expect(observer.record.governance?.runsStartedWhilePaused).toEqual([]); + const resumeDone = observer.record.governance!.resume!.doneAt!; + paused = false; + // Dispatched while still paused, but only visible after the resume. + runs = [{ runId: "sneaky", status: "succeeded", startedAt: new Date(resumeDone - 1).toISOString(), createdAt: null, finishedAt: null }]; + const artifacts = await observer.collectArtifacts(); + expect(artifacts.governance?.runsStartedWhilePaused).toEqual(["sneaky"]); + }); + + it("records a pause that clears before the board resumes it, and does not resume", async () => { + let paused = true; + const { api, writes } = fakeApi({ + "/budgets/overview": () => ({ policies: [{ scopeType: "company", observedAmount: 0 }], activeIncidents: [{ id: "inc1", scopeType: "company" }] }), + "/api/companies/c1": () => paused ? { id: "c1", status: "paused", pauseReason: "budget" } : { id: "c1", status: "active" }, + }); + const observer = new TaskObserver(h1Plan, api, lab, new Timeline(null), ready); + await observer.launch(); + const start = observer.record.createdAt!; + await observer.poll(start + 1_000); + observer.faultDue(start + 1_001); + observer.noteFault("f01", start + 1_001); + paused = false; + await observer.poll(start + 1_000 + BUDGET_PAUSE_HOLD_MS + 10_000); + await observer.poll(start + 1_000 + 2 * BUDGET_PAUSE_HOLD_MS + 20_000); + expect(observer.record.governance?.pauseLostAt).toBeDefined(); + expect(observer.record.governance?.resume).toBeUndefined(); + expect(writes.some(write => write.route.includes("/resolve"))).toBe(false); + }); + + it("finds runs that started while the company was paused", () => { + const sample = (runs: Array<{ id: string; startedAt: string | null }>): TaskSample => ({ at: 0, reachable: true, serverReady: true, + runs: runs.map(run => ({ ...run, status: "running", createdAt: null, finishedAt: null, errorCode: null, retryOfRunId: null, + wakeCommentIds: [], livenessState: null })) }); + expect(runsStartedAfter([sample([{ id: "before", startedAt: new Date(1_000).toISOString() }]), + sample([{ id: "after", startedAt: new Date(5_000).toISOString() }, { id: "queued", startedAt: null }])], 2_000)).toEqual(["after"]); + }); + + it("grades a missing pause, a dispatch while paused and a resume that crossed no restart as failures", () => { + const base: OracleArtifacts = { issueStatus: "done", documents: { brief: "CHAOS-abcd1234\n- a\n- b\n- c" }, agentComments: [], files: {}, + commands: {}, workspacePath: null, governance: { budgetSetCents: 41, budgetPausedAt: 1, runsStartedWhilePaused: [], + resume: { at: 2, amount: 3055, incidentId: "inc1", httpStatus: 200, error: null, afterFault: true }, companyActiveAtRest: true } }; + expect(workloadDefinitions.H1.grade("abcd1234", base).verdict).toBe("pass"); + const failed = (governance: OracleArtifacts["governance"]) => workloadDefinitions.H1.grade("abcd1234", { ...base, governance }) + .checks.filter(check => !check.ok).map(check => check.name); + expect(failed({ ...base.governance, budgetPausedAt: null })).toContain("company_paused_for_budget"); + expect(failed({ ...base.governance, pauseLostAt: 5 })).toEqual(["pause_held_until_resume"]); + expect(failed({ ...base.governance, runsStartedWhilePaused: ["r2"] })).toEqual(["nothing_dispatched_while_paused"]); + expect(failed({ ...base.governance, resume: { ...base.governance!.resume!, afterFault: false } })).toEqual(["board_resumed_after_restart"]); + }); +}); + +describe("K14 replay", () => { + it("replays a task that launched after the upgrade on the current build, its K14 restarts as K2", () => { + const plan = { schema: "paperclip.restart-chaos.wave-plan/v1", waveId: "w", seed: 1, createdAt: "", + options: { env: "local", adapters: ["codex"], workloads: ["A1"], faults: ["K14", "K1"], concurrency: 1, tasks: 1, seed: 1, + budgetCents: 3_000, pollMs: 3_000, upgradeFrom: "/old" }, + tasks: [{ taskId: "t02", index: 0, workload: "A1", adapter: "codex", fault: "K14", fireAfterMs: 1, holdMs: 0, followUpAfterMs: null, + nonce: "n", replayFaults: [{ kind: "K14", holdMs: 0, atMs: 5 }, { kind: "K1", holdMs: 0, atMs: 9 }] }] } as WavePlan; + const replay = replayOnCurrentBuild(plan); + expect(replay.options.upgradeFrom).toBeUndefined(); + expect(replay.options.faults).toEqual(["K2", "K1"]); + expect(replay.tasks[0]).toMatchObject({ fault: "K2", replayFaults: [{ kind: "K2" }, { kind: "K1" }] }); + }); +}); + +describe("governance wave options", () => { + it("keeps G1 and H1 out of default waves and gates their triggers", () => { + const command = parseCommand(["--seed", "3"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.workloads).not.toContain("G1"); + expect(command.options.workloads).not.toContain("H1"); + expect(triggerFitsWorkload("approval", "G1")).toBe(true); + expect(triggerFitsWorkload("approval", "A1")).toBe(false); + expect(triggerFitsWorkload("paused", "H1")).toBe(true); + expect(() => planTasks({ seed: 1, tasks: 2, workloads: ["A1"], adapters: ["codex"], faults: ["K1"], triggers: ["approval"] })) + .toThrow(/needs workload G1/); + const tasks = planTasks({ seed: 1, tasks: 4, workloads: ["G1"], adapters: ["codex"], faults: ["K1", "K2"], triggers: ["approval"] }); + expect(tasks.every(task => task.trigger === "approval")).toBe(true); + }); + + it("runs H1 alone, one task at a time, with the hard stop on", () => { + expect(parseCommand(["--workloads", "H1", "--concurrency", "1", "--triggers", "paused"]).mode).toBe("wave"); + expect(() => parseCommand(["--workloads", "H1,A1", "--concurrency", "1"])).toThrow(/its own wave/); + expect(() => parseCommand(["--workloads", "H1"])).toThrow(/--concurrency 1/); + expect(() => parseCommand(["--workloads", "H1", "--concurrency", "1", "--rc06-masked"])).toThrow(/hard stop/); + }); +}); diff --git a/tests/restart-chaos/grader.ts b/tests/restart-chaos/grader.ts index 03b565b187e..822a905059d 100644 --- a/tests/restart-chaos/grader.ts +++ b/tests/restart-chaos/grader.ts @@ -1,4 +1,5 @@ import type { FaultReceipt, OracleResult, RuleId, RuleResult, TaskGrade, TaskRecord, TaskSample } from "./types.js"; +import { g1Decision } from "./workloads.js"; /** Pure R1–R6 grading over observer samples (plan §2). No I/O. */ @@ -51,16 +52,22 @@ export function outageOverlapMs(record: TaskRecord, faults: readonly FaultReceip }, 0); } +/** The status a task rests in when it succeeds: `done`, or `cancelled` for a G1 request the board rejected. */ +export function expectedFinalStatus(record: TaskRecord): string { + return record.plan.workload === "G1" && g1Decision(record.plan.nonce) === "reject" ? "cancelled" : "done"; +} + export function gradeR1(record: TaskRecord, budgetMs: number, faults: readonly FaultReceipt[]): RuleResult { if (record.launchError) return { verdict: "fail", detail: `launch_failed: ${record.launchError}` }; if (record.retry) return { verdict: "n/a", detail: "task rested needing Retry; graded under R3" }; - if (record.finalStatus !== "done") { + const expected = expectedFinalStatus(record); + if (record.finalStatus !== expected) { return { verdict: "fail", detail: record.timedOut ? `no_rest_within_budget (last status ${record.finalStatus ?? "unknown"})` : `rested_not_done (status ${record.finalStatus ?? "unknown"}, no Retry offered)` }; } const allowed = budgetMs + outageOverlapMs(record, faults); const took = (record.restAt ?? Number.POSITIVE_INFINITY) - record.createdAt!; return took <= allowed - ? { verdict: "pass", detail: `done after ${Math.round(took / 1000)}s (allowed ${Math.round(allowed / 1000)}s)` } + ? { verdict: "pass", detail: `${expected} after ${Math.round(took / 1000)}s (allowed ${Math.round(allowed / 1000)}s)` } : { verdict: "fail", detail: `done_too_late ${Math.round(took / 1000)}s > ${Math.round(allowed / 1000)}s` }; } @@ -88,7 +95,7 @@ export function gradeR2(record: TaskRecord, oracle: OracleResult): RuleResult { export function gradeR3(record: TaskRecord): RuleResult { const last = lastReachable(record); if (!record.retry) { - if (record.finalStatus !== "done" && !record.timedOut && !record.launchError) { + if (record.finalStatus !== expectedFinalStatus(record) && !record.timedOut && !record.launchError) { return { verdict: "fail", detail: `missing_retry: rested ${record.finalStatus ?? "unknown"} with canRetry=${String(last?.interruption?.canRetry ?? false)}` }; } return { verdict: "n/a", detail: "no Retry needed" }; @@ -99,7 +106,7 @@ export function gradeR3(record: TaskRecord): RuleResult { const retryAt = record.retry.at; const newRun = record.samples.some(sample => (sample.runs ?? []).some(run => (ms(run.createdAt) || 0) >= retryAt - 1_000)); if (!newRun) return { verdict: "fail", detail: "retry_no_new_run" }; - if (record.finalStatus !== "done") return { verdict: "fail", detail: `retry_did_not_complete (status ${record.finalStatus ?? "unknown"}${record.timedOut ? ", timed out" : ""})` }; + if (record.finalStatus !== expectedFinalStatus(record)) return { verdict: "fail", detail: `retry_did_not_complete (status ${record.finalStatus ?? "unknown"}${record.timedOut ? ", timed out" : ""})` }; const clicks = 1 + (record.priorRetries?.length ?? 0); const failed = failedRunCodes(record).join(",") || "unknown"; if (clicks > 1) return { verdict: "fail", detail: `retry_needed_${clicks}_clicks (${failed})` }; diff --git a/tests/restart-chaos/main.ts b/tests/restart-chaos/main.ts index f8c765c4dee..dc02edc393a 100644 --- a/tests/restart-chaos/main.ts +++ b/tests/restart-chaos/main.ts @@ -23,7 +23,7 @@ import { checkPlanInvariants, summarizeViolations, type InvariantCheck } from ". import { appendLedger, ledgerLine, writeSummary, type LedgerLine } from "./ledger.js"; import { collectWaveSpeed, speedMarkdown } from "./wave-speed.js"; import { TaskObserver } from "./observer.js"; -import { StormRun, TASK_FAULT_LOG, observedFaultSchedule, planWave, replayRecordFor, replayWave, type TaskFaultLog } from "./scheduler.js"; +import { StormRun, TASK_FAULT_LOG, observedFaultSchedule, planWave, replayOnCurrentBuild, replayRecordFor, replayWave, type TaskFaultLog } from "./scheduler.js"; import { chaosLifecycleOverrides, seedChaosLab } from "./seed.js"; import { ChaosSupervisor } from "./supervisor.js"; import { isBrokenStdoutError, markTimelineEchoBroken, Timeline } from "./timeline.js"; @@ -45,14 +45,23 @@ async function loadPlan(): Promise<{ plan: WavePlan; keep: boolean }> { const source = JSON.parse(await readFile(path.join(sourceDir, "plan.json"), "utf8")) as WavePlan; // Replay the faults that actually crossed the task, including other tasks' faults. const graded = await readFile(path.join(sourceDir, `${command.ref.taskId}.grade.json`), "utf8") - .then(text => JSON.parse(text) as { record: { createdAt: number | null; faultIds: string[] } }).catch(() => null); + .then(text => JSON.parse(text) as { record: { createdAt: number | null; faultIds: string[]; serverBuildAtLaunch?: "upgrade_from" | "current" } }) + .catch(() => null); // An interrupted or wedged wave has no grade files; its fault log was written after every fault. const logged = await readFile(path.join(sourceDir, TASK_FAULT_LOG), "utf8").then(text => JSON.parse(text) as TaskFaultLog).catch(() => null); const crossed = replayRecordFor(command.ref.taskId, graded?.record ?? null, logged); const faults = await readFile(path.join(sourceDir, "faults.json"), "utf8").then(text => JSON.parse(text) as FaultReceipt[]).catch(() => []); const observed = crossed ? observedFaultSchedule(crossed.createdAt, crossed.faultIds, faults, command.ref.taskId) : []; if (!observed.length) console.warn("Replay: no recorded fault sequence for this task; using its planned fault only."); - return { plan: replayWave(source, command.ref, Date.now(), command.overrides, observed), keep: command.keep }; + let plan = replayWave(source, command.ref, Date.now(), command.overrides, observed); + // K14: a task that launched after the upgrade replays on the current build. + const launchBuild = graded?.record.serverBuildAtLaunch ?? logged?.[command.ref.taskId]?.serverBuildAtLaunch; + if (plan.options.upgradeFrom && launchBuild === "current") plan = replayOnCurrentBuild(plan); + // H1 needs the hard stop that the RC-06 mask turns off. + if (plan.tasks.some(task => task.workload === "H1") && plan.options.masks?.includes("rc06")) { + throw new Error("An H1 replay needs the budget hard stop; do not combine it with --rc06-masked"); + } + return { plan, keep: command.keep }; } return { plan: planWave(waveIdFor(command.options.seed), command.options), keep: command.keep }; } @@ -110,6 +119,24 @@ function preflight(plan: WavePlan) { const runner = process.env.PAPERCLIP_RUNNER_BINARY ?? path.join(repositoryRoot, "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"); if (!existsSync(runner)) throw new Error(`Runner binary missing at ${runner}. Run: pnpm --filter @paperclipai/paperclip-runner run build:runner-binaries`); + // K14: the older checkout must be prepared (installed and built) before the wave, like this one. + const upgradeFrom = plan.options.upgradeFrom; + // One runner binary for both builds would not test the older runner's saved state. + if (upgradeFrom && process.env.PAPERCLIP_RUNNER_BINARY) { + throw new Error("--upgrade-from runs each build's own runner; unset PAPERCLIP_RUNNER_BINARY"); + } + if (upgradeFrom) { + for (const required of ["tests/runner-e2e/server-entry.ts", "node_modules", "packages/paperclip-runner/runner/target/debug/paperclip-runnerd"]) { + if (!existsSync(path.join(upgradeFrom, required))) { + throw new Error(`--upgrade-from ${upgradeFrom} is not a prepared checkout (missing ${required}). ` + + "Create it with `git worktree add `, then run `node scripts/agent-preflight.mjs` in it."); + } + } + if (path.resolve(upgradeFrom) === repositoryRoot) throw new Error("--upgrade-from must be a different checkout than the one running the harness"); + const oldBuild = spawnSync(process.execPath, [path.join(upgradeFrom, "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs")], + { cwd: upgradeFrom, stdio: "inherit" }); + if (oldBuild.status !== 0) throw new Error("Verified provider entrypoint build failed in the --upgrade-from checkout"); + } // Same provider bundle preparation as tests/runner-e2e/verified-provider-bundles.setup.mjs. const build = spawnSync(process.execPath, [path.join(repositoryRoot, "packages/paperclip-runner/scripts/build-verified-provider-entrypoints.mjs")], { cwd: repositoryRoot, stdio: "inherit" }); @@ -143,7 +170,8 @@ async function main(): Promise { // Baseline only after the harness's own build steps (preflight rewrites the runner's dist). startFingerprint = worktreeFingerprint(repositoryRoot, guardedOutputs); - const supervisor = await ChaosSupervisor.create({ repositoryRoot, waveDir, timeline }); + const supervisor = await ChaosSupervisor.create({ repositoryRoot, waveDir, timeline, + ...(plan.options.upgradeFrom ? { upgradeFrom: plan.options.upgradeFrom } : {}) }); // Crash-point suite: one booted server, one point at a time. Each point gets // its own batch of tasks; the point is armed once the batch launched and the // server kills itself at its nth hit (restarted like K1). Invariants are @@ -240,7 +268,8 @@ async function main(): Promise { for (const candidate of exposed) { candidate.noteFault(receipt.faultId, receipt.startedAt); candidate.extendDeadline((receipt.readyAt ?? Date.now()) - receipt.startedAt); - taskFaultLog[candidate.plan.taskId] = { createdAt: candidate.record.createdAt, faultIds: [...candidate.record.faultIds] }; + taskFaultLog[candidate.plan.taskId] = { createdAt: candidate.record.createdAt, faultIds: [...candidate.record.faultIds], + ...(candidate.record.serverBuildAtLaunch ? { serverBuildAtLaunch: candidate.record.serverBuildAtLaunch } : {}) }; } void persistFaultLog(); }, (error: Error & { receipt?: FaultReceipt }) => { @@ -250,7 +279,8 @@ async function main(): Promise { faults.push(receipt); for (const candidate of exposed) { candidate.noteFault(receipt.faultId, receipt.startedAt); - taskFaultLog[candidate.plan.taskId] = { createdAt: candidate.record.createdAt, faultIds: [...candidate.record.faultIds] }; + taskFaultLog[candidate.plan.taskId] = { createdAt: candidate.record.createdAt, faultIds: [...candidate.record.faultIds], + ...(candidate.record.serverBuildAtLaunch ? { serverBuildAtLaunch: candidate.record.serverBuildAtLaunch } : {}) }; } void persistFaultLog(); } @@ -267,7 +297,8 @@ async function main(): Promise { for (const candidate of active.filter(observer => !observer.done)) { candidate.noteFault(receipt.faultId, receipt.startedAt); candidate.extendDeadline((receipt.readyAt ?? Date.now()) - receipt.startedAt); - taskFaultLog[candidate.plan.taskId] = { createdAt: candidate.record.createdAt, faultIds: [...candidate.record.faultIds] }; + taskFaultLog[candidate.plan.taskId] = { createdAt: candidate.record.createdAt, faultIds: [...candidate.record.faultIds], + ...(candidate.record.serverBuildAtLaunch ? { serverBuildAtLaunch: candidate.record.serverBuildAtLaunch } : {}) }; } checkWorktree(receipt.faultId); void persistFaultLog(); diff --git a/tests/restart-chaos/observer.ts b/tests/restart-chaos/observer.ts index 7d84f17e39d..919d4fdead5 100644 --- a/tests/restart-chaos/observer.ts +++ b/tests/restart-chaos/observer.ts @@ -9,12 +9,13 @@ import { ACTIVE_RUN_STATUSES, isStalledSample, STALL_LIMIT_MS } from "./grader.j import type { ChaosLab } from "./seed.js"; import type { Timeline } from "./timeline.js"; import type { - AdapterId, InterruptionSample, OracleArtifacts, RestSweep, RetryAttempt, RunSample, ScheduledFault, TaskPlan, TaskRecord, TaskSample, + AdapterId, GovernanceRecord, InterruptionSample, OracleArtifacts, RestSweep, RetryAttempt, RunSample, ScheduledFault, TaskPlan, TaskRecord, + TaskSample, } from "./types.js"; import { restartProcessStartMatches } from "../runner-e2e/process-tree-owner.js"; import { runOracleCommand } from "./oracle-sandbox.js"; import { taskFaultSchedule } from "./scheduler.js"; -import { workloadDefinitions, type WorkloadAgents } from "./workloads.js"; +import { g1ChildTitle, g1Decision, workloadDefinitions, type WorkloadAgents } from "./workloads.js"; const IGNORED_DIRECTORIES = new Set([".git", ".bench", "__pycache__", "node_modules"]); const MAX_SNAPSHOT_FILES = 2_000; @@ -50,6 +51,11 @@ export const REST_CONFIRM_MS = 12_000; export const CHILD_WAKE_GRACE_MS = 60_000; /** Extra time granted after a Retry click before R3 is graded as timed out. */ export const RETRY_BUDGET_MS = 15 * 60_000; +/** H1: how long the pause must hold with the server ready before the board resumes. */ +export const BUDGET_PAUSE_HOLD_MS = 30_000; +/** H1: the raised budget's headroom above observed spend, in cents. */ +export const BUDGET_RESUME_HEADROOM_CENTS = 3_000; +const DECISION_NOTE = "Restart chaos harness: board decision after a control-plane restart."; type Json = Record; @@ -112,6 +118,22 @@ export function latestIso(values: ReadonlyArray): string | null { export const RUN_EVENT_PAGE = 500; +/** G1: an approval this task asked for (its title names the nonce; a redacted title still counts). */ +export function approvalTitleMatches(approval: Json, nonce: string): boolean { + const title = approval.payload?.title; + return typeof title !== "string" || title.includes(nonce); +} + +/** H1: runs that started at or after `since` and before `until` (epoch ms), across every sample. */ +export function runsStartedAfter(samples: readonly TaskSample[], since: number, until = Number.POSITIVE_INFINITY): string[] { + const started = new Set(); + for (const sample of samples) for (const run of sample.runs ?? []) { + const at = run.startedAt ? Date.parse(run.startedAt) : Number.NaN; + if (Number.isFinite(at) && at >= since && at < until) started.add(run.id); + } + return [...started]; +} + export function toInterruption(value: Json | null | undefined): InterruptionSample | null { if (!value) return null; return { state: String(value.state), canRetry: value.canRetry === true, retryWait: Boolean(value.retryWait), @@ -142,6 +164,8 @@ export class TaskObserver { private restCandidateSince: number | null = null; private childrenSettledAt: number | null = null; private stallSince: number | null = null; + /** First sample of the current stretch with the server ready (H1 holds the pause across it). */ + private readySince: number | null = null; private readonly faultSchedule: ScheduledFault[]; private readonly pendingFaults: ScheduledFault[]; done = false; @@ -150,7 +174,8 @@ export class TaskObserver { processAlive = (pid: number, startedAt: string) => this.lab.environmentKind === "local" && localProcessAlive(pid, startedAt); constructor(readonly plan: TaskPlan, private readonly api: ChaosApi, private readonly lab: ChaosLab, - private readonly timeline: Timeline, private readonly supervisor: { serverReady: boolean; faultInProgress: boolean }, + private readonly timeline: Timeline, + private readonly supervisor: { serverReady: boolean; faultInProgress: boolean; serverBuild?: "upgrade_from" | "current" }, /** Paths oracle commands may not read: the operator's home, the instance root (secrets key, database) and results. */ private readonly protectedRoots: readonly string[] = [os.homedir()]) { this.record = { plan, issueId: null, identifier: null, createdAt: null, launchError: null, samples: [], retry: null, @@ -199,6 +224,7 @@ export class TaskObserver { async launch(options: { assign?: boolean } = {}): Promise { const definition = this.definition; const assign = options.assign !== false; + if (this.plan.workload === "H1") await this.lowerBudgetToSpend(); try { const issue = await this.api.post(`/api/companies/${this.lab.companyId}/issues`, { title: definition.title(this.plan.nonce), @@ -210,6 +236,8 @@ export class TaskObserver { }); this.record.issueId = String(issue.id); this.record.identifier = issue.identifier ?? null; + // K14: which build launched the task, so a replay starts it on the same one. + if (this.supervisor.serverBuild) this.record.serverBuildAtLaunch = this.supervisor.serverBuild; this.record.createdAt = Date.now(); this.record.deadlineAt = this.record.createdAt + definition.budgetMs; this.timeline.add("task_launched", { taskId: this.plan.taskId, issueId: issue.id, identifier: issue.identifier, @@ -308,6 +336,8 @@ export class TaskObserver { return queued && !started ? first.at : "missed"; } case "tool": return this.record.benchFirstSeenAt ?? null; + case "approval": return this.record.governance?.approvalSeenAt ?? null; + case "paused": return this.record.governance?.budgetPausedAt ?? null; case "result": { // B8: the run that received the follow-up has accepted its result but // has not settled yet. A follow-up posted at rest cannot cross it. @@ -363,13 +393,16 @@ export class TaskObserver { if (this.done || !this.record.issueId) return; const sample = await this.sample(now); this.record.samples.push(sample); + this.readySince = sample.reachable && sample.serverReady ? (this.readySince ?? now) : null; await this.observeBenchProgress(); + await this.governanceStep(sample, now); if (await this.maybeFollowUp(sample, now)) { // The comment response is not the agent's answer: require fresh samples. this.restCandidateSince = null; return; } - this.stallSince = isStalledSample(sample) ? (this.stallSince ?? now) : null; + // A task waiting on the board (G1's approval, H1's paused company) is in a named wait, not a stall. + this.stallSince = isStalledSample(sample) && !this.awaitingBoard() ? (this.stallSince ?? now) : null; if (this.stallSince !== null && now - this.stallSince > STALL_GIVE_UP_MS) { this.record.restAt = this.stallSince; this.record.finalStatus = sample.issueStatus ?? null; @@ -383,7 +416,8 @@ export class TaskObserver { this.restCandidateSince ??= now; if (now - this.restCandidateSince >= REST_CONFIRM_MS) { await this.onRest(sample, now); - return; + // A task still waiting (for the board, or on a Retry it just got) keeps its deadline. + if (this.done) return; } } if (this.record.deadlineAt !== null && now > this.record.deadlineAt && !this.supervisor.faultInProgress) { @@ -456,6 +490,12 @@ export class TaskObserver { private async onRest(sample: TaskSample, now: number) { const status = sample.issueStatus ?? null; + // The board has not acted yet: the task waits on it, so it has not rested (and gets no Retry). + if (this.awaitingBoard()) { this.restCandidateSince = null; return; } + if (this.plan.workload === "H1") { + const company = await this.api.get(`/api/companies/${this.lab.companyId}`, 5_000).catch(() => null); + (this.record.governance ??= {}).companyActiveAtRest = company?.status === "active"; + } // C1: a follow-up posted after rest must be served before the task counts as resting. if (this.definition.followUp && !this.record.followUp) return; const faultSinceRetry = this.record.retry !== null && (this.record.lastFaultAt ?? 0) > this.record.retry.at; @@ -515,7 +555,128 @@ export class TaskObserver { workloadAgents(): WorkloadAgents { const self = this.lab.agents[this.plan.adapter]; const other = Object.values(this.lab.agents).find(agent => agent.id !== self.id) ?? self; - return { self, other }; + return { self, other, companyId: this.lab.companyId }; + } + + /** G1 before the board's decision, or H1 while its company is paused and not yet resumed. */ + private awaitingBoard(): boolean { + const governance = this.record.governance; + if (this.plan.workload === "G1") return Boolean(governance?.approvalSeenAt) && !governance?.decision; + if (this.plan.workload === "H1") return Boolean(governance?.budgetPausedAt) && !governance?.resume && !governance?.pauseLostAt; + return false; + } + + /** The board acts only after every fault this task planned has fired (or was missed), with the server ready. */ + private boardMayAct(sample: TaskSample): boolean { + return sample.reachable && sample.serverReady && !this.supervisor.faultInProgress && this.pendingFaults.length === 0; + } + + /** + * H1: set the company budget to its observed spend plus one cent through the board's budget + * route, so the task's first priced usage crosses the hard stop mid-run. + */ + private async lowerBudgetToSpend(): Promise { + const governance = this.record.governance ??= {}; + try { + const overview = await this.api.get(`/api/companies/${this.lab.companyId}/budgets/overview`, 10_000); + const policy = ((overview?.policies ?? []) as Json[]).find(row => row.scopeType === "company"); + const observed = typeof policy?.observedAmount === "number" ? policy.observedAmount : 0; + // The budget routes take whole cents; observed spend can be fractional. + governance.budgetSetCents = Math.ceil(observed) + 1; + await this.api.patch(`/api/companies/${this.lab.companyId}/budgets`, { budgetMonthlyCents: governance.budgetSetCents }); + this.timeline.add("governance_budget_set", { taskId: this.plan.taskId, budgetCents: governance.budgetSetCents, observedCents: observed }); + } catch (error) { + governance.budgetSetError = error instanceof Error ? error.message : String(error); + this.timeline.add("governance_budget_set_failed", { taskId: this.plan.taskId, error: governance.budgetSetError }); + } + } + + /** G1 and H1: what the board sees and does, through the same routes the board UI uses. */ + private async governanceStep(sample: TaskSample, now: number): Promise { + if (!sample.reachable || !this.record.issueId) return; + if (this.plan.workload === "G1") await this.approvalStep(sample, now); + if (this.plan.workload === "H1") await this.budgetStep(sample, now); + } + + private async approvalStep(sample: TaskSample, now: number): Promise { + const governance = this.record.governance ??= {}; + if (governance.decision) return; + const approvals = await this.api.get(`/api/issues/${this.record.issueId}/approvals`, 5_000).catch(() => null); + const pending = (approvals ?? []).find(approval => approval.status === "pending" && approvalTitleMatches(approval, this.plan.nonce)); + if (!pending) return; + if (governance.approvalSeenAt == null) { + governance.approvalSeenAt = now; + governance.approvalId = String(pending.id); + this.timeline.add("governance_approval_pending", { taskId: this.plan.taskId, approvalId: pending.id }); + } + if (!this.boardMayAct(sample)) return; + // The gated action must not exist before the board decides. + const before = await this.collectChildren(this.record.issueId!).catch(() => null); + governance.childrenBeforeDecision = before === null ? null + : before.filter(child => (child.title ?? "").includes(g1ChildTitle(this.plan.nonce))).length; + const kind = g1Decision(this.plan.nonce); + const decision: NonNullable = { kind, at: now, httpStatus: null, error: null, + pendingWhenDecided: true, afterFault: (this.record.lastFaultAt ?? 0) > governance.approvalSeenAt }; + try { + await this.api.post(`/api/approvals/${pending.id}/${kind}`, { decisionNote: DECISION_NOTE }); + // A rejected request also closes the task: the board cancels it. + if (kind === "reject") await this.api.patch(`/api/issues/${this.record.issueId}`, { status: "cancelled" }); + decision.httpStatus = 200; + } catch (error) { + decision.httpStatus = error instanceof ChaosApiError ? error.status : null; + decision.error = error instanceof Error ? error.message : String(error); + } + governance.decision = decision; + this.restCandidateSince = null; + this.timeline.add("governance_approval_decided", { taskId: this.plan.taskId, approvalId: pending.id, ...decision }); + } + + private async budgetStep(sample: TaskSample, now: number): Promise { + const governance = this.record.governance ??= {}; + if (governance.resume) return; + const company = await this.api.get(`/api/companies/${this.lab.companyId}`, 5_000).catch(() => null); + if (!company) return; + const paused = company.status === "paused" && company.pauseReason === "budget"; + // A pause must hold until the board resumes it: one that clears on its own (a restart lost it) + // is recorded, and the board does not resume what is no longer paused. + if (!paused && governance.budgetPausedAt != null) { + governance.pauseLostAt ??= now; + this.timeline.add("governance_budget_pause_lost", { taskId: this.plan.taskId, status: company.status ?? null, + pauseReason: company.pauseReason ?? null }); + return; + } + if (paused && governance.budgetPausedAt == null) { + const recorded = typeof company.pausedAt === "string" ? Date.parse(company.pausedAt) : Number.NaN; + governance.budgetPausedAt = Number.isFinite(recorded) ? Math.min(recorded, now) : now; + this.timeline.add("governance_budget_paused", { taskId: this.plan.taskId, pausedAt: company.pausedAt ?? null }); + } + if (governance.budgetPausedAt == null || !this.boardMayAct(sample)) return; + if (this.readySince === null || now - this.readySince < BUDGET_PAUSE_HOLD_MS || now - governance.budgetPausedAt < BUDGET_PAUSE_HOLD_MS) return; + // While paused nothing may start: every run that started after the pause is a violation. + governance.runsStartedWhilePaused = runsStartedAfter(this.record.samples, governance.budgetPausedAt); + const resume: NonNullable = { at: now, amount: null, incidentId: null, httpStatus: null, error: null, + afterFault: (this.record.lastFaultAt ?? 0) > governance.budgetPausedAt }; + try { + const overview = await this.api.get(`/api/companies/${this.lab.companyId}/budgets/overview`, 10_000); + const incident = ((overview?.activeIncidents ?? []) as Json[]).find(row => row.scopeType === "company"); + if (!incident) throw new Error("no active company budget incident to resolve"); + const policy = ((overview?.policies ?? []) as Json[]).find(row => row.scopeType === "company"); + const observed = Math.max(Number(incident.amountObserved) || 0, Number(policy?.observedAmount) || 0); + resume.incidentId = String(incident.id); + resume.amount = Math.ceil(observed) + BUDGET_RESUME_HEADROOM_CENTS; + await this.api.post(`/api/companies/${this.lab.companyId}/budget-incidents/${incident.id}/resolve`, + { action: "raise_budget_and_resume", amount: resume.amount, decisionNote: DECISION_NOTE }); + resume.httpStatus = 200; + } catch (error) { + resume.httpStatus = error instanceof ChaosApiError ? error.status : null; + resume.error = error instanceof Error ? error.message : String(error); + } + // Never before the poll that decided to resume (the pause window cannot end before it starts). + resume.doneAt = Math.max(Date.now(), now); + governance.resume = resume; + this.restCandidateSince = null; + this.timeline.add("governance_budget_resumed", { taskId: this.plan.taskId, ...resume, + runsStartedWhilePaused: governance.runsStartedWhilePaused }); } async childSpeedTasks(): Promise }>> { @@ -544,7 +705,8 @@ export class TaskObserver { const document = await this.api.get(`/api/issues/${row.id}/documents/${key}`).catch(() => null); documents[key] = typeof document?.body === "string" ? document.body : null; } - children.push({ id: String(row.id), status: typeof row.status === "string" ? row.status : null, + children.push({ id: String(row.id), title: typeof row.title === "string" ? row.title : null, + status: typeof row.status === "string" ? row.status : null, assigneeAgentId: typeof row.assigneeAgentId === "string" ? row.assigneeAgentId : null, documents }); } return children; @@ -588,9 +750,33 @@ export class TaskObserver { files, commands, workspacePath, ...(this.plan.workload === "B5" ? { benchProgressObserved: this.record.benchProgressSeen?.length ?? 0 } : {}), ...(definition.collect.childDocuments ? { children: await this.collectChildren(issueId) } : {}), + ...(this.plan.workload === "G1" ? { approvals: await this.collectApprovals(issueId) } : {}), + ...(this.plan.workload === "H1" ? await this.finalPauseCheck(issueId) : {}), + ...(this.record.governance ? { governance: this.record.governance } : {}), }; } + /** + * H1: the pause window ends when the resume request returned, after the last poll before it. Read + * the task's whole run history again, so a run that started in that gap is still caught. + */ + private async finalPauseCheck(issueId: string): Promise> { + const governance = this.record.governance; + if (!governance?.budgetPausedAt) return {}; + const runs = await this.api.get(`/api/issues/${issueId}/runs`).then(rows => (rows ?? []).map(toRunSample)).catch(() => []); + const history: TaskSample[] = [...this.record.samples, { at: Date.now(), reachable: true, serverReady: true, runs }]; + governance.runsStartedWhilePaused = runsStartedAfter(history, governance.budgetPausedAt, + governance.resume?.doneAt ?? Number.POSITIVE_INFINITY); + return {}; + } + + /** G1: every approval linked to the task, whatever its status. */ + private async collectApprovals(issueId: string): Promise> { + const rows = await this.api.get(`/api/issues/${issueId}/approvals`).catch(() => []); + return (rows ?? []).map(row => ({ id: String(row.id), status: String(row.status), + title: typeof row.payload?.title === "string" ? row.payload.title : null })); + } + /** R6 sweep: nothing still pending once the wave is quiet. */ async sweep(): Promise { this.closeUnfiredTriggers(); diff --git a/tests/restart-chaos/scheduler.test.ts b/tests/restart-chaos/scheduler.test.ts index b0b9bb0dcc1..525cc04228d 100644 --- a/tests/restart-chaos/scheduler.test.ts +++ b/tests/restart-chaos/scheduler.test.ts @@ -46,6 +46,22 @@ describe("scheduler determinism", () => { expect(() => parseCommand(["--replay", "w-20261106T101112-s42/t03", "--k4-hold-max-seconds", "400"])).toThrow(/do not combine/); }); + it("pairs K14 (upgrade mid-run) with --upgrade-from, locally, with no hold and outside storm defaults", () => { + const command = parseCommand(["--faults", "K14", "--upgrade-from", "../older-checkout", "--seed", "5"]); + if (command.mode !== "wave") throw new Error("expected a wave"); + expect(command.options.upgradeFrom).toMatch(/older-checkout$/); + expect(command.options.upgradeFrom!.startsWith("/")).toBe(true); + expect(planTasks(command.options).every(task => task.fault === "K14" && task.holdMs === 0)).toBe(true); + expect(() => parseCommand(["--faults", "K14"])).toThrow(/go together/); + expect(() => parseCommand(["--faults", "K1", "--upgrade-from", "../older-checkout"])).toThrow(/go together/); + expect(() => parseCommand(["--faults", "K14", "--upgrade-from", "x", "--env", "daytona", + "--daytona-image", `ghcr.io/x/y@sha256:${"a".repeat(64)}`])).toThrow(/only with --env local/); + const storm = parseCommand(["--storm-minutes", "30"]); + if (storm.mode !== "wave") throw new Error("expected a wave"); + expect(storm.options.faults).not.toContain("K14"); + expect(() => parseCommand(["--replay", "w-20261106T101112-s42/t03", "--upgrade-from", "x"])).toThrow(/do not combine/); + }); + it("covers every workload × adapter × fault cell once per cycle", () => { const cells = options.workloads.length * options.adapters.length * options.faults.length; const tasks = planTasks({ ...options, tasks: cells }); diff --git a/tests/restart-chaos/scheduler.ts b/tests/restart-chaos/scheduler.ts index 9d3ed10700c..63b5131408a 100644 --- a/tests/restart-chaos/scheduler.ts +++ b/tests/restart-chaos/scheduler.ts @@ -42,14 +42,21 @@ export const TRIGGER_DELAY_RANGE_MS: Record, readonly queued: [0, 0], tool: [15_000, 60_000], result: [0, 3_000], + approval: [0, 3_000], + paused: [0, 3_000], }; export function triggerFitsWorkload(trigger: TriggerId, workload: WorkloadId): boolean { if (trigger === "comment" || trigger === "result") return workload === "C1"; if (trigger === "tool") return workload === "B5"; + if (trigger === "approval") return workload === "G1"; + if (trigger === "paused") return workload === "H1"; return true; } +/** The workload a trigger needs, for the error when none is selected. */ +const TRIGGER_WORKLOAD: Partial> = { comment: "C1", result: "C1", tool: "B5", approval: "G1", paused: "H1" }; + /** Faults a `result` task may fire inside a storm: a fast restart that lands on the boundary. */ const STORM_RESULT_FAULTS: readonly FaultKind[] = ["K1", "K2"]; @@ -76,7 +83,7 @@ export function planTasks(options: Pick triggerFitsWorkload(trigger, workload))) { - throw new Error(`Trigger "${trigger}" needs workload ${trigger === "comment" || trigger === "result" ? "C1" : "B5"} in --workloads`); + throw new Error(`Trigger "${trigger}" needs workload ${TRIGGER_WORKLOAD[trigger] ?? "?"} in --workloads`); } } const cells: Array<{ workload: WorkloadId; adapter: AdapterId; fault: TaskFaultKind; trigger: TriggerId }> = []; @@ -224,7 +231,19 @@ export function stormFaultDue(storm: StormPlan, state: StormState, now: number): /** Written after every fault: each task's launch time and the faults that crossed it so far. */ export const TASK_FAULT_LOG = "task-faults.json"; -export type TaskFaultLog = Record; +export type TaskFaultLog = Record; + +/** + * A replay of a K14 wave task that launched on the current build: it starts on the current build + * too (no `--upgrade-from`), and its recorded K14 restarts were current-build restarts (K2). + */ +export function replayOnCurrentBuild(plan: WavePlan): WavePlan { + const asK2 = (item: T): T => ({ ...item, + ...(item.kind === "K14" ? { kind: "K2" as const } : {}), ...(item.fault === "K14" ? { fault: "K2" as const } : {}) }); + const { upgradeFrom: _upgradeFrom, ...options } = plan.options; + return { ...plan, options: { ...options, faults: options.faults.map(kind => kind === "K14" ? "K2" : kind) }, + tasks: plan.tasks.map(task => ({ ...asK2(task), ...(task.replayFaults ? { replayFaults: task.replayFaults.map(asK2) } : {}) })) }; +} /** The crossing record for a replay: the graded record, else the live fault log (interrupted or wedged waves). */ export function replayRecordFor(taskId: string, graded: { createdAt: number | null; faultIds: string[] } | null, diff --git a/tests/restart-chaos/supervisor.ts b/tests/restart-chaos/supervisor.ts index b1c218d4a3a..53d5b8a26f5 100644 --- a/tests/restart-chaos/supervisor.ts +++ b/tests/restart-chaos/supervisor.ts @@ -13,7 +13,7 @@ * database (that is K9). The controller runtime cache is kept: discarding it is * the separate K11 fault. */ -import { spawn, type ChildProcess } from "node:child_process"; +import { execFileSync, spawn, type ChildProcess } from "node:child_process"; import { randomBytes } from "node:crypto"; import { createWriteStream, readFileSync, statSync, writeFileSync, type WriteStream } from "node:fs"; import { mkdir, readFile, symlink } from "node:fs/promises"; @@ -44,7 +44,12 @@ const HEALTH_TIMEOUT_MS = (() => { const READY_TIMEOUT_MS = 300_000; const STOP_TIMEOUT_MS = 30_000; -export const FAULT_SIGNAL: Record = { K1: "SIGKILL", K2: "SIGTERM", K4: "SIGKILL", K6: "SIGKILL" }; +export const FAULT_SIGNAL: Record = { K1: "SIGKILL", K2: "SIGTERM", K4: "SIGKILL", K6: "SIGKILL", K14: "SIGTERM" }; + +/** The checkout's commit, for the K14 receipt. */ +function checkoutSha(root: string): string | null { + try { return execFileSync("git", ["rev-parse", "HEAD"], { cwd: root, encoding: "utf8" }).trim(); } catch { return null; } +} export const CRASH_LOOP_TARGETS = ["recovering", "ready"] as const; const PHASE_POLL_MS = 50; @@ -117,9 +122,17 @@ export class ChaosSupervisor { onCrashPoint: ((receipt: FaultReceipt) => void) | null = null; /** Crash-point suite: keep the control plane down this long after a self-kill (a K4-style hold). */ crashPointHoldMs = 0; + /** The checkout the server runs from: the older build (`--upgrade-from`) until the first K14, then this one. */ + private serverRoot: string; + /** The environment overrides the wave passed to `start`, kept for every restart and for the K14 switch. */ + private extraEnvironment: Record = {}; + /** Wave secrets, kept across a K14 switch as an in-place upgrade keeps its configuration. */ + private readonly secrets = { PAPERCLIP_AGENT_JWT_SECRET: secret(), PAPERCLIP_DECISION_SIGNING_SECRET: secret(), + PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: secret(), BETTER_AUTH_SECRET: secret() }; private constructor(readonly repositoryRoot: string, readonly port: number, readonly paths: SupervisorPaths, - private readonly timeline: Timeline) { + private readonly timeline: Timeline, upgradeFrom: string | null) { + this.serverRoot = upgradeFrom ?? repositoryRoot; this.baseUrl = `http://127.0.0.1:${port}`; this.log = createWriteStream(paths.serverLogPath, { flags: "a", mode: 0o600 }); this.stopper = createRunnerE2EServerStopper({ @@ -130,34 +143,57 @@ export class ChaosSupervisor { }); } - static async create(input: { repositoryRoot: string; waveDir: string; timeline: Timeline }): Promise { + static async create(input: { repositoryRoot: string; waveDir: string; timeline: Timeline; upgradeFrom?: string }): Promise { const temporaryRoot = await createRunnerE2ETemporaryRoot(); const home = path.join(temporaryRoot, "home"); const paths: SupervisorPaths = { temporaryRoot, home, configPath: path.join(home, "instances", "default", "config.json"), serverLogPath: path.join(input.waveDir, "server.log") }; const port = await reserveRunnerE2EServerPort(); - const supervisor = new ChaosSupervisor(input.repositoryRoot, port, paths, input.timeline); - input.timeline.add("supervisor_created", { temporaryRoot, port }); + const supervisor = new ChaosSupervisor(input.repositoryRoot, port, paths, input.timeline, input.upgradeFrom ?? null); + input.timeline.add("supervisor_created", { temporaryRoot, port, ...(input.upgradeFrom + ? { upgradeFrom: input.upgradeFrom, upgradeFromSha: checkoutSha(input.upgradeFrom) } : {}) }); return supervisor; } + /** K14 waves: whether the older build or this checkout is serving. */ + get serverBuild(): "upgrade_from" | "current" { return this.serverRoot === this.repositoryRoot ? "current" : "upgrade_from"; } + serverLogOffset(): number { try { return statSync(this.paths.serverLogPath).size + this.log.writableLength; } catch { return 0; } } /** `extraEnvironment` is kept for every restart (e.g. the Daytona image's runnerd and provider pack paths). */ async start(extraEnvironment: Record = {}): Promise { - const { temporaryRoot, home, configPath } = this.paths; + const { temporaryRoot, configPath } = this.paths; const reservation = await prepareRunnerE2EServerConfig({ temporaryRoot, configPath, serverPort: this.port }); await reservation?.close(); // Wrapper-owned Postgres: survives K1/K2/K4 like a managed database would. this.database = await prepareControllerDatabase(configPath, temporaryRoot); this.timeline.add("database_started", {}); + this.extraEnvironment = extraEnvironment; + await this.prepareServerEnvironment(this.serverRoot); + const child = await this.spawnServer(); + await this.waitForHealth(child); + await this.waitForReadyPhase([]); + this.timeline.add("server_ready", { pid: child.pid, serverRoot: this.serverRoot }); + } + + /** + * The server environment for one checkout: its own runner binary, pinned Codex and source + * dependencies. The database, instance directory, port and secrets stay the same across a K14 + * switch, as an in-place upgrade keeps them. + */ + private async prepareServerEnvironment(root: string): Promise { + const { temporaryRoot, home, configPath } = this.paths; + const extraEnvironment = this.extraEnvironment; const runnerBinary = resolvePaperclipRunnerBinaryForHarness( - [{ profile: { generation: "native" } } as unknown as MatrixExecution], this.repositoryRoot); + [{ profile: { generation: "native" } } as unknown as MatrixExecution], root); + // One provider PATH per checkout: the older build keeps its own pinned Codex. + const providerRoot = root === this.repositoryRoot ? temporaryRoot : path.join(temporaryRoot, "upgrade-from"); + await mkdir(providerRoot, { recursive: true }); const source = { ...buildRunnerE2EProcessEnvironment(process.env, []), - PATH: await prepareProviderPath(this.repositoryRoot, temporaryRoot, process.env.PATH), + PATH: await prepareProviderPath(root, providerRoot, process.env.PATH), PAPERCLIP_RUNNER_E2E_EXECUTION_IDS: "[]", PAPERCLIP_RUNNER_BINARY: runnerBinary, PAPERCLIP_VITE_CACHE_DIR: path.join(temporaryRoot, "vite-cache"), @@ -165,8 +201,7 @@ export class ChaosSupervisor { const overrides = { NODE_ENV: "test", PORT: String(this.port), XDG_CACHE_HOME: path.join(temporaryRoot, "xdg-cache"), PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: configPath, PAPERCLIP_INSTANCE_ID: "default", - PAPERCLIP_AGENT_JWT_SECRET: secret(), PAPERCLIP_DECISION_SIGNING_SECRET: secret(), - PAPERCLIP_TOOL_ACTION_SIGNING_SECRET: secret(), BETTER_AUTH_SECRET: secret(), + ...this.secrets, PAPERCLIP_BIND: "loopback", PAPERCLIP_BIND_HOST: "127.0.0.1", PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", PAPERCLIP_DEPLOYMENT_EXPOSURE: "private", SERVE_UI: "true", PAPERCLIP_STORAGE_PROVIDER: "local_disk", PAPERCLIP_STORAGE_LOCAL_DIR: path.join(temporaryRoot, "storage"), PAPERCLIP_SECRETS_PROVIDER: "local_encrypted", @@ -181,21 +216,18 @@ export class ChaosSupervisor { const environment = buildPaperclipServerEnvironment(source, overrides); assertIsolatedServerEnvironment(environment, { temporaryRoot, paperclipHome: home, configPath }); this.environment = Object.fromEntries(Object.entries(environment).filter((entry): entry is [string, string] => entry[1] !== undefined)); - await prepareSourceServerDependencies(this.repositoryRoot, this.environment, false); - const child = await this.spawnServer(); - await this.waitForHealth(child); - await this.waitForReadyPhase([]); - this.timeline.add("server_ready", { pid: child.pid }); + await prepareSourceServerDependencies(root, this.environment, false); } private async spawnServer(): Promise { if (this.shuttingDown) throw new Error("Refusing to start Paperclip during harness shutdown"); - const entry = path.join(this.repositoryRoot, "tests/runner-e2e/server-entry.ts"); + const root = this.serverRoot; + const entry = path.join(root, "tests/runner-e2e/server-entry.ts"); const control = this.crashPointControl; const env = control ? { ...this.environment, PAPERCLIP_CRASH_POINT_CONTROL: control.controlPath, PAPERCLIP_CRASH_POINT_RECEIPT: control.receiptPath } : this.environment; - const child = spawn(process.execPath, runnerE2ETypeScriptProcessArgs(this.repositoryRoot, entry, ["onboard", "--yes", "--run"]), - { cwd: this.repositoryRoot, env, stdio: ["ignore", "pipe", "pipe"], detached: runnerE2EServerDetached }); + const child = spawn(process.execPath, runnerE2ETypeScriptProcessArgs(root, entry, ["onboard", "--yes", "--run"]), + { cwd: root, env, stdio: ["ignore", "pipe", "pipe"], detached: runnerE2EServerDetached }); this.child = child; child.stdout?.on("data", chunk => this.log.write(chunk)); @@ -389,7 +421,13 @@ export class ChaosSupervisor { waitForUnhealthy: async () => { await this.waitForHealthToStop(); receipt.stoppedAt = Date.now(); this.timeline.add("fault_server_down", { faultId: receipt.faultId }); }, // K1/K2/K4 keep the controller runtime cache; K11 will discard it. discardCache: async () => {}, - start: async () => { receipt.restartedAt = Date.now(); this.timeline.add("fault_restarting", { faultId: receipt.faultId }); return this.spawnServer(); }, + start: async () => { + receipt.restartedAt = Date.now(); + // K14: the replacement is the current build, on the same database and instance directory. + if (kind === "K14") receipt.upgrade = await this.switchToCurrentBuild(receipt.faultId); + this.timeline.add("fault_restarting", { faultId: receipt.faultId }); + return this.spawnServer(); + }, waitForReady: async replacement => { receipt.healthyAt = await this.waitForHealth(replacement, receipt.phases); }, cancelled: () => this.shuttingDown, record: async progress => { outage = progress; }, @@ -412,6 +450,23 @@ export class ChaosSupervisor { return receipt; } + /** + * K14: move the server from the older build to this checkout. Only the first K14 switches; a later + * one restarts the current build (recorded with `switched: false`). + */ + private async switchToCurrentBuild(faultId: string): Promise> { + const fromRoot = this.serverRoot; + const switched = fromRoot !== this.repositoryRoot; + if (switched) { + this.serverRoot = this.repositoryRoot; + await this.prepareServerEnvironment(this.repositoryRoot); + } + const upgrade = { fromRoot, fromSha: checkoutSha(fromRoot), toRoot: this.repositoryRoot, toSha: checkoutSha(this.repositoryRoot), switched }; + this.timeline.add("upgrade_switched", { faultId, ...upgrade }); + this.log.write(`\n[restart-chaos] ${faultId} K14 upgrade ${upgrade.fromSha ?? fromRoot} -> ${upgrade.toSha ?? this.repositoryRoot}${switched ? "" : " (already upgraded)"}\n`); + return upgrade; + } + private async waitForCrashLoopPhase(child: ChildProcess, target: (typeof CRASH_LOOP_TARGETS)[number], phases: FaultReceipt["phases"]): Promise { const deadline = Date.now() + READY_TIMEOUT_MS; diff --git a/tests/restart-chaos/types.ts b/tests/restart-chaos/types.ts index 9e9776270f8..ea36a6305a8 100644 --- a/tests/restart-chaos/types.ts +++ b/tests/restart-chaos/types.ts @@ -1,17 +1,19 @@ /** Shared types for the control-plane restart chaos harness (plan §5). */ -export const FAULT_KINDS = ["K1", "K2", "K4", "K6"] as const; +/** K14 (upgrade mid-run): a graceful stop, then the replacement starts from the current build (`--upgrade-from`). */ +export const FAULT_KINDS = ["K1", "K2", "K4", "K6", "K14"] as const; export type FaultKind = (typeof FAULT_KINDS)[number]; /** A task's planned fault. "K13" (restart storm) tasks have none of their own; the wave's storm crosses them. */ export type TaskFaultKind = FaultKind | "K13"; -export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1"] as const; +/** G1: an approval-gated action across a restart. H1: a budget hard stop crossed mid-run with a restart (its own wave). */ +export const WORKLOAD_IDS = ["A1", "B1", "B5", "C1", "M1", "G1", "H1"] as const; export type WorkloadId = (typeof WORKLOAD_IDS)[number]; export const ADAPTER_IDS = ["codex", "claude", "opencode"] as const; export type AdapterId = (typeof ADAPTER_IDS)[number]; -export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool", "result"] as const; +export const TRIGGER_IDS = ["time", "comment", "retry", "queued", "tool", "result", "approval", "paused"] as const; export type TriggerId = (typeof TRIGGER_IDS)[number]; export const ENVIRONMENT_KINDS = ["local", "daytona"] as const; @@ -64,6 +66,12 @@ export interface WaveOptions { crashPoint?: string; /** Crash-point suite: hold the control plane down this long after each self-kill (K4-style). */ crashPointHoldMs?: number; + /** + * K14 upgrade mid-run (`--upgrade-from `): an older, prepared checkout. The wave's server starts + * from it; the first K14 stops it gracefully and starts the current checkout on the same database + * and instance directory. Later K14 faults restart the current build, like K2. + */ + upgradeFrom?: string; } export interface StormFault { @@ -195,6 +203,8 @@ export interface FaultReceipt { crashLoop?: Array<{ pid: number | null; target: string; phase: string; at: number }>; /** Crash-point suite: the server killed itself at this named point (no external signal). */ crashPoint?: { name: string; hit: number; pid: number; token?: string }; + /** K14: the build the stopped server ran and the build its replacement runs. */ + upgrade?: { fromRoot: string; fromSha: string | null; toRoot: string; toSha: string | null; switched: boolean }; serverLogOffsetAtStart: number; serverLogOffsetAtReady: number | null; error: string | null; @@ -209,7 +219,39 @@ export interface OracleArtifacts { workspacePath: string | null; /** B5: distinct `.bench/progress` values the harness itself saw while the task was live. */ benchProgressObserved?: number; - children?: Array<{ id: string; status: string | null; assigneeAgentId: string | null; documents: Record }>; + children?: Array<{ id: string; title?: string | null; status: string | null; assigneeAgentId: string | null; documents: Record }>; + /** G1: approvals linked to the task. */ + approvals?: Array<{ id: string; status: string; title: string | null }>; + /** G1 and H1: the board's actions and what the observer saw around them. */ + governance?: GovernanceRecord; +} + +/** What the board did for a governance task (G1, H1), and what the observer saw. */ +export interface GovernanceRecord { + /** G1: first time the observer saw the task's approval pending. */ + approvalSeenAt?: number | null; + approvalId?: string | null; + /** G1: the board's decision, made once the task's faults fired and the server was ready. */ + decision?: { kind: "approve" | "reject"; at: number; httpStatus: number | null; error: string | null; + /** The approval was still pending when the board decided (it survived the restart). */ + pendingWhenDecided: boolean; afterFault: boolean }; + /** H1: the company budget the harness set before launch, in cents (observed spend + 1). */ + budgetSetCents?: number | null; + budgetSetError?: string | null; + /** H1: first time the observer saw the company paused for its budget. */ + budgetPausedAt?: number | null; + /** H1: runs of the task that started while the company was paused (must be none). */ + runsStartedWhilePaused?: string[]; + /** H1: the board's raise-and-resume, made once the task's faults fired and the pause held. */ + resume?: { at: number; amount: number | null; incidentId: string | null; httpStatus: number | null; error: string | null; afterFault: boolean; + /** When the resume request returned: the pause window ends here. */ + doneAt?: number }; + /** H1: the company was active again at rest. */ + companyActiveAtRest?: boolean; + /** H1: the pause cleared on its own (before the board resumed it). */ + pauseLostAt?: number | null; + /** G1: subtasks named after the nonce that existed before the board decided (must be 0). */ + childrenBeforeDecision?: number | null; } export interface OracleResult { verdict: Exclude; checks: Array<{ name: string; ok: boolean; detail?: string }> } @@ -257,6 +299,9 @@ export interface TaskRecord { /** The run whose result armed the `result` trigger; the fault fires only while it is still running. */ resultRunId?: string | null; triggerMissed?: TriggerId[]; + governance?: GovernanceRecord; + /** K14 waves: the build serving when the task launched. */ + serverBuildAtLaunch?: "upgrade_from" | "current"; } export interface TaskGrade { diff --git a/tests/restart-chaos/workloads.ts b/tests/restart-chaos/workloads.ts index e694a0e2547..d78efa993df 100644 --- a/tests/restart-chaos/workloads.ts +++ b/tests/restart-chaos/workloads.ts @@ -22,7 +22,14 @@ export interface WorkloadDefinition { grade(nonce: string, artifacts: OracleArtifacts, agents?: WorkloadAgents): OracleResult; } -export interface WorkloadAgents { self: { id: string; name: string }; other: { id: string; name: string } } +export interface WorkloadAgents { self: { id: string; name: string }; other: { id: string; name: string }; companyId?: string } + +/** G1: the board's decision for this task, fixed by its nonce (about half approve, half reject). */ +export function g1Decision(nonce: string): "approve" | "reject" { + return Number.parseInt(nonce.slice(0, 1), 16) % 2 === 0 ? "approve" : "reject"; +} +export const g1ApprovalTitle = (nonce: string) => `Approve G1-${nonce}`; +export const g1ChildTitle = (nonce: string) => `Approved G1-${nonce}`; const partMarker = (nonce: string, part: number) => `PART${part}-${nonce}`; @@ -85,7 +92,83 @@ const delegateDescription = (nonce: string, agents?: WorkloadAgents) => { ].join("\n"); }; +const approvalDescription = (nonce: string, agents?: WorkloadAgents) => [ + "This task needs the board's approval before you act.", + "1. Request approval exactly once with the `call_api` tool: operationId `POST /api/companies/{companyId}/approvals`," + + ` pathParams \`{ "companyId": "${agents?.companyId ?? ""}" }\`, and this body:`, + ` \`{ "type": "request_board_approval", "issueIds": [""], "payload": { "title": "${g1ApprovalTitle(nonce)}",` + + ` "summary": "Create the follow-up subtask for ${nonce}.", "recommendedAction": "Approve", "risks": [] } }\``, + "2. Then set this task's status to `blocked` and end your turn. Do not create anything else yet.", + `3. Only if you are woken because that approval was approved: create exactly one subtask titled "${g1ChildTitle(nonce)}"`, + " with this task as its parent and status `backlog`, unassigned. Then mark this task done.", + "If the approval is rejected, do not create the subtask.", +].join("\n"); + +function approvalChecks(nonce: string, artifacts: OracleArtifacts) { + const decision = g1Decision(nonce); + const governance = artifacts.governance; + const approvals = (artifacts.approvals ?? []).filter(approval => approval.title === null || approval.title.includes(nonce)); + const children = (artifacts.children ?? []).filter(child => (child.title ?? "").includes(g1ChildTitle(nonce))); + return [ + { name: "issue_status", ok: artifacts.issueStatus === (decision === "approve" ? "done" : "cancelled"), + detail: `decision=${decision} status=${artifacts.issueStatus ?? "unknown"}` }, + // The agent's create is not idempotent: a restart that makes it ask twice is a finding. + { name: "one_approval_requested", ok: approvals.length === 1, detail: `approvals=${approvals.length}` }, + { name: "board_decided_after_restart", ok: governance?.decision !== undefined && governance.decision.error === null + && governance.decision.afterFault, detail: JSON.stringify(governance?.decision ?? null) }, + { name: "approval_pending_across_restart", ok: governance?.decision?.pendingWhenDecided === true }, + { name: "action_gated_by_approval", ok: governance?.childrenBeforeDecision === 0, + detail: `subtasks_before_decision=${governance?.childrenBeforeDecision ?? "not checked"}` }, + { name: "approval_decided", ok: approvals.length === 1 && approvals[0]!.status === (decision === "approve" ? "approved" : "rejected"), + detail: approvals.map(approval => approval.status).join(",") }, + { name: decision === "approve" ? "approved_action_ran_once" : "rejected_action_never_ran", + ok: children.length === (decision === "approve" ? 1 : 0), detail: `subtasks=${children.length}` }, + ]; +} + +function budgetChecks(artifacts: OracleArtifacts) { + const governance = artifacts.governance; + return [ + { name: "budget_set", ok: typeof governance?.budgetSetCents === "number" && !governance.budgetSetError, + detail: governance?.budgetSetError ?? `budget=${governance?.budgetSetCents ?? "unset"}` }, + { name: "company_paused_for_budget", ok: typeof governance?.budgetPausedAt === "number" }, + { name: "pause_held_until_resume", ok: typeof governance?.budgetPausedAt === "number" && !governance.pauseLostAt }, + { name: "nothing_dispatched_while_paused", ok: governance?.runsStartedWhilePaused !== undefined && governance.runsStartedWhilePaused.length === 0, + detail: (governance?.runsStartedWhilePaused ?? ["not checked"]).join(",") }, + { name: "board_resumed_after_restart", ok: governance?.resume !== undefined && governance.resume.error === null && governance.resume.afterFault, + detail: JSON.stringify(governance?.resume ?? null) }, + { name: "company_active_at_rest", ok: governance?.companyActiveAtRest === true }, + ]; +} + export const workloadDefinitions: Record = { + G1: { + id: "G1", + label: "Approval-gated action across a restart", + usesRepository: false, + // The `approval` trigger fires on the pending approval; a `time` fault lands around it. + fireWindowMs: [20_000, 90_000], + budgetMs: 15 * 60_000, + title: nonce => `Chaos G1 approval ${nonce}`, + description: approvalDescription, + collect: { documents: [], files: () => [], commands: [], childDocuments: [] }, + grade(nonce, artifacts) { + return result(approvalChecks(nonce, artifacts)); + }, + }, + H1: { + id: "H1", + label: "Budget hard stop crossed mid-run, across a restart", + usesRepository: false, + fireWindowMs: [15_000, 60_000], + budgetMs: 15 * 60_000, + title: nonce => `Chaos H1 budget ${nonce}`, + description: briefDescription, + collect: { documents: ["brief"], files: () => [], commands: [] }, + grade(nonce, artifacts) { + return result([statusCheck(artifacts), ...briefChecks(artifacts.documents.brief, [briefMarker(nonce)]), ...budgetChecks(artifacts)]); + }, + }, M1: { id: "M1", label: "Delegate to two subtasks",