From 9cc3760552522171f4ded98d63bce807abe69ff9 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 7 Nov 2026 03:11:10 -0600 Subject: [PATCH] test(runner-e2e): allow controller outages up to ten minutes (#19797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The restart chaos harness and the runner-e2e recover-controller cells stop the control plane with `replaceController` (`tests/runner-e2e/controller-outage.ts`), hold it down, then start it again > - `parseControllerOutage` refuses holds longer than 300000 ms > - Restart chaos needs longer holds. One example is proving that a Daytona sandbox survives an outage longer than its old 5-minute auto-stop but inside the 7-minute runner reconnect grace (RC-54) > - With the cap, such a fault errors before the controller stops, so the scenario cannot run > - This pull request raises the test-harness cap to 600000 ms > - The benefit is that chaos runs can now inject 5–10 minute outages; production code and qualification contracts are unchanged ## Linked Issues or Issue Description **Bug description** A restart chaos replay with a 360000 ms K4 hold failed with "Controller outage requires SIGTERM/SIGKILL and 0..300000 integer milliseconds". The controller was never stopped. **Expected behavior** The test harness accepts outages of up to 10 minutes. The recover-controller receipt check keeps its own readiness limit (`readyAfterMs < 420000`). Refs #19544 ## What Changed - `tests/runner-e2e/controller-outage.ts`: adds `MAX_CONTROLLER_OUTAGE_MS = 600_000` and uses it in `parseControllerOutage` and its error message. `validControllerOutageReceipt` is unchanged and still requires readiness inside the 7-minute grace. - `tests/runner-e2e/controller-outage.test.ts`: the rejection cases now use 600001. A new test accepts 360000 (SIGKILL) and 600000 (SIGTERM). The existing five-minute contract test is unchanged. ## Verification - `vitest run tests/runner-e2e/controller-outage.test.ts tests/runner-e2e/server-restart-request.test.ts`: 52/52 pass. - `server-restart-request.test.ts` now uses 600001 as its out-of-range `controllerOutage`. The separate restart-request `controllerDowntimeMs` limit (five minutes) is unchanged. ## Risks - This change is test-only. No product or server code changes. - A suite that sets `PAPERCLIP_RUNNER_E2E_CONTROLLER_OUTAGE_MS` above 300000 now runs instead of being refused. The recover-controller receipt check still fails any receipt whose readiness is outside the grace. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, in Claude Code with tool use (shell, file edits, GitHub CLI). Default context window, extended thinking. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- tests/runner-e2e/controller-outage.test.ts | 8 ++++++-- tests/runner-e2e/controller-outage.ts | 12 +++++++++++- tests/runner-e2e/server-restart-request.test.ts | 2 +- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/controller-outage.test.ts b/tests/runner-e2e/controller-outage.test.ts index d366c3f1096..f95778b617c 100644 --- a/tests/runner-e2e/controller-outage.test.ts +++ b/tests/runner-e2e/controller-outage.test.ts @@ -88,10 +88,14 @@ describe("controller outage qualification", () => { expect(f.input.start).not.toHaveBeenCalled(); }); - it.each([-1, 300001, 0.5, NaN, Infinity, "300000", null])("rejects invalid duration %s before signaling", holdMs => { + it.each([-1, 600001, 0.5, NaN, Infinity, "300000", null])("rejects invalid duration %s before signaling", holdMs => { expect(() => parseControllerOutage({ signal: "SIGKILL", holdMs })).toThrow(); }); - it.each(["", "-1", "1e3", "NaN", " 1000", "300001"])("rejects invalid environment duration %s", value => { + it("accepts chaos holds up to ten minutes", () => { + expect(parseControllerOutage({ signal: "SIGKILL", holdMs: 360_000 })).toEqual({ signal: "SIGKILL", holdMs: 360_000 }); + expect(parseControllerOutage({ signal: "SIGTERM", holdMs: 600_000 })).toEqual({ signal: "SIGTERM", holdMs: 600_000 }); + }); + it.each(["", "-1", "1e3", "NaN", " 1000", "600001"])("rejects invalid environment duration %s", value => { expect(() => configuredControllerOutage({ PAPERCLIP_RUNNER_E2E_CONTROLLER_OUTAGE_MS: value })).toThrow(); }); it("accepts the five-minute contract without extending the runner grace", () => { diff --git a/tests/runner-e2e/controller-outage.ts b/tests/runner-e2e/controller-outage.ts index 758f80a8c33..2f07b834089 100644 --- a/tests/runner-e2e/controller-outage.ts +++ b/tests/runner-e2e/controller-outage.ts @@ -26,11 +26,21 @@ export interface ControllerOutageReceipt extends ControllerOutage { recoveryProgress?: Partial; } +/** + * Longest controller outage a harness may inject. Ten minutes lets chaos + * holds pass Daytona's sandbox auto-stop and the 7-minute runner reconnect + * grace. The recover-controller receipt check below still requires readiness + * inside that grace, so this does not loosen the qualification contract. + */ +export const MAX_CONTROLLER_OUTAGE_MS = 600_000; + export function parseControllerOutage(value: unknown): ControllerOutage { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid controller outage"); const { signal, holdMs } = value as Record; if ((signal !== "SIGTERM" && signal !== "SIGKILL") || !Number.isSafeInteger(holdMs) - || Number(holdMs) < 0 || Number(holdMs) > 300_000) throw new Error("Controller outage requires SIGTERM/SIGKILL and 0..300000 integer milliseconds"); + || Number(holdMs) < 0 || Number(holdMs) > MAX_CONTROLLER_OUTAGE_MS) { + throw new Error(`Controller outage requires SIGTERM/SIGKILL and 0..${MAX_CONTROLLER_OUTAGE_MS} integer milliseconds`); + } return { signal, holdMs: Number(holdMs) }; } diff --git a/tests/runner-e2e/server-restart-request.test.ts b/tests/runner-e2e/server-restart-request.test.ts index bfec299ead9..958c9face87 100644 --- a/tests/runner-e2e/server-restart-request.test.ts +++ b/tests/runner-e2e/server-restart-request.test.ts @@ -62,7 +62,7 @@ it("rejects cache replacement for unselected, local and legacy cells", async () it.each([ { controllerOutage: { signal: "SIGKILL", holdMs: 125_000 }, discardControllerCache: false }, - { controllerOutage: { signal: "SIGKILL", holdMs: 300_001 }, discardControllerCache: true }, + { controllerOutage: { signal: "SIGKILL", holdMs: 600_001 }, discardControllerCache: true }, { controllerOutage: { signal: "SIGKILL", holdMs: 125_000 }, discardControllerCache: true, preserveRunner: {} }, { controllerOutage: { signal: "SIGKILL", holdMs: 125_000 }, discardControllerCache: true, crashAfterResult: {} }, { recoveryBinding: {} },