From 803cb3bfe37aa0424f99897f5ad23e12351ca527 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:59:34 -0600 Subject: [PATCH] fix(native): re-bootstrap an OpenCode runner drained before its first turn (#19704) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Native runs start a local `paperclip-runnerd`. After a restart, recovery may start the bootstrap again when the durable state root proves that no provider work started. > - #19665 added a proof for a graceful drain that lands after `session.open` started a fresh session and before the first turn. It requires every recorded provider process to have exited. > - OpenCode records provider startup ownership twice during `session.open`. The first record comes before launch, with `processId: null` and `processGroupId: null`. > - The proof treated `processId: null` as a process it could not prove gone. An OpenCode run drained in this window still failed with `runner_state_identity_mismatch: bootstrap_not_proven_incomplete`. > - This pull request treats a null process id as "no process yet". The benefit is that a graceful restart during OpenCode bootstrap continues automatically, as it already does for ACPX. ## Linked Issues or Issue Description Refs #19665, #19524 **What happened?** A restart chaos wave sent SIGTERM to the control plane about 11 s after an OpenCode run started. The drain suspended the run. At the next boot, recovery verified that the local runner had stopped and chose `bootstrap_incomplete`. The executor then failed the run with `adapter_failed` and `runner_state_identity_mismatch: bootstrap_not_proven_incomplete`. The kept state root had: - `run.prepare` and `session.open` completed, with a freshly started session (`status: "started"`); - only `session.started`, `session.capabilities.updated` and `session.goal.snapshot` committed; - two `harness.diagnostic` events of type `provider_startup_ownership` in the runner outbox, the first with null process ids and the second with the server pid and process group; - no turn command, and the OpenCode server process had exited. **Steps to reproduce** 1. Start an OpenCode task on a local runner. 2. Send SIGTERM to the control plane after `session.open` completes and before the first turn. 3. Let the control plane restart. **Expected behavior** Recovery starts the bootstrap again, and the task continues. **Actual behavior** The run fails. The task needs a manual Retry. ## What Changed - `native-bootstrap-proof.ts`: the process-id collector skips `processId`, `pid` and `processGroupId` values that are `null`. A null value records a launch that has not produced a process yet. Any other non-number value still fails the proof, and every numeric id must still be proven gone. - Tests: a red test with the OpenCode startup-ownership shape, plus counter-tests for a live OpenCode server process, a non-numeric process id, and a started turn. ## Verification - Red before the fix: "allows a fresh bootstrap after a drain once OpenCode opened its session before the first turn" failed with `expected false to be true`. Green after the fix: the `native incomplete-bootstrap evidence` block passes 36/36. - I ran the proof read-only against kept real roots: - the OpenCode drained root now returns `true`; - an OpenCode root from the same wave where a turn ran (108 events) still returns `false`; - the three roots from #19524 and #19665 still return `true`. - `native-runner-restart-recovery.integration.test.ts` and `native-restart-recovery.test.ts`: 134 passed. `tsc --noEmit -p server/tsconfig.json` passes. - Live replay: see the PR comment. ## Risks - Low. The change only skips null ids. Numeric ids still need a proven exit. Any turn, provider output, resumed session, or remote runner still fails closed. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`), used through Claude Code with tool use (shell, file edits, test execution). Default context window. Extended reasoning was available. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../native-runtime/native-bootstrap-proof.ts | 4 +- .../native-session-executor.test.ts | 38 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/server/src/services/native-runtime/native-bootstrap-proof.ts b/server/src/services/native-runtime/native-bootstrap-proof.ts index 1e0476d4e27..bf7660c5ccb 100644 --- a/server/src/services/native-runtime/native-bootstrap-proof.ts +++ b/server/src/services/native-runtime/native-bootstrap-proof.ts @@ -97,7 +97,9 @@ function collectRecordedProcessIds(value: unknown, into: Array<{ id: unknown; gr if (Array.isArray(value)) return value.every(entry => collectRecordedProcessIds(entry, into, depth + 1)); if (!value || typeof value !== "object") return true; for (const [key, entry] of Object.entries(value)) { - if (key === "processId" || key === "pid") into.push({ id: entry, group: false }); + // A null id records a launch that had not produced a process yet, so + // there is nothing to prove gone. Any other non-number fails the proof. + if (key === "processId" || key === "pid") { if (entry !== null) into.push({ id: entry, group: false }); } else if (key === "processGroupId") { if (entry !== null) into.push({ id: entry, group: true }); } else if (!collectRecordedProcessIds(entry, into, depth + 1)) return false; } diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 0cfb4377f7d..fe16e4da45e 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1857,6 +1857,44 @@ describe("native incomplete-bootstrap evidence", () => { it("stays fail-closed after session.open for a runner whose processes are on another host", async () => { expect(await provesLocally(drainedAfterSessionOpen(await deadPid()), false)).toBe(false); }); + + // OpenCode records provider startup ownership twice while session.open + // runs: first before launch (no process yet, ids null), then with the + // launched server's process and process group. + const openCodeStartup = (sourceSeq: number, processId: number | null) => ({ + sourceSeq, priority: 0, eventType: "harness.diagnostic", envelope: { kind: "event", payload: { + eventType: "harness.diagnostic", payload: { code: "provider_startup_ownership", startup: { + attemptedProcessGeneration: 1, authenticatedThreadId: null, directChildExitObserved: false, exitCode: null, + failedStage: null, command: { commandType: "session.open", controllerSeq: 2 }, + processId, processGroupId: processId, + } } } }, + }); + const drainedOpenCodeAfterSessionOpen = (pid: number) => { + const fixture = drainedAfterSessionOpen(pid); + fixture.runner.nextSourceSeq = 6; + fixture.runner.outbox = [openCodeStartup(4, null), openCodeStartup(5, pid)]; + return fixture; + }; + + it("allows a fresh bootstrap after a drain once OpenCode opened its session before the first turn", async () => { + expect(await provesLocally(drainedOpenCodeAfterSessionOpen(await deadPid()))).toBe(true); + }); + + it.each<[string, (fixture: ReturnType) => void]>([ + ["the OpenCode server process is alive", f => { + (f.runner.outbox as unknown[])[1] = openCodeStartup(5, process.pid); + }], + ["a startup process id is not a number", f => { + const startup = openCodeStartup(5, null); + (startup.envelope.payload.payload.startup as Record).processId = "8174"; + (f.runner.outbox as unknown[])[1] = startup; + }], + ["OpenCode started a turn", f => { (f.runner.outbox as unknown[]).push(sessionEvent(6, "turn.started")); }], + ])("stays fail-closed for OpenCode when %s", async (_label, mutate) => { + const fixture = drainedOpenCodeAfterSessionOpen(await deadPid()); + mutate(fixture); + expect(await provesLocally(fixture)).toBe(false); + }); }); describe("native provider usage normalization", () => {