From 289b402c26f2ec419e1ef219c8f573ccc329a2aa Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Tue, 10 Nov 2026 05:13:21 -0600 Subject: [PATCH] fix(runner): settle a saved governed wait whose parked turn was lost on restore (#20519) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A native agent run can ask the board a question and park on that saved (governed) wait > - PR #20494 made a saved wait settle after a control-plane restart for Codex, which reports the parked turn as interrupted > - Claude runs through ACPX. After a kill -9 restart, Runnerd restores the conversation and reports the parked turn as `turn.failed` with `provider_turn_lost_on_restore` > - The governed-wait drain and the saved-replay check rejected every failed terminal as unproven, so the saved question failed and the task needed a human Retry > - Runnerd emits `provider_turn_lost_on_restore` only after it proves that the old provider lifetime ended. That is the same proof as an interrupted terminal > - This pull request accepts that recoverable restart loss as proof for a saved governed wait, and only that failure > - The benefit is that a Claude question survives a control-plane kill the same way a Codex question does ## Linked Issues or Issue Description Refs #20494. Related: #20498 (Claude lost cancel reply; this change does not touch that code). **What happened?** Restart chaos wave w-20261110T094930-s214 (Claude, Q1, PV1 and PV2 with `--provider-fault-restart K1`) failed 2 of 2 Q1 tasks with `R3:retry_needed`. The asking run ended `failed` with `native_provider_terminal_failed`. Its events show the question saved, the K1, recovery with `native_governed_wait_settlement_pending`, and then `turn.failed` with `provider_turn_lost_on_restore` for the parked turn. The thrown `providerCode` was `governed_wait_provider_terminal_unproven`. **What did you expect?** The saved question settles after the restart. The asking run ends `succeeded`. The board answer wakes the agent without a Retry. **Steps to reproduce** Run the harness with `--adapters claude --workloads Q1 --faults PV1,PV2 --provider-fault-restart K1 --provider-fault-delays 3 --provider-fault-restart-after-seconds 1 --concurrency 1 --tasks 4`. The new unit-test variants `lost-on-restore-after-restart` and `lost-on-restore-saved` failed with the same `providerCode` before this change. ## What Changed - `native-session-runtime.ts`: add `governedWaitTerminalProven(event)`. A failed terminal proves the stop only when `isNativeRestartInterruption(error)` is true (code `provider_turn_lost_on_restore`, `recoverable: true`). Other terminals keep the old rule (`providerTerminalObserved !== false`). - The governed-wait accounting drain and the saved-replay check both use that helper. - Tests: two new variants of the #20494 test (a lost terminal after the restart, and a lost terminal already saved before the next recovery). Two new rejection cases: a non-restart failed terminal and a non-recoverable restart loss still fail with `governed_wait_provider_terminal_unproven`. ## Verification - `pnpm --filter @paperclipai/paperclip-runner exec vitest run src/native-session-runtime`: 265 tests pass. The two new settle variants failed before the fix. - `pnpm --filter @paperclipai/paperclip-runner exec tsc --noEmit -p .` passes. - A live restart chaos wave on this branch (Claude Q1, PV1/PV2 + K1) is linked in a comment below when it finishes. ## Risks - Low risk. The change accepts one more terminal as proof for a saved governed wait. That terminal comes only from Runnerd after it proves the old provider lifetime ended, and the code already trusts it to continue a restart-lost turn (`nativeRestartInterruptedTurnId`). - A failed terminal with any other code, or a non-recoverable restart loss, is still rejected. ## Model Used - Claude (Anthropic), model ID `claude-opus-5-5`, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude --- .../src/native-session-runtime.test.ts | 19 +++++++++++++++---- .../src/native-session-runtime.ts | 18 ++++++++++++++++-- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index f6eca1ad66d..9b9d6c170ac 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -9587,7 +9587,7 @@ describe("executeNativeSession recovery", () => { expect(history.filter(event => event.payload.kind === "usage")).toEqual([usage]); }); - it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "different-turn", "foreign-settlement", "active-settled-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { + it.each(["missing-trigger", "changed-trigger", "unproven-terminal", "failed-terminal", "lost-not-recoverable", "different-turn", "foreign-settlement", "active-settled-turn"])("rejects unsafe governed-wait recovery: %s", async fault => { const trigger = { ...runnerEvent(1, "item.completed", { kind: "dynamicToolCall" }), turnId: "turn-waiting" }; const checkpoint: PersistedNativeSession = { backendKind: "mock", sessionId: identity.sessionId, identity, activeTurnId: trigger.turnId, governedWait: { sourceEvent: trigger, result: yieldedResult } }; @@ -9596,6 +9596,11 @@ describe("executeNativeSession recovery", () => { runId: fault === "foreign-settlement" ? "another-run" : identity.runId, normalizedSessionId: identity.sessionId, turnId: trigger.turnId, }; if (fault === "unproven-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "unproven", eventType: "turn.failed", payload: { providerTerminalObserved: false } }); + // Only a recoverable restart loss proves the old provider lifetime ended. + if (fault === "failed-terminal") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "failed", eventType: "turn.failed", + payload: { status: "failed", error: { code: "model_not_found", recoverable: false } } }); + if (fault === "lost-not-recoverable") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "lost", eventType: "turn.failed", + payload: { status: "failed", error: { code: "provider_turn_lost_on_restore", recoverable: false } } }); if (fault === "different-turn") history.push({ ...trigger, sourceSeq: 2, sourceEventId: "new-turn", turnId: "different", eventType: "turn.started" }); const start = vi.fn(); const complete = vi.fn(); @@ -9615,15 +9620,21 @@ describe("executeNativeSession recovery", () => { // the parked turn's terminal and the controller died during the accounting // drain. The drain had already discarded numbered non-accounting events, so // the saved replay has gaps after the trigger. - it.each(["terminal-after-restart", "late-terminal-saved"])("settles a saved governed wait whose drain left sequence gaps before a restart (%s)", async variant => { - const lateTerminalSaved = variant === "late-terminal-saved"; + // Claude (ACPX) Q1 with PV1/PV2 + K1 (wave w-20261110T094930-s214): the + // restored runner proves the old provider lifetime ended and reports the + // parked turn as lost on restore instead of interrupted. + it.each(["terminal-after-restart", "late-terminal-saved", "lost-on-restore-after-restart", "lost-on-restore-saved"])("settles a saved governed wait whose drain left sequence gaps before a restart (%s)", async variant => { + const lateTerminalSaved = variant.endsWith("-saved"); + const lostOnRestore = variant.startsWith("lost-on-restore"); const turn = (event: PrpEvent) => ({ ...event, turnId: "turn-waiting" }); const trigger = turn(runnerEvent(1, "item.completed", { kind: "dynamicToolCall", item: { id: "ask-1", name: "request_human_input" } })); const acknowledgement = turn(runnerEvent(2, "item.completed", { kind: "interrupt_acknowledgement", status: "acknowledged" })); const discardedMessage = turn(runnerEvent(3, "item.completed", { kind: "agentMessage", text: "I asked the board." })); const discardedToolEnd = turn(runnerEvent(4, "tool.execution.completed", { name: "request_human_input", status: "completed" })); const usage = turn(runnerEvent(5, "item.completed", { kind: "usage", usage: { runDelta: { inputTokens: 7, outputTokens: 3 }, runDeltaComplete: true } })); - const stopped = turn(runnerEvent(6, "turn.interrupted", { providerTerminalObserved: true })); + const stopped = lostOnRestore + ? turn(runnerEvent(6, "turn.failed", { provider: "acpx", status: "failed", providerTerminalObserved: false, code: "provider_turn_lost_on_restore", error: { code: "provider_turn_lost_on_restore", recoverable: true } })) + : turn(runnerEvent(6, "turn.interrupted", { providerTerminalObserved: true })); const history: PrpEvent[] = []; let checkpoint: PersistedNativeSession | undefined; const start = vi.fn(async () => ({ turnId: trigger.turnId })); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index f06e80af74c..178c28ec56b 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -424,6 +424,20 @@ function isTurnTerminal(event: PrpEvent): boolean { * parked turn. The drain discards every other numbered event, so the saved * replay has sequence gaps exactly where those discarded events were. */ +/** + * A parked turn's terminal proves the provider stopped its work. An interrupted + * or cancelled terminal does unless the provider never confirmed it. A failed + * terminal does only when Runnerd reports the turn lost on restore: it emits + * that cause after proving the old provider lifetime ended and restoring the + * conversation (ACPX reports a restart-killed turn this way; Codex reports it + * interrupted). That event carries `providerTerminalObserved: false`, because + * the provider itself sent no terminal; the restart proof stands in for it. + */ +function governedWaitTerminalProven(event: PrpEvent): boolean { + if (event.eventType === "turn.failed") return isNativeRestartInterruption(event.payload.error); + return event.payload.providerTerminalObserved !== false; +} + function isGovernedDrainRetainedEvent(event: PrpEvent, turnId: string | undefined): boolean { if (event.turnId !== turnId) return false; return isTurnTerminal(event) @@ -1235,7 +1249,7 @@ async function consumeTurn( eventCount += receipt.disposition === "committed" ? 1 : 0; highestContiguousSourceSeq = Math.max(highestContiguousSourceSeq, receipt.highestContiguousSourceSeq); if (terminal) { - if (trailing.eventType === "turn.failed" || trailing.payload.providerTerminalObserved === false) { + if (!governedWaitTerminalProven(trailing)) { throw new NativeProviderTerminalFailure("governed_wait_provider_terminal_unproven", false); } governedProviderTerminalObserved = true; @@ -2870,7 +2884,7 @@ export async function executeNativeSession( const governedTerminal = governedWait ? governedReplay.find(event => event.turnId === governedWait!.sourceEvent.turnId && isTurnTerminal(event)) : undefined; - if (governedTerminal?.eventType === "turn.failed" || governedTerminal?.payload.providerTerminalObserved === false) { + if (governedTerminal && !governedWaitTerminalProven(governedTerminal)) { throw new NativeProviderTerminalFailure("governed_wait_provider_terminal_unproven", false); } if (governedWait && recoveredActiveTurnId && recoveredActiveTurnId !== governedWait.sourceEvent.turnId) {