From c910651115c4c5e3be1c2e19ca033fb8c3dccb49 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 08:49:38 -0600 Subject: [PATCH] fix(native): start each rebound run with a fresh restart-continuation marker (#20053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - When the control plane restarts during an agent turn, the runner restores the same conversation. A Codex, Claude (ACPX) or OpenCode turn lost in the restart is marked `provider_turn_lost_on_restore`, and the run continues it once in the same conversation > - That one continuation is guarded by a disposition-only recovery marker in the session checkpoint. It is run state: it names a turn of the run that used it > - A follow-up run (a user comment on the same task) reuses the provider session by re-binding the previous run's checkpoint to the new run. Re-binding clears the run-scoped turn, result and request state, but it kept the marker > - The chaos K13 storm (wave w-20261108T134237-s9401) hit this three times (t24 and t45 on Claude, t77 on Codex). The task's first run had already used its restart continuation. The follow-up run inherited a spent marker naming a turn it never had, so when a restart lost its own turn, the driver kept the session closed. The run failed, and the user had to click Retry > - This pull request resets the marker at both re-bind sites, as for every other run-scoped field > - The benefit is that a restart during a follow-up turn continues on its own instead of putting a Retry click on the user ## Linked Issues or Issue Description Refs #19831 (durable ownership plan, restart continuation). Found by the restart chaos campaign. In all three failed follow-up runs: - After `resume_dead_runner`, the run emitted `turn.failed {code: provider_turn_lost_on_restore, recoverable: true}` for its own active turn. - The checkpoint still carried `dispositionOnlyRecoveryConsumed: true`, with `dispositionOnlyRecoveryTurnId` set to the previous run's continuation turn. ## What Changed - `native-session-resume.ts` (`rebindNativeSessionCheckpoint`) and `native-session-executor.ts` (warm checkpoint rebind): a rebound run starts with `dispositionOnlyRecoveryConsumed: false` and `dispositionOnlyRecoveryTurnId: null`. - Nothing changes inside a run: the driver's reopen rule (`mapTerminalTurn`) and `nativeRestartInterruptedTurnId` are untouched. The first commit changed the helper; review found the driver keeps its own copy of the rule, and the second commit reverts the helper in favour of this fix. ## Verification - `native-session-resume.test.ts` "retains provider identity but clears prior turn and event state" now seeds a spent marker on the previous run. It fails on `master` and passes here. - `native-session-executor.test.ts` "starts a new run's usage from the prior run's total when a warm checkpoint resumes" (3 cases) seeds the first run's checkpoint with a spent marker and asserts that the second run starts with it reset. It fails on `master` and passes here. - The other `native-session-resume.test.ts` tests pass. One test, "archives a damaged prior epoch before completing guarded same-task replacement with real runnerd", fails locally identically with and without this change (ENOENT under `authority-epochs`, from the real-runnerd environment). It's left to CI. - Next: a targeted chaos rerun (Codex and Claude C1 under repeated K2) on this branch. I'll add the result here. ## Risks - **Previous-release story (upgrade-safe run state):** no persisted shape changes. A run re-bound by this build gets a fresh marker. A run re-bound before the deploy, still in flight with an inherited spent marker, behaves as before: a lost turn falls back to Retry, once. - Inside a run, nothing changes: the marker still bounds same-run continuations, together with the provider-attempt restart budget. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../services/native-runtime/native-session-executor.test.ts | 6 ++++++ .../src/services/native-runtime/native-session-executor.ts | 2 ++ .../services/native-runtime/native-session-resume.test.ts | 4 ++++ server/src/services/native-runtime/native-session-resume.ts | 2 ++ 4 files changed, 14 insertions(+) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 58eaaa7c44a..aff0735736e 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -12321,6 +12321,8 @@ describe("native warm session supervision", () => { terminalTurns: [], pendingRuntimeRequests: [], codexUsageBaseline: { baseline: {}, latest, ...(accounted ? { accounted } : {}) }, + dispositionOnlyRecoveryConsumed: true, + dispositionOnlyRecoveryTurnId: "turn-first-run-restart-continuation", }; const result = { result: { summary: "completed" }, @@ -12336,6 +12338,7 @@ describe("native warm session supervision", () => { }; const firstClose = vi.fn(async () => undefined); let resumedBaseline: unknown; + let resumedMarker: unknown; state.execute .mockReset() .mockImplementationOnce(async (options) => { @@ -12345,6 +12348,8 @@ describe("native warm session supervision", () => { }) .mockImplementationOnce(async (options) => { resumedBaseline = options.persistedSession?.codexUsageBaseline; + resumedMarker = { consumed: options.persistedSession?.dispositionOnlyRecoveryConsumed, + turnId: options.persistedSession?.dispositionOnlyRecoveryTurnId }; return result; }); @@ -12365,6 +12370,7 @@ describe("native warm session supervision", () => { latest, ...(accounted ? { accounted } : {}), }); + expect(resumedMarker).toEqual({ consumed: false, turnId: null }); } finally { if (previousPaperclipHome === undefined) { delete process.env.PAPERCLIP_HOME; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index f4aacfdb298..5a3c30b02cd 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7698,6 +7698,8 @@ function loadWarmNativeCheckpoint( activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], + dispositionOnlyRecoveryConsumed: false, + dispositionOnlyRecoveryTurnId: null, governedWait: undefined, ...(envelope.snapshot.codexUsageBaseline ? { codexUsageBaseline: codexUsageBaselineForNextRun(envelope.snapshot.codexUsageBaseline) } diff --git a/server/src/services/native-runtime/native-session-resume.test.ts b/server/src/services/native-runtime/native-session-resume.test.ts index df3c17a2a2d..ce3f8314711 100644 --- a/server/src/services/native-runtime/native-session-resume.test.ts +++ b/server/src/services/native-runtime/native-session-resume.test.ts @@ -2020,6 +2020,8 @@ describe("rebindNativeSessionCheckpoint", () => { const profile = prior.runnerProfileJson as Record; const checkpoint = profile.sessionCheckpoint as Record; checkpoint.governedWait = { sourceEvent: { runId: previousRunId, turnId: "old-turn" }, result: { reportedWorkDisposition: "yielded" } }; + checkpoint.dispositionOnlyRecoveryConsumed = true; + checkpoint.dispositionOnlyRecoveryTurnId = "old-restart-continuation"; const before = structuredClone(prior); const rebound = rebindNativeSessionCheckpoint({ previousRun: prior, @@ -2037,6 +2039,8 @@ describe("rebindNativeSessionCheckpoint", () => { activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], + dispositionOnlyRecoveryConsumed: false, + dispositionOnlyRecoveryTurnId: null, providerRecoveryPolicy: "allow_replacement_after_resume_failure", identity: { runId: currentRunId, diff --git a/server/src/services/native-runtime/native-session-resume.ts b/server/src/services/native-runtime/native-session-resume.ts index 2a2155d9d7a..7708c725ce2 100644 --- a/server/src/services/native-runtime/native-session-resume.ts +++ b/server/src/services/native-runtime/native-session-resume.ts @@ -717,6 +717,8 @@ export function rebindNativeSessionCheckpoint(input: { activeTurnId: null, terminalTurns: [], pendingRuntimeRequests: [], + dispositionOnlyRecoveryConsumed: false, + dispositionOnlyRecoveryTurnId: null, // Settlement evidence belongs to the prior heartbeat run, even when the // provider session is reused for the user's response. governedWait: undefined,