From 6451d2bfbf616d6193f7c014438c95032c6e3674 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 8 Nov 2026 17:59:56 -0600 Subject: [PATCH] fix(native): verify a staged attachment restart on the claimed sandbox lease (#20136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A follow-up codex run on a per-turn Daytona sandbox gets a fresh sandbox, with harness state restored from the previous turn's checkpoint, and attaches the previous runner session to the new run through a staged attachment > - A graceful restart during that start suspends the run. The next controller reattaches to the surviving remote runner, after proving it is on the sandbox lease the restart claim names > - The staged-attachment path required the lease to have been acquired as `resumed`. A per-turn follow-up's lease is `created`, so every such restart was refused with `native_runner_staged_attachment_recovery_unproven` > - The run then needed a Retry > - This pull request binds the staged restart to the claimed lease the same way the ordinary remote reattach does: same provider lease, same remote directory, and not a replacement sandbox > - The benefit is that a per-turn follow-up interrupted by a restart continues on its own ## Linked Issues or Issue Description Refs #20103 **What happened?** Daytona per-turn restart-chaos wave w-20261108T225520-s9501 (master 15b987d9b8, image cbbe8224). Two codex C1 follow-up runs, created during a K2 graceful drain at 23:11:45, were suspended mid-start: - `4d1d3563` (t07); - `71b43ea1` (t08). Two to three seconds after the controller came back, both failed with `native_runner_staged_attachment_recovery_unproven`, rested in `native_continuation_requires_reconciliation`, and needed a Retry. The stack shows the first guard in `verifyStagedAttachmentRestart`. Both runs' environment leases were acquired with `sandboxLeaseAcquisition.outcome: "created"`, because per-turn mode creates a fresh sandbox for each turn. The guard accepted only `"resumed"`. **Expected behavior** When the restart claim's provider lease and remote directory match the run's own lease, the staged attachment's restart is verified as before, and the run continues without a Retry. The rest of the staged proof is unchanged. A replacement sandbox, or a different provider lease or directory, is still refused. **Steps to reproduce** 1. Run a codex follow-up on a per-turn Daytona environment, so the lease is `created` and the start stages an attachment. 2. Gracefully restart the control plane during the start. 3. Before this change, the restarted run fails with `native_runner_staged_attachment_recovery_unproven`. **Paperclip version or commit** 15b987d9b8 (wave). Branch rebased on current master. ## What Changed - `verifyStagedAttachmentRestart` (`native-session-executor.ts`) no longer requires `sandboxLeaseAcquisition.outcome === "resumed"`. It requires the same lease binding as `verifyRemoteRunnerReattachment`: - the target's provider lease equals the claim's `remote.providerLeaseId`; - the target's remote directory equals the claim's `remoteCwd`; - the lease is not a `replacement` sandbox. - A new test case in `native-session-executor.test.ts` covers a created sandbox (admitted), a replacement sandbox, another provider lease, and another remote directory (all refused). ## Verification - Red/green: on master, the created-sandbox case fails with `native_runner_staged_attachment_recovery_unproven`. With the change it reaches the staged recheck. The three refusal cases are refused on both. - `vitest run` of `native-session-executor.test.ts` and `native-restart-recovery-isolation.test.ts`: 2 files, 1305 tests passed. - `PAPERCLIP_TYPECHECK_LOW_MEMORY=1 pnpm typecheck` (`server`): passed. ## Risks - Low risk. The check now matches the ordinary remote reattach, which already admits created leases. Everything after it is unchanged: the encrypted staged header, the source run and result, the template digest, and the remote runner identity are all still proven before any RPC. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use (shell, file edits, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../native-session-executor.test.ts | 38 +++++++++++++++++++ .../native-runtime/native-session-executor.ts | 4 +- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index d5dfd2a44ea..fb0650cd6ef 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -15634,6 +15634,44 @@ describe("runnerd provider runtime wiring", () => { } finally { restore.mockRestore(); header.mockRestore(); verify.mockRestore(); await controller.stop(); } }); + it.each([ + { label: "a sandbox this run created", outcome: "created", providerLeaseId: "sandbox", remoteCwd: "/workspace", admitted: true }, + { label: "a replacement sandbox", outcome: "replacement", providerLeaseId: "sandbox", remoteCwd: "/workspace", admitted: false }, + { label: "another provider lease", outcome: "created", providerLeaseId: "other-sandbox", remoteCwd: "/workspace", admitted: false }, + { label: "another remote directory", outcome: "created", providerLeaseId: "sandbox", remoteCwd: "/elsewhere", admitted: false }, + ] as const)("binds a staged attachment restart to the claimed sandbox lease: $label", async ({ outcome, providerLeaseId, remoteCwd, admitted }) => { + const current = scopedNativeExecution(`staged-attachment-lease-${outcome}-${providerLeaseId}-${remoteCwd.length}`); + current.task.prompt = "Inspect attachment ref:original"; + const root = await initializeScopedRunnerRoot(current, "attachment-runner"); + const sourceIdentity = { runnerInstanceId: "attachment-runner", environmentLeaseId: "attachment-lease", + runId: "attachment-source", normalizedSessionId: current.session.normalizedSessionId!, turnId: "source-turn", itemId: "source-item" }; + const targetIdentity = { ...sourceIdentity, runId: current.binding.runId, turnId: "target-turn", itemId: "target-item" }; + const controller = new DurablePrpControlPlane({ stateDirectory: join(root, "control-plane"), identity: sourceIdentity, + expectedRunnerVersion: "fixture", expectedRunnerDigest: `sha256:${"a".repeat(64)}` }); + controller.stageRunAttachment(targetIdentity, { provider: { kind: "codex", instructions: "prepared attachment grant" } }); + const saved = JSON.parse(await readFile(join(root, "control-plane/control-plane-state.json"), "utf8")); + state.stageNativeRunnerWakeAttachments.mockImplementationOnce(async () => + ({ attachments: [{ id: "attachment-original" }], cleanup: async () => undefined }) as never); + state.renderNativeRunnerStagedAttachmentPrompt.mockReturnValueOnce("Materialized attachment: .paperclip-inbound/current/original"); + const reached = new Error("verified staged admission reached recheck"); + const restore = vi.spyOn(runnerCheckpoint, "restoreNativeRunnerCheckpoint").mockResolvedValue(null); + const header = vi.spyOn(runnerCheckpoint, "loadNativeRunnerCheckpointHeader").mockResolvedValue("staged header"); + const verify = vi.spyOn(stagedAttachmentRecovery, "resolveNativeStagedAttachmentRecovery") + .mockResolvedValueOnce({ currentExecution: structuredClone(current), sourceIdentity, targetIdentity, + templateDigest: saved.stagedRunAttachment.templateDigest } as never) + .mockRejectedValueOnce(reached); + try { + const resume = executePaperclipNativeSession({ db: leaseDb(current), execution: current, useRunnerd: true, + runnerInstanceId: "attachment-runner", restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId, + leaseOwner: "controller", controllerGeneration: 2, providerAttempt: 1, restartKind: "hard", recoveryRequestId: null, + remote: { kind: "sandbox", leaseId: "attachment-lease", providerLeaseId: "sandbox", remoteCwd: "/workspace" } } as never, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", leaseId: "attachment-lease", remoteCwd, + runner: { execute: vi.fn() }, sandboxLeaseAcquisition: { outcome, providerLeaseId } } as never }); + if (admitted) { await expect(resume).rejects.toBe(reached); expect(verify).toHaveBeenCalledTimes(2); } + else { await expect(resume).rejects.toThrow("staged_attachment_recovery_unproven"); expect(verify).not.toHaveBeenCalled(); } + } finally { restore.mockRestore(); header.mockRestore(); verify.mockRestore(); await controller.stop(); } + }); + it("passes the run checkpoint active turn into restart recovery", async () => { state.createBackend.mockClear(); state.createTransport.mockClear(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e31831b35ba..bdc1a2c5b92 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6464,7 +6464,9 @@ async function verifyStagedAttachmentRestart(input: { const target = input.runnerExecutionTarget; if (input.restartRecovery?.kind !== "reattach_remote_runner" || input.restartRecovery.remote.kind === "plugin_task" || target?.kind !== "remote" || target.transport !== "sandbox" || target.runnerTask || - target.sandboxLeaseAcquisition?.outcome !== "resumed") return deny(); + target.sandboxLeaseAcquisition?.outcome === "replacement" || + input.restartRecovery.remote.providerLeaseId !== target.sandboxLeaseAcquisition?.providerLeaseId || + input.restartRecovery.remote.remoteCwd !== target.remoteCwd) return deny(); const [current] = await input.db.select().from(heartbeatRuns).where(and( eq(heartbeatRuns.id, input.execution.binding.runId), eq(heartbeatRuns.companyId, input.execution.binding.companyId), eq(heartbeatRuns.agentId, input.execution.binding.agentId), eq(heartbeatRuns.nativeIssueId, input.execution.binding.issueId)));