From 088bdf7cfa78eda23bdfb4c25dbae90e1861c0dc Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 9 Nov 2026 03:00:14 -0600 Subject: [PATCH] fix(daytona): wait out a sandbox state change before starting it for workspace realize (#20212) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - On Daytona, every run realizes its workspace in the leased sandbox. Realizing it first makes sure the sandbox is started > - Right after a control-plane restart, the sandbox can still be mid-transition, for example stopping or starting from the previous lifecycle step > - The plugin called `sandbox.start` straight away. Daytona rejects that with "Sandbox state change in progress", so the run failed `setup_failed` and needed a Retry > - This pull request waits out a sandbox's state change before starting it, and retries a start the provider rejected for that reason, within the existing start deadline > - The benefit is that a run whose sandbox is briefly transitioning starts on its own ## Linked Issues or Issue Description Refs #20211 **What happened?** Daytona per-turn restart-chaos K13 storm w-20261109T074128-s9503 (revision 9046884811), t07 (claude A1): run `5aabeb55` failed at 07:45:27 with `setup_failed` and "Failed to realize workspace for environment … (sandbox): Sandbox state change in progress". The sandbox was mid-transition after the restart, and realize neither waited nor retried. A Retry succeeded. **Expected behavior** Realize waits for a transitional sandbox state to settle, then starts the sandbox. If the provider still reports the state change, it retries, all within the existing start deadline. Any other start error fails at once, and a transition that outlasts the deadline still fails. **Steps to reproduce** 1. Realize a workspace while the Daytona sandbox is `stopping` or `starting`, or while `start` returns "Sandbox state change in progress". 2. Before this change, realize fails at once with that error. **Paperclip version or commit** 9046884811 (wave). Branch rebased on current master. ## What Changed - `packages/plugins/sandbox-providers/daytona/src/plugin.ts`: `ensureSandboxStarted` now polls `refreshData` while the sandbox is in a transitional state. - The transitional states are `starting`, `stopping`, `restoring`, `archiving`, `creating`, `pending_build`, `pulling_snapshot`, `building_snapshot` and `resizing`. - It polls every second, until the start deadline from `timeoutMs`. Each read keeps the existing liveness bound. - If the start fails with "state change in progress", it refreshes and tries again, within the same deadline. - The wait, every retry and the start share one deadline. Each read is bounded by the time left, and the start receives only the remaining seconds. - As soon as any state other than `started` is observed, the lease's stored session is dropped before waiting, so a restart that happened outside the plugin never reuses a dead session. - Starting a settled sandbox (`startSettledSandbox`) is unchanged. - `plugin.test.ts`: - realize waits for a stopping sandbox, then starts it; - realize retries a start that was rejected for a state change; - realize gives up once the transition outlasts the deadline; - another start error fails at once; - the start receives only the budget the wait left; - a restart outside the plugin that the wait sees through opens a fresh session. - The tests stub the API key with `vi.stubEnv`. ## Verification - Red/green: on master, the wait and the retry cases fail with "Sandbox state change in progress". With the change they pass, and the deadline and other-error cases hold. - `vitest run` in `packages/plugins/sandbox-providers/daytona`: 14 files, 614 tests passed (8 skipped). - `tsc --noEmit` for the Daytona plugin package: passed. ## Risks - Low risk. The wait is bounded by the start deadline the plugin already used. Only transitional states and the provider's state-change rejection are waited out, and every other error behaves as before. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use (shell, file edits, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../daytona/src/plugin.test.ts | 81 +++++++++++++++++++ .../sandbox-providers/daytona/src/plugin.ts | 40 +++++++++ 2 files changed, 121 insertions(+) diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index bfa76665615..c1bf861a4d8 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -278,6 +278,67 @@ describe("Daytona idle eligibility", () => { expect(await plugin.definition.onIdleDrain!(signal)).toBe("none"); }); + describe("a sandbox in a state change during workspace realize", () => { + beforeEach(() => { vi.stubEnv("DAYTONA_API_KEY", "host-key"); }); + const realize = (timeoutMs = 300_000) => plugin.definition.onEnvironmentRealizeWorkspace!({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", + config: { timeoutMs, reuseLease: false }, + lease: { providerLeaseId: "sandbox-transition", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + workspace: { localPath: "/tmp/workspace" }, + } as never); + const stateChangeInProgress = () => Object.assign(new Error("Sandbox state change in progress"), { statusCode: 409 }); + + it("waits for a stopping sandbox to settle, then starts it", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopping" }); + mockGet.mockResolvedValue(sandbox); + sandbox.refreshData.mockImplementation(async () => { sandbox.state = "stopped"; }); + sandbox.start.mockImplementation(async () => { + if (sandbox.state !== "stopped") throw stateChangeInProgress(); + sandbox.state = "started"; + }); + await expect(realize()).resolves.toMatchObject({ cwd: "/home/daytona/paperclip-workspace" }); + expect(sandbox.start).toHaveBeenCalledTimes(1); + expect(sandbox.fs.createFolder).toHaveBeenCalledWith("/home/daytona/paperclip-workspace", "755"); + }); + + it("retries a start the provider rejected while its state was changing", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopped" }); + mockGet.mockResolvedValue(sandbox); + sandbox.start.mockRejectedValueOnce(stateChangeInProgress()) + .mockImplementationOnce(async () => { sandbox.state = "started"; }); + await expect(realize()).resolves.toMatchObject({ cwd: "/home/daytona/paperclip-workspace" }); + expect(sandbox.start).toHaveBeenCalledTimes(2); + }); + + it("gives up when the state change outlasts the start deadline", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopping" }); + mockGet.mockResolvedValue(sandbox); + sandbox.start.mockRejectedValue(stateChangeInProgress()); + await expect(realize(1_000)).rejects.toThrow("is still stopping at its start deadline"); + expect(sandbox.start).not.toHaveBeenCalled(); + expect(sandbox.fs.createFolder).not.toHaveBeenCalled(); + }); + + it("gives the start only the budget the wait left", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopping" }); + mockGet.mockResolvedValue(sandbox); + let reads = 0; + sandbox.refreshData.mockImplementation(async () => { if (++reads === 2) sandbox.state = "stopped"; }); + sandbox.start.mockImplementation(async () => { sandbox.state = "started"; }); + await expect(realize(4_000)).resolves.toMatchObject({ cwd: "/home/daytona/paperclip-workspace" }); + expect(reads).toBe(2); + expect(sandbox.start.mock.calls[0]![0]).toBeLessThanOrEqual(2); + }); + + it("fails other start errors at once", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopped" }); + mockGet.mockResolvedValue(sandbox); + sandbox.start.mockRejectedValue(new Error("quota exceeded")); + await expect(realize()).rejects.toThrow("quota exceeded"); + expect(sandbox.start).toHaveBeenCalledTimes(1); + }); + }); + it("keeps a stale stopped reply from clearing a start that finished during the read", async () => { const sandbox = createMockSandbox({ state: "stopped" }); mockGet.mockResolvedValue(sandbox); @@ -3389,6 +3450,26 @@ describe("Daytona sandbox provider plugin", () => { expect(lastExec[0]).toBe(freshSessionId); }); + it("opens a fresh session after a restart outside the plugin that realize waited out", async () => { + vi.stubEnv("DAYTONA_API_KEY", "host-key"); + const sandbox = autoStoppedSandbox(); + mockGet.mockResolvedValue(sandbox); + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + const staleSessionId = sandbox.process.createSession.mock.calls[0]![0] as string; + sandbox.state = "starting"; + sandbox.refreshData.mockImplementationOnce(async () => { sandbox.state = "started"; }); + await plugin.definition.onEnvironmentRealizeWorkspace?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", + config: { timeoutMs: 300000, reuseLease: false }, + lease: { providerLeaseId: "sandbox-123", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + workspace: { localPath: "/tmp/workspace" }, + } as never); + expect(sandbox.start).not.toHaveBeenCalled(); + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(2); + expect(sandbox.process.createSession.mock.calls[1]![0]).not.toBe(staleSessionId); + }); + it("opens a fresh session after sync-in restarts an auto-stopped sandbox", async () => { process.env.DAYTONA_API_KEY = "host-key"; const sandbox = autoStoppedSandbox(); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index efd2f96292a..15839c36ad0 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -637,7 +637,47 @@ async function terminateAtProvider(scope: SandboxScope, operation: string, ac // restart, so the next command opens a fresh session instead of sending a dead // id and failing with "session not found". A running sandbox keeps its live // session, so the id is left in place when no restart happens. +const SANDBOX_TRANSITION_STATES: ReadonlySet = new Set([ + "starting", "stopping", "restoring", "archiving", "creating", "pending_build", "pulling_snapshot", "building_snapshot", "resizing", +]); +const SANDBOX_TRANSITION_POLL_MS = 1_000; + +function isSandboxStateChangeInProgress(error: unknown): boolean { + return /state change in progress/i.test(error instanceof Error ? error.message : String(error)); +} + +async function waitOutSandboxTransition(sandbox: Sandbox, deadline: number, readTimeoutMs: number): Promise { + while (SANDBOX_TRANSITION_STATES.has(sandbox.state ?? "") && Date.now() + SANDBOX_TRANSITION_POLL_MS < deadline) { + await new Promise(resolve => setTimeout(resolve, SANDBOX_TRANSITION_POLL_MS)); + await withLivenessTimeout("sandbox.refreshData", Math.min(readTimeoutMs, Math.max(1, deadline - Date.now())), + () => sandbox.refreshData()); + } +} + async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number, scope: SandboxScope): Promise { + if (sandbox.state === "started") return; + sandboxHandleSessionStore.clear(scope); + const deadline = Date.now() + timeoutSeconds * 1_000; + const readTimeoutMs = Math.min(scope.config.livenessTimeoutMs, 30_000); + for (;;) { + await waitOutSandboxTransition(sandbox, deadline, readTimeoutMs); + if (SANDBOX_TRANSITION_STATES.has(sandbox.state ?? "") || Date.now() >= deadline) { + throw new Error(`Daytona sandbox ${sandbox.id} is still ${sandbox.state ?? "changing state"} at its start deadline`); + } + const remainingSeconds = Math.max(1, Math.ceil((deadline - Date.now()) / 1_000)); + try { + await startSettledSandbox(sandbox, remainingSeconds, scope); + return; + } catch (error) { + if (!isSandboxStateChangeInProgress(error) || Date.now() + SANDBOX_TRANSITION_POLL_MS >= deadline) throw error; + await new Promise(resolve => setTimeout(resolve, SANDBOX_TRANSITION_POLL_MS)); + await withLivenessTimeout("sandbox.refreshData", Math.min(readTimeoutMs, Math.max(1, deadline - Date.now())), + () => sandbox.refreshData()); + } + } +} + +async function startSettledSandbox(sandbox: Sandbox, timeoutSeconds: number, scope: SandboxScope): Promise { if (sandbox.state === "started") return; sandboxHandleSessionStore.clear(scope); // Bound the lifecycle call just past its own SDK deadline. A normal slow start