diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index bfa76665615..c1bf861a4d8 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -278,6 +278,67 @@ describe("Daytona idle eligibility", () => { expect(await plugin.definition.onIdleDrain!(signal)).toBe("none"); }); + describe("a sandbox in a state change during workspace realize", () => { + beforeEach(() => { vi.stubEnv("DAYTONA_API_KEY", "host-key"); }); + const realize = (timeoutMs = 300_000) => plugin.definition.onEnvironmentRealizeWorkspace!({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", + config: { timeoutMs, reuseLease: false }, + lease: { providerLeaseId: "sandbox-transition", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + workspace: { localPath: "/tmp/workspace" }, + } as never); + const stateChangeInProgress = () => Object.assign(new Error("Sandbox state change in progress"), { statusCode: 409 }); + + it("waits for a stopping sandbox to settle, then starts it", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopping" }); + mockGet.mockResolvedValue(sandbox); + sandbox.refreshData.mockImplementation(async () => { sandbox.state = "stopped"; }); + sandbox.start.mockImplementation(async () => { + if (sandbox.state !== "stopped") throw stateChangeInProgress(); + sandbox.state = "started"; + }); + await expect(realize()).resolves.toMatchObject({ cwd: "/home/daytona/paperclip-workspace" }); + expect(sandbox.start).toHaveBeenCalledTimes(1); + expect(sandbox.fs.createFolder).toHaveBeenCalledWith("/home/daytona/paperclip-workspace", "755"); + }); + + it("retries a start the provider rejected while its state was changing", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopped" }); + mockGet.mockResolvedValue(sandbox); + sandbox.start.mockRejectedValueOnce(stateChangeInProgress()) + .mockImplementationOnce(async () => { sandbox.state = "started"; }); + await expect(realize()).resolves.toMatchObject({ cwd: "/home/daytona/paperclip-workspace" }); + expect(sandbox.start).toHaveBeenCalledTimes(2); + }); + + it("gives up when the state change outlasts the start deadline", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopping" }); + mockGet.mockResolvedValue(sandbox); + sandbox.start.mockRejectedValue(stateChangeInProgress()); + await expect(realize(1_000)).rejects.toThrow("is still stopping at its start deadline"); + expect(sandbox.start).not.toHaveBeenCalled(); + expect(sandbox.fs.createFolder).not.toHaveBeenCalled(); + }); + + it("gives the start only the budget the wait left", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopping" }); + mockGet.mockResolvedValue(sandbox); + let reads = 0; + sandbox.refreshData.mockImplementation(async () => { if (++reads === 2) sandbox.state = "stopped"; }); + sandbox.start.mockImplementation(async () => { sandbox.state = "started"; }); + await expect(realize(4_000)).resolves.toMatchObject({ cwd: "/home/daytona/paperclip-workspace" }); + expect(reads).toBe(2); + expect(sandbox.start.mock.calls[0]![0]).toBeLessThanOrEqual(2); + }); + + it("fails other start errors at once", async () => { + const sandbox = createMockSandbox({ id: "sandbox-transition", state: "stopped" }); + mockGet.mockResolvedValue(sandbox); + sandbox.start.mockRejectedValue(new Error("quota exceeded")); + await expect(realize()).rejects.toThrow("quota exceeded"); + expect(sandbox.start).toHaveBeenCalledTimes(1); + }); + }); + it("keeps a stale stopped reply from clearing a start that finished during the read", async () => { const sandbox = createMockSandbox({ state: "stopped" }); mockGet.mockResolvedValue(sandbox); @@ -3389,6 +3450,26 @@ describe("Daytona sandbox provider plugin", () => { expect(lastExec[0]).toBe(freshSessionId); }); + it("opens a fresh session after a restart outside the plugin that realize waited out", async () => { + vi.stubEnv("DAYTONA_API_KEY", "host-key"); + const sandbox = autoStoppedSandbox(); + mockGet.mockResolvedValue(sandbox); + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + const staleSessionId = sandbox.process.createSession.mock.calls[0]![0] as string; + sandbox.state = "starting"; + sandbox.refreshData.mockImplementationOnce(async () => { sandbox.state = "started"; }); + await plugin.definition.onEnvironmentRealizeWorkspace?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", + config: { timeoutMs: 300000, reuseLease: false }, + lease: { providerLeaseId: "sandbox-123", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + workspace: { localPath: "/tmp/workspace" }, + } as never); + expect(sandbox.start).not.toHaveBeenCalled(); + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(2); + expect(sandbox.process.createSession.mock.calls[1]![0]).not.toBe(staleSessionId); + }); + it("opens a fresh session after sync-in restarts an auto-stopped sandbox", async () => { process.env.DAYTONA_API_KEY = "host-key"; const sandbox = autoStoppedSandbox(); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index efd2f96292a..15839c36ad0 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -637,7 +637,47 @@ async function terminateAtProvider(scope: SandboxScope, operation: string, ac // restart, so the next command opens a fresh session instead of sending a dead // id and failing with "session not found". A running sandbox keeps its live // session, so the id is left in place when no restart happens. +const SANDBOX_TRANSITION_STATES: ReadonlySet = new Set([ + "starting", "stopping", "restoring", "archiving", "creating", "pending_build", "pulling_snapshot", "building_snapshot", "resizing", +]); +const SANDBOX_TRANSITION_POLL_MS = 1_000; + +function isSandboxStateChangeInProgress(error: unknown): boolean { + return /state change in progress/i.test(error instanceof Error ? error.message : String(error)); +} + +async function waitOutSandboxTransition(sandbox: Sandbox, deadline: number, readTimeoutMs: number): Promise { + while (SANDBOX_TRANSITION_STATES.has(sandbox.state ?? "") && Date.now() + SANDBOX_TRANSITION_POLL_MS < deadline) { + await new Promise(resolve => setTimeout(resolve, SANDBOX_TRANSITION_POLL_MS)); + await withLivenessTimeout("sandbox.refreshData", Math.min(readTimeoutMs, Math.max(1, deadline - Date.now())), + () => sandbox.refreshData()); + } +} + async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number, scope: SandboxScope): Promise { + if (sandbox.state === "started") return; + sandboxHandleSessionStore.clear(scope); + const deadline = Date.now() + timeoutSeconds * 1_000; + const readTimeoutMs = Math.min(scope.config.livenessTimeoutMs, 30_000); + for (;;) { + await waitOutSandboxTransition(sandbox, deadline, readTimeoutMs); + if (SANDBOX_TRANSITION_STATES.has(sandbox.state ?? "") || Date.now() >= deadline) { + throw new Error(`Daytona sandbox ${sandbox.id} is still ${sandbox.state ?? "changing state"} at its start deadline`); + } + const remainingSeconds = Math.max(1, Math.ceil((deadline - Date.now()) / 1_000)); + try { + await startSettledSandbox(sandbox, remainingSeconds, scope); + return; + } catch (error) { + if (!isSandboxStateChangeInProgress(error) || Date.now() + SANDBOX_TRANSITION_POLL_MS >= deadline) throw error; + await new Promise(resolve => setTimeout(resolve, SANDBOX_TRANSITION_POLL_MS)); + await withLivenessTimeout("sandbox.refreshData", Math.min(readTimeoutMs, Math.max(1, deadline - Date.now())), + () => sandbox.refreshData()); + } + } +} + +async function startSettledSandbox(sandbox: Sandbox, timeoutSeconds: number, scope: SandboxScope): Promise { if (sandbox.state === "started") return; sandboxHandleSessionStore.clear(scope); // Bound the lifecycle call just past its own SDK deadline. A normal slow start