diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md index 618f0df429..a7e0198f8c 100644 --- a/packages/plugins/sandbox-providers/daytona/README.md +++ b/packages/plugins/sandbox-providers/daytona/README.md @@ -28,6 +28,7 @@ Notes: - The current published Daytona SDK package is `@daytonaio/sdk`. - The driver supports both `snapshot`-based and `image`-based sandbox creation. If both are set, validation rejects the config as ambiguous. - Reusable leases map to Daytona stop/start semantics. Non-reusable leases are deleted on release. A provider-resolved `target` does not change the identity of an existing sandbox. Release closes the same scoped lease that a later sentinel-verified resume reopens. +- A sandbox record can survive the loss of its underlying container. Resume treats it as expired only when a fresh provider read confirms the exact missing-container error for that sandbox and marks it unrecoverable. Unknown errors and failed confirmation reads preserve the lease. The host still requires a verified native-runner backup before replacement. ## Local development diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 30066a5aaa..066436c998 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -1264,6 +1264,132 @@ describe("Daytona sandbox provider plugin", () => { }); }); + describe("missing-container resume", () => { + const sandboxId = "00000000-0000-4000-8000-000000000001"; + const missing = `not found: failed to inspect sandbox container ${sandboxId}: Error response from daemon: No such container: ${sandboxId}`; + const params = { + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", providerLeaseId: sandboxId, + config: { apiKey: "host-key", timeoutMs: 300000, livenessTimeoutMs: 100, reuseLease: true }, + leaseMetadata: { workspaceSentinel: { path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", token: "sentinel-token" } }, + }; + const resume = () => plugin.definition.onEnvironmentResumeLease!(params); + function missingSandbox() { + return { ...createMockSandbox({ id: sandboxId, state: "error", recoverable: false }), errorReason: missing }; + } + function allowSentinel(sandbox: ReturnType) { + sandbox.process.executeCommand.mockResolvedValueOnce({ exitCode: 0, + result: JSON.stringify({ token: "sentinel-token" }), artifacts: { stdout: JSON.stringify({ token: "sentinel-token" }) }, + }); + } + + it("expires a provider record with a freshly confirmed missing container without replacing it", async () => { + const sandbox = missingSandbox(); + mockGet.mockResolvedValue(sandbox); + for (let attempt = 0; attempt < 2; attempt++) { + await expect(resume()).resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } }); + } + expect(mockGet).toHaveBeenCalledTimes(2); + expect(sandbox.refreshData).toHaveBeenCalledTimes(2); + expect(sandbox.start).not.toHaveBeenCalled(); + expect(sandbox.recover).not.toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it.each(["sandbox-opaque", "sandbox.with+[literal](characters)"])("matches the opaque sandbox ID literally: %s", async (id) => { + const sandbox = { ...missingSandbox(), id, errorReason: missing.replaceAll(sandboxId, id) }; + mockGet.mockResolvedValue(sandbox); + await expect(plugin.definition.onEnvironmentResumeLease!({ ...params, providerLeaseId: id })) + .resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } }); + expect(sandbox.refreshData).toHaveBeenCalledOnce(); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it.each([ + "not found: provider temporarily unavailable", + missing.replace("No such container:", "No such volume:"), + missing.replace(/000000000001$/, "000000000002"), + missing.replaceAll(sandboxId, "00000000-0000-4000-8000-000000000002"), + ])("preserves an unexplained unrecoverable error: %s", async (errorReason) => { + const sandbox = { ...missingSandbox(), errorReason }; + mockGet.mockResolvedValue(sandbox); + await expect(resume()).rejects.toThrow("unrecoverable error state"); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("uses provider recovery when the sandbox is recoverable", async () => { + const sandbox = { ...missingSandbox(), recoverable: true }; + allowSentinel(sandbox); + mockGet.mockResolvedValue(sandbox); + await expect(resume()).resolves.toMatchObject({ providerLeaseId: sandboxId }); + expect(sandbox.recover).toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + }); + + it("reuses a sandbox whose fresh state disproves the cached loss", async () => { + const sandbox = missingSandbox(); + sandbox.refreshData.mockImplementation(async () => { sandbox.state = "started"; }); + allowSentinel(sandbox); + mockGet.mockResolvedValue(sandbox); + await expect(resume()).resolves.toMatchObject({ providerLeaseId: sandboxId }); + expect(sandbox.start).not.toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it.each([new MockDaytonaTimeoutError("timed out"), new Error("provider 503")])("preserves the lease if confirmation fails: %s", async (error) => { + const sandbox = missingSandbox(); + sandbox.refreshData.mockRejectedValue(error); + mockGet.mockResolvedValue(sandbox); + await expect(resume()).rejects.toThrow(error.message); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("preserves an unknown error returned by the fresh provider read", async () => { + const sandbox = missingSandbox(); + sandbox.refreshData.mockImplementation(async () => { sandbox.errorReason = "provider unavailable"; }); + mockGet.mockResolvedValue(sandbox); + await expect(resume()).rejects.toThrow("unrecoverable error state"); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("rejects a refreshed handle belonging to a different sandbox", async () => { + const sandbox = missingSandbox(); + sandbox.refreshData.mockImplementation(async () => { sandbox.id = "another-sandbox"; }); + mockGet.mockResolvedValue(sandbox); + await expect(resume()).rejects.toThrow("handle mismatch"); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("accepts a typed not-found result while confirming the missing container", async () => { + const sandbox = missingSandbox(); + sandbox.refreshData.mockRejectedValue(new MockDaytonaNotFoundError("missing")); + mockGet.mockResolvedValue(sandbox); + await expect(resume()).resolves.toEqual({ providerLeaseId: null, metadata: { expired: true } }); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it("bounds a stalled confirmation and preserves the lease", async () => { + vi.useFakeTimers(); + try { + const sandbox = missingSandbox(); + sandbox.refreshData.mockImplementation(() => new Promise(() => {})); + mockGet.mockResolvedValue(sandbox); + const result = resume().then(() => null, error => error as Error); + await vi.advanceTimersByTimeAsync(101); + expect((await result)?.message).toContain("sandbox.refreshData"); + expect(sandbox.delete).not.toHaveBeenCalled(); + expect(mockCreate).not.toHaveBeenCalled(); + } finally { vi.useRealTimers(); } + }); + }); + it("expires a reusable lease when the workspace sentinel does not match", async () => { process.env.DAYTONA_API_KEY = "host-key"; const sandbox = createMockSandbox({ id: "sandbox-reuse", state: "stopped" }); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 88ae8f3fdb..a053af7e7c 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -502,6 +502,13 @@ async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): P await withLivenessTimeout("sandbox.start", startBoundMs, () => sandbox.start(timeoutSeconds)); } +function hasMissingSandboxContainer(sandbox: Sandbox): boolean { + if (sandbox.state !== "error" || sandbox.recoverable !== false || typeof sandbox.errorReason !== "string") return false; + if (typeof sandbox.id !== "string" || !sandbox.id) return false; + return sandbox.errorReason === + `not found: failed to inspect sandbox container ${sandbox.id}: Error response from daemon: No such container: ${sandbox.id}`; +} + async function resolveSandboxWorkingDirectory(sandbox: Sandbox): Promise { const root = (await sandbox.getWorkDir())?.trim() || (await sandbox.getUserHomeDir())?.trim() @@ -2228,6 +2235,26 @@ const plugin = definePlugin({ return { providerLeaseId: null, metadata: { expired: true } }; } + // Daytona can retain the API record after losing its container. Confirm + // the exact provider report again before allowing the host's existing + // backup-guarded replacement path. Unknown errors preserve the lease. + if (hasMissingSandboxContainer(sandbox)) { + try { + await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData()); + assertHandleMatchesLease(sandbox, params.providerLeaseId); + } catch (error) { + evictSandboxHandle(scope); + if (error instanceof DaytonaNotFoundError) { + return { providerLeaseId: null, metadata: { expired: true } }; + } + throw error; + } + if (hasMissingSandboxContainer(sandbox)) { + evictSandboxHandle(scope); + return { providerLeaseId: null, metadata: { expired: true } }; + } + } + // A stopped sandbox loses its session shell, so the stored session id is // stale after a real restart. Clear the id only when the sandbox is not // already running, and clear it before the restart. A stopped sandbox has