diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 11e0c65cff4..7a3180bb1bb 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -3019,6 +3019,90 @@ describe("Daytona sandbox provider plugin", () => { expect(sandbox.process.deleteSession).toHaveBeenCalledWith(sessionId); }); + // An idle gap longer than `autoStopInterval` (for example, a long host-side + // workspace pack) lets Daytona stop the sandbox. Workspace realize and + // sync-in then restart it before the next command. A restarted sandbox has + // no process sessions, so the next command must open a fresh session + // instead of sending the dead id and failing with "session not found". + const autoStoppedSandbox = () => { + const sandbox = createMockSandbox({ state: "started" }); + sandbox.start.mockImplementation(async () => { + sandbox.state = "started"; + }); + return sandbox; + }; + + it("opens a fresh session after workspace realize restarts an auto-stopped sandbox", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = autoStoppedSandbox(); + mockGet.mockResolvedValue(sandbox); + + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(1); + const staleSessionId = sandbox.process.createSession.mock.calls[0]![0] as string; + + sandbox.state = "stopped"; + await plugin.definition.onEnvironmentRealizeWorkspace?.({ + driverKey: "daytona", + companyId: "company-1", + environmentId: "env-1", + config: { timeoutMs: 300000, reuseLease: false }, + lease: { providerLeaseId: "sandbox-123", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + workspace: { localPath: "/tmp/workspace" }, + } as never); + expect(sandbox.start).toHaveBeenCalledTimes(1); + + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(2); + const freshSessionId = sandbox.process.createSession.mock.calls[1]![0] as string; + expect(freshSessionId).not.toBe(staleSessionId); + const lastExec = sandbox.process.executeSessionCommand.mock.calls.at(-1)!; + expect(lastExec[0]).toBe(freshSessionId); + }); + + it("opens a fresh session after sync-in restarts an auto-stopped sandbox", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = autoStoppedSandbox(); + mockGet.mockResolvedValue(sandbox); + + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(1); + + sandbox.state = "stopped"; + await plugin.definition.onEnvironmentSyncIn?.({ + driverKey: "daytona", + companyId: "company-1", + environmentId: "env-1", + config: { timeoutMs: 300000, reuseLease: false }, + lease: { providerLeaseId: "sandbox-123", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + operations: [], + } as never); + expect(sandbox.start).toHaveBeenCalledTimes(1); + + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(2); + }); + + it("keeps the live session when realize finds the sandbox already running", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = autoStoppedSandbox(); + mockGet.mockResolvedValue(sandbox); + + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + await plugin.definition.onEnvironmentRealizeWorkspace?.({ + driverKey: "daytona", + companyId: "company-1", + environmentId: "env-1", + config: { timeoutMs: 300000, reuseLease: false }, + lease: { providerLeaseId: "sandbox-123", metadata: { remoteCwd: "/home/daytona/paperclip-workspace" } }, + workspace: { localPath: "/tmp/workspace" }, + } as never); + await plugin.definition.onEnvironmentExecute?.(sessionExecParams()); + + expect(sandbox.start).not.toHaveBeenCalled(); + expect(sandbox.process.createSession).toHaveBeenCalledTimes(1); + }); + it("emits a session.open span on create and a session.close span on delete", async () => { process.env.DAYTONA_API_KEY = "host-key"; const sandbox = createMockSandbox(); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 2761032d6ee..4300b21f938 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -539,8 +539,15 @@ async function terminateAtProvider(scope: SandboxScope, operation: string, ac return withLivenessTimeout(operation, timeoutMs + LIVENESS_START_TIMEOUT_MARGIN_MS, action); } -async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number): Promise { +// Start (or recover) a sandbox that is not running. Every restart path goes +// through here with the lease scope, because a restart destroys every Daytona +// process session. The stored session id for the lease is dropped before the +// restart, so the next command opens a fresh session instead of sending a dead +// id and failing with "session not found". A running sandbox keeps its live +// session, so the id is left in place when no restart happens. +async function ensureSandboxStarted(sandbox: Sandbox, timeoutSeconds: number, scope: SandboxScope): Promise { if (sandbox.state === "started") return; + sandboxHandleSessionStore.clear(scope); // Bound the lifecycle call just past its own SDK deadline. A normal slow start // finishes within `timeoutSeconds`; only a connection-level hang the SDK // deadline misses reaches this wrapper bound. @@ -2891,7 +2898,7 @@ const plugin = definePlugin(trackDaytonaHooks({ await traceLeaseStep("lease.channel_cleanup", () => closeDaytonaDuplexChannelsForLease(params.providerLeaseId)); } const resumedFromState = sandbox.state ?? null; - await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); + await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs), scope); try { return await traceLeaseStep("lease.verify", async () => { const remoteCwd = await traceLeaseStep("lease.workspace", () => resolveSandboxWorkingDirectory(sandbox, false)); @@ -3144,7 +3151,7 @@ const plugin = definePlugin(trackDaytonaHooks({ }; await withSandboxActivityGate(scope, async () => { const sandbox = await getSandbox(scope, { bypassTeardownGate: true }); - await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); + await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs), scope); await sandbox.fs.createFolder(remoteCwd, "755"); }); } @@ -3239,7 +3246,7 @@ const plugin = definePlugin(trackDaytonaHooks({ }; } - await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs)); + await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs), scope); const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs)); const connection = params.includeConnectionPayload === true @@ -3458,13 +3465,10 @@ const plugin = definePlugin(trackDaytonaHooks({ providerLeaseId, config, }; - if (sandbox.state !== "started") { - // A provider restart destroys Daytona process sessions. Drop the stale - // session id before starting the sandbox so runnerd recovery opens a - // new session instead of retrying a dead one for its whole grace. - sandboxHandleSessionStore.clear(scope); - } - await ensureSandboxStarted(sandbox, toTimeoutSeconds(resolveTimeoutMs(params.timeoutMs, config))); + // A provider restart destroys Daytona process sessions; the start helper + // drops the stale session id so runnerd recovery opens a new session + // instead of retrying a dead one for its whole grace. + await ensureSandboxStarted(sandbox, toTimeoutSeconds(resolveTimeoutMs(params.timeoutMs, config)), scope); // Dispatch the command. A normal command runs in the persistent session: // the provider opens the one session on a cache miss and runs every command // in it. The provider never falls back to a one-shot command to open a @@ -3507,23 +3511,18 @@ const plugin = definePlugin(trackDaytonaHooks({ throw new Error("Daytona runner ingress requires a provider lease id."); } const config = parseDriverConfig(params.config); + const scope: SandboxScope = { + driverKey: params.driverKey, + companyId: params.companyId, + environmentId: params.environmentId, + providerLeaseId, + config, + }; return await withSandboxActivityGate( - { - driverKey: params.driverKey, - companyId: params.companyId, - environmentId: params.environmentId, - providerLeaseId, - config, - }, + scope, async () => { - const sandbox = await acquireRunnerIngress("sandbox_lookup", () => getSandbox({ - driverKey: params.driverKey, - companyId: params.companyId, - environmentId: params.environmentId, - providerLeaseId, - config, - })); - await acquireRunnerIngress("sandbox_start", () => ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs))); + const sandbox = await acquireRunnerIngress("sandbox_lookup", () => getSandbox(scope)); + await acquireRunnerIngress("sandbox_start", () => ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs), scope)); await acquireRunnerIngress("sandbox_refresh", () => withLivenessTimeout( "sandbox.refreshData", config.livenessTimeoutMs, @@ -3584,7 +3583,7 @@ const plugin = definePlugin(trackDaytonaHooks({ sandboxHandleWritableDirs.recordWritableTargets(scope, params.operations); return await withSandboxActivityGate(scope, async () => { const sandbox = await getSandbox(scope, { bypassTeardownGate: true }); - await ensureSandboxStarted(sandbox, timeoutSeconds); + await ensureSandboxStarted(sandbox, timeoutSeconds, scope); const result = await performSyncIn({ sandbox, operations: params.operations, @@ -3617,7 +3616,7 @@ const plugin = definePlugin(trackDaytonaHooks({ return await withSandboxActivityGate(scope, async () => { const sandbox = await withEnvironmentSyncTransferStep("sandbox_access", async () => { const resolved = await getSandbox(scope, { bypassTeardownGate: true }); - await ensureSandboxStarted(resolved, timeoutSeconds); + await ensureSandboxStarted(resolved, timeoutSeconds, scope); return resolved; }); const result = await performSyncOut({