diff --git a/packages/plugins/sandbox-providers/daytona/README.md b/packages/plugins/sandbox-providers/daytona/README.md index 24f2a38f531..87dcb168832 100644 --- a/packages/plugins/sandbox-providers/daytona/README.md +++ b/packages/plugins/sandbox-providers/daytona/README.md @@ -27,6 +27,7 @@ Notes: - The current published Daytona SDK package is `@daytonaio/sdk`. - The driver supports both `snapshot`-based and `image`-based sandbox creation. If both are set, validation rejects the config as ambiguous. +- Optional `ttlMinutes` sets a positive whole-minute wall-clock lifetime in the creation request. It applies to image and snapshot sandboxes, including probes and interactive setup. The driver requires the provider to confirm the expiry before it returns the sandbox. It preserves an earlier creation expiry when a caller supplies a later deadline. Daytona destroys the sandbox and its files at expiry, even while active, stopped, paused, or archived. Omit this setting to keep existing lifetime behavior. This is separate from inactivity cleanup. - Each cold create uses a unique provider name and ownership labels. If creation fails after Daytona has allocated a sandbox, the driver looks up that exact name, checks every ownership label, and waits for deletion. Failed, missing, or timed-out lookups and failed deletions report unconfirmed cleanup with the provider name; they never return a usable lease. A missing name lookup after an uncertain create is not proof that a delayed provider request cannot create a resource. - For plugin-backed sandbox lease acquisition, unresolved creation cleanup crosses the worker RPC as a validated ownership envelope. The host records `pending_cleanup` before retrying deletion. If only a creation name is known, the provider first returns an ownership-verified sandbox ID without deleting it. The host saves that observation before authorizing deletion. Its existing cleanup sweep and durable spool preserve retries across controller restarts and environment deletion. A missing observed ID confirms cleanup after a lost deletion reply or database update; a name that has never been observed remains unresolved. Provider exceptions and resolved credentials are excluded from that envelope. This requires the matching host and plugin SDK update; probe and custom-image interactive-setup calls still report unconfirmed immediate cleanup without this lease-recovery path. - Reusable leases map to Daytona stop/start semantics. Non-reusable leases are deleted on release. A provider-resolved `target` does not change the identity of an existing sandbox. Release closes the same scoped lease that a later sentinel-verified resume reopens. diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.ts index c722e827782..fd3b668f544 100644 --- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts +++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts @@ -15,7 +15,7 @@ export const DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS = 300_000; // 0.1.6 adds private authenticated WebSocket ingress for paperclip_runner. // 0.1.7 exposes host-owned warm/cold runner lifecycle controls. // 0.1.8 declares the default provider acquisition budget to the host. -const PLUGIN_VERSION = "0.1.8"; +const PLUGIN_VERSION = "0.1.9"; const manifest: PaperclipPluginManifestV1 = { id: PLUGIN_ID, @@ -151,6 +151,11 @@ const manifest: PaperclipPluginManifestV1 = { "Per-call timeout in milliseconds for the sandbox liveness read (refreshData). A silently unresponsive sandbox connection surfaces as a fast error instead of stalling until the outer RPC ceiling. The start and recovery calls derive their own deadline from timeoutMs, not this bound. `0` or less disables the bound. Defaults to 30000 when unset.", default: 30000, }, + ttlMinutes: { + type: "integer", + minimum: 1, + description: "Optional wall-clock lifetime in minutes from creation. Daytona destroys the sandbox and its files at expiry in any state, even while active. Omit to leave the lifetime unchanged.", + }, autoStopInterval: { type: "number", description: diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index d4e69cbba87..94838ec5c11 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -537,7 +537,7 @@ describe("Daytona sandbox provider plugin", () => { // The bundled-plugin boot reconcile refreshes the stored manifest for an // existing install only when the version changes. The acquisition budget needs // the bump to reach an existing install. - expect(manifest.version).toBe("0.1.8"); + expect(manifest.version).toBe("0.1.9"); }); it.each([false, true])("closes duplex routes on lease release even when bridge drain hangs: %s", async (hangDrain) => { @@ -732,6 +732,7 @@ describe("Daytona sandbox provider plugin", () => { autoStopInterval: 15, autoArchiveInterval: 60, autoDeleteInterval: -1, + ttlMinutes: null, reuseLease: true, archiveOnRelease: false, }, @@ -1329,6 +1330,119 @@ describe("Daytona sandbox provider plugin", () => { expect(lease?.expiresAt ?? null).toBeNull(); }); + it.each([{ image: "node:20" }, { snapshot: "base-snapshot" }])("sets the wall-clock ttl in the create request for %j", async source => { + process.env.DAYTONA_API_KEY = "host-key"; + const autoDestroyAt = new Date(Date.now() + 30 * 60_000).toISOString(); + const sandbox = createMockSandbox({ autoDestroyAt }); + mockCreate.mockResolvedValue(sandbox); + const lease = await plugin.definition.onEnvironmentAcquireLease?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", + config: { ...source, ttlMinutes: 30, reuseLease: false }, + }); + expect(mockCreate).toHaveBeenCalledWith(expect.objectContaining({ ...source, ttlMinutes: 30 }), expect.anything()); + expect(lease?.expiresAt).toBe(autoDestroyAt); + expect(sandbox.setTtl).not.toHaveBeenCalled(); + }); + + it("treats a cleared creation ttl as unset", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const config = { image: "node:20", ttlMinutes: "", reuseLease: false }; + expect(await plugin.definition.onEnvironmentValidateConfig?.({ driverKey: "daytona", config })) + .toMatchObject({ ok: true, normalizedConfig: { ttlMinutes: null } }); + const sandbox = createMockSandbox(); + mockCreate.mockResolvedValue(sandbox); + const lease = await plugin.definition.onEnvironmentAcquireLease?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", config, + }); + expect(mockCreate.mock.calls[0][0]).not.toHaveProperty("ttlMinutes"); + expect(lease?.expiresAt ?? null).toBeNull(); + }); + + it.each(["acquire", "probe", "interactive"])("bounds a stalled creation expiry read for %s", async mode => { + vi.useFakeTimers(); + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = createMockSandbox({ autoDestroyAt: new Date(Date.now() + 60_000).toISOString() }); + let finishRead: (() => void) | undefined; + sandbox.refreshData.mockImplementation(() => new Promise(resolve => { finishRead = resolve; })); + mockCreate.mockImplementation(async params => Object.assign(sandbox, { name: params.name, labels: params.labels })); + mockGet.mockResolvedValue(sandbox); + const params = { driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", sessionId: "setup-1", + config: { image: "node:20", ttlMinutes: 1, timeoutMs: 60_000, livenessTimeoutMs: 5, reuseLease: false } }; + const pending = mode === "probe" ? plugin.definition.onEnvironmentProbe!(params) + : mode === "interactive" ? plugin.definition.onEnvironmentStartInteractiveSetup!(params) + : plugin.definition.onEnvironmentAcquireLease!(params); + let settled = false; + const observed = pending.then(value => ({ value, error: null }), error => ({ value: null, error })); + void observed.then(() => { settled = true; }); + try { + await vi.advanceTimersByTimeAsync(20); + expect(settled).toBe(true); + const result = await observed; + expect(result.error?.message ?? (result.value as { metadata?: { error?: string } })?.metadata?.error) + .toContain("did not respond within 5 ms"); + expect(sandbox.delete).toHaveBeenCalled(); + expect(sandbox.process.executeCommand).not.toHaveBeenCalled(); + } finally { + finishRead?.(); + await observed; + vi.useRealTimers(); + } + }); + + it.each([0, -1, 1.5, "invalid", Infinity])("rejects invalid creation ttl %s before allocation", async ttlMinutes => { + process.env.DAYTONA_API_KEY = "host-key"; + const config = { image: "node:20", ttlMinutes, reuseLease: false }; + expect(await plugin.definition.onEnvironmentValidateConfig?.({ driverKey: "daytona", config })) + .toMatchObject({ ok: false, errors: expect.arrayContaining([expect.stringContaining("ttlMinutes")]) }); + await expect(plugin.definition.onEnvironmentAcquireLease?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", config, + })).rejects.toThrow("ttlMinutes"); + expect(mockCreate).not.toHaveBeenCalled(); + }); + + it.each([undefined, "not-a-date", new Date(Date.now() - 60_000).toISOString(), new Date(Date.now() + 60 * 60_000).toISOString()])( + "deletes a new sandbox whose configured creation ttl is not confirmed: %s", async autoDestroyAt => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = createMockSandbox({ autoDestroyAt }); + mockCreate.mockImplementation(async params => Object.assign(sandbox, { name: params.name, labels: params.labels })); + mockGet.mockResolvedValue(sandbox); + await expect(plugin.definition.onEnvironmentAcquireLease?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", + config: { image: "node:20", ttlMinutes: 30, reuseLease: false }, + })).rejects.toThrow("TTL"); + expect(sandbox.delete).toHaveBeenCalled(); + expect(sandbox.process.executeCommand).not.toHaveBeenCalled(); + }, + ); + + it("does not extend the creation ttl for a later caller deadline", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const autoDestroyAt = new Date(Date.now() + 5 * 60_000).toISOString(); + const sandbox = createMockSandbox({ autoDestroyAt }); + mockCreate.mockResolvedValue(sandbox); + const lease = await plugin.definition.onEnvironmentAcquireLease?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", + config: { image: "node:20", ttlMinutes: 30, reuseLease: false }, + requestedExpiresAt: new Date(Date.now() + 10 * 60_000).toISOString(), + }); + expect(lease?.expiresAt).toBe(autoDestroyAt); + expect(sandbox.setTtl).not.toHaveBeenCalled(); + }); + + it("shortens a configured creation ttl for an earlier caller deadline", async () => { + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = createMockSandbox({ autoDestroyAt: new Date(Date.now() + 30 * 60_000).toISOString() }); + sandbox.setTtl.mockImplementation(async minutes => { sandbox.autoDestroyAt = new Date(Date.now() + minutes * 60_000).toISOString(); }); + mockCreate.mockResolvedValue(sandbox); + const requestedExpiresAt = new Date(Date.now() + 5 * 60_000 + 30_000).toISOString(); + const lease = await plugin.definition.onEnvironmentAcquireLease?.({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", + config: { image: "node:20", ttlMinutes: 30, reuseLease: false }, requestedExpiresAt, + }); + expect(sandbox.setTtl).toHaveBeenCalledWith(5); + expect(Date.parse(lease!.expiresAt!)).toBeLessThanOrEqual(Date.parse(requestedExpiresAt)); + }); + it("configures a provider ttl at or before the requested expiry and returns the provider expiry", async () => { process.env.DAYTONA_API_KEY = "host-key"; const autoDestroyAt = new Date(Date.now() + 30 * 60_000).toISOString(); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 8dba1aaadf1..77a3b618953 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -184,6 +184,7 @@ interface DaytonaDriverConfig { autoStopInterval: number | null; autoArchiveInterval: number | null; autoDeleteInterval: number | null; + ttlMinutes: number | null; reuseLease: boolean; archiveOnRelease: boolean; } @@ -307,6 +308,7 @@ function parseDriverConfig(raw: Record): DaytonaDriverConfig { autoStopInterval: parseOptionalInteger(raw.autoStopInterval) ?? DEFAULT_AUTO_STOP_INTERVAL_MINUTES, autoArchiveInterval: parseOptionalInteger(raw.autoArchiveInterval) ?? DEFAULT_AUTO_ARCHIVE_INTERVAL_MINUTES, autoDeleteInterval: parseOptionalInteger(raw.autoDeleteInterval) ?? DEFAULT_AUTO_DELETE_INTERVAL_MINUTES, + ttlMinutes: raw.ttlMinutes == null || raw.ttlMinutes === "" ? null : Number(raw.ttlMinutes), reuseLease: raw.reuseLease === true, archiveOnRelease: raw.archiveOnRelease === true, }; @@ -354,6 +356,7 @@ function buildCreateParams( autoStopInterval: config.autoStopInterval ?? undefined, autoArchiveInterval: config.autoArchiveInterval ?? undefined, autoDeleteInterval: config.autoDeleteInterval ?? undefined, + ...(config.ttlMinutes != null ? { ttlMinutes: config.ttlMinutes } : {}), }; if (config.image) { return { @@ -385,6 +388,12 @@ function validateRuntimeResourceRequest(config: DaytonaDriverConfig): string | n return "Daytona resource settings require image-backed sandbox creation; default sandbox creation cannot override CPU, memory, disk, or GPU."; } +function validateCreationTtl(config: DaytonaDriverConfig): string | null { + return config.ttlMinutes != null && (!Number.isSafeInteger(config.ttlMinutes) || config.ttlMinutes <= 0) + ? "ttlMinutes must be a positive integer when configured." + : null; +} + function buildSandboxLabels(input: { companyId: string; environmentId: string; @@ -689,6 +698,7 @@ function leaseMetadata(input: { ...(input.snapshotIdentity ? { snapshotIdentity: input.snapshotIdentity } : {}), target: input.sandbox.target, timeoutMs: input.config.timeoutMs, + ...(input.config.ttlMinutes != null ? { ttlMinutes: input.config.ttlMinutes } : {}), reuseLease: input.config.reuseLease, // Persisted so the release path (which rebuilds config from lease // metadata) still knows to archive instead of delete. @@ -721,22 +731,17 @@ function expiresAtForMinutes(minutes: number): string { return new Date(Date.now() + minutes * 60_000).toISOString(); } -// Configure a provider-side time-to-live so Daytona destroys the sandbox at or -// before the caller-requested deadline, even after a Paperclip crash or outage. -// `setTtl` counts wall-clock time regardless of the sandbox state, so the destroy -// happens even when the sandbox is stopped, paused, or archived. The function -// returns the real provider destroy time (`autoDestroyAt`) as evidence of the -// provider-side bound. It returns null when the caller sets no deadline, when the -// deadline is invalid, or when the deadline is less than one minute away (Daytona -// TTL granularity is one minute, so a nearer deadline maps to no valid TTL). The -// server then fails closed on a null expiry and releases the lease. async function configureSandboxExpiry(input: { sandbox: Sandbox; requestedExpiresAt: string | null | undefined; nowMs: number; + preserveCreationExpiry: boolean; }): Promise { + const creationExpiry = input.preserveCreationExpiry ? input.sandbox.autoDestroyAt : undefined; + const creationExpiryMs = creationExpiry ? Date.parse(creationExpiry) : Number.NaN; const requestedMs = input.requestedExpiresAt ? Date.parse(input.requestedExpiresAt) : Number.NaN; - if (!Number.isFinite(requestedMs)) return null; + if (!Number.isFinite(requestedMs)) return Number.isFinite(creationExpiryMs) ? creationExpiry! : null; + if (Number.isFinite(creationExpiryMs) && creationExpiryMs <= requestedMs) return creationExpiry!; // Round DOWN so the provider destroy time never lands after the deadline. const ttlMinutes = Math.floor((requestedMs - input.nowMs) / 60_000); if (ttlMinutes < 1) return null; @@ -965,7 +970,7 @@ async function createSandbox( onSnapshotIdentity?: (identity: SnapshotIdentity) => void; } = {}, ): Promise { - const resourceRequestError = validateRuntimeResourceRequest(config); + const resourceRequestError = validateCreationTtl(config) ?? validateRuntimeResourceRequest(config); if (resourceRequestError) { throw new Error(resourceRequestError); } @@ -1014,6 +1019,15 @@ async function createSandbox( timeout: toTimeoutSeconds(config.timeoutMs), })); idleActivity.observe(allocation, providerActivityKey(params, config, sandbox.id)); + if (config.ttlMinutes != null) { + cleanup.observedProviderLeaseId = sandbox.id; + await withLivenessTimeout("sandbox.refreshData", config.livenessTimeoutMs, () => sandbox.refreshData()); + const expiry = Date.parse(sandbox.autoDestroyAt ?? ""); + const nowMs = Date.now(); + if (!Number.isFinite(expiry) || expiry <= nowMs || expiry > nowMs + config.ttlMinutes * 60_000) { + throw new Error("Daytona did not confirm the configured creation TTL."); + } + } return sandbox; } catch (createError) { // Observe the create failure before compensating deletion can stall or fail. @@ -2534,6 +2548,8 @@ const plugin = definePlugin(trackDaytonaHooks({ if (!config.apiKey && !(process.env.DAYTONA_API_KEY?.trim())) { errors.push("Daytona sandbox environments require an API key in config or DAYTONA_API_KEY."); } + const ttlError = validateCreationTtl(config); + if (ttlError) errors.push(ttlError); const resourceRequestError = validateResourceRequest(config); if (resourceRequestError) { errors.push(resourceRequestError); @@ -2676,6 +2692,7 @@ const plugin = definePlugin(trackDaytonaHooks({ sandbox, requestedExpiresAt: params.requestedExpiresAt, nowMs: Date.now(), + preserveCreationExpiry: config.ttlMinutes != null, }); assertActive(); phase = "sentinel"; diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index e0ed06db889..a97715ed1ed 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -941,6 +941,14 @@ pnpm test:e2e:runner:models:update ## Daytona image +The ephemeral Daytona fixture requests a 60-minute wall-clock lifetime at creation. +The provider destroys each sandbox at that expiry in any state. This bound includes +room for each declared attempt and the acquisition budget. It supplements the +fixture's inactivity cleanup and remains in force after a controller crash. +The provider must confirm the expiry before setup can use the sandbox. Cost reports +still require lifecycle evidence and preserve uncertain cleanup or charges. + + Use the immutable digest printed by the `Publish verified Daytona image` job, or publish the current source locally: diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 09c95dd259a..bb499b6970c 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -253,6 +253,21 @@ describe("runner E2E catalog", () => { ).toEqual({ local: 8 * 60_000, daytona: 12 * 60_000 }); }); + it("bounds ephemeral Daytona lifetime beyond each declared attempt and acquire budget", () => { + const environment = runnerEnvironments.find(entry => entry.id === "daytona")!; + const payload = environment.buildEnvironment({ + executionId: "bounded-lifetime", daytonaImage: `runner@sha256:${"a".repeat(64)}`, + secretRefs: { DAYTONA_API_KEY: { type: "secret_ref", secretId: "22222222-2222-4222-8222-222222222222", version: "latest" } }, + }); + const lifetimeMs = Number(payload.config?.ttlMinutes) * 60_000; + expect(Number.isFinite(lifetimeMs)).toBe(true); + const cells = runnerMatrix.filter(entry => entry.environment === environment); + expect(cells.length).toBeGreaterThan(0); + for (const cell of cells) { + expect(lifetimeMs, cell.id).toBeGreaterThan(cell.task.attemptTimeoutMs.daytona + Number(payload.config?.timeoutMs)); + } + }); + it("defines the warm Daytona continuity fixture as exactly two Codex cells", () => { expect(daytonaWarmEnvironment).toMatchObject({ id: "daytona", diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index 0f49c6e653e..24691ef102c 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -538,6 +538,7 @@ export const runnerEnvironments: readonly EnvironmentFixture[] = [ autoStopInterval: 5, autoArchiveInterval: 15, autoDeleteInterval: 60, + ttlMinutes: 60, timeoutMs: 300_000, livenessTimeoutMs: 30_000, },