diff --git a/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md b/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md index c831656799..8cc2ea4100 100644 --- a/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md +++ b/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md @@ -10,6 +10,29 @@ Read [Sandbox file-sync lifecycle hooks](./SANDBOX_FILE_SYNC_HOOKS.md) for the native file-transfer hooks. Read the driver declaration shape in [the plugin specification](./PLUGIN_SPEC.md). +## Fresh lease acquisition timeout + +A driver can declare `defaultAcquireTimeoutMs` as a positive integer of at most +86,400,000 milliseconds. The host uses it for `environmentAcquireLease` when the +resolved config has no positive finite numeric `timeoutMs`. A positive numeric +`bridgeRequestTimeoutMs` can extend that budget. The host adds 30 seconds for RPC +overhead. This applies to both sandbox providers and generic plugin drivers. + +Daytona declares 300,000 milliseconds for the entire fresh acquisition. Creation, +workspace setup, shell detection, expiry setup, and inline failure cleanup share +that budget. Its host RPC can therefore wait 330 seconds instead of the worker's +normal 30 seconds. On timeout the provider returns the attempt's scoped cleanup +record for durable host reconciliation. Late SDK results cannot admit the lease +or start the next setup phase. +Drivers that omit the declaration keep the existing fallback. The host does not +infer this budget from config-schema defaults: a field named `timeoutMs` can +describe sandbox lifetime instead of the time needed to acquire it. + +This declaration does not change provider config, lease expiry, the resume +deadline, or other lifecycle calls. Providers must still enforce their operation +timeouts and report uncertain allocations for cleanup. A bundled plugin must bump +its manifest version when adding the field so existing installations receive it. + ## How the host resolves an effective capability The host never trusts a declaration alone. For every run it resolves each diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.ts index 3eced73a22..c722e82778 100644 --- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts +++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts @@ -1,6 +1,7 @@ import type { PaperclipPluginManifestV1 } from "@paperclipai/plugin-sdk"; const PLUGIN_ID = "paperclip.daytona-sandbox-provider"; +export const DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS = 300_000; // The bundled-plugin boot reconcile refreshes the persisted manifest for an // existing install only when PLUGIN_VERSION changes. A manifest change without a // version bump never reaches an existing install. The reconcile also reads the @@ -13,7 +14,8 @@ const PLUGIN_ID = "paperclip.daytona-sandbox-provider"; // 0.1.5 adds the `duplexCommandStream` sandbox capability to the driver. // 0.1.6 adds private authenticated WebSocket ingress for paperclip_runner. // 0.1.7 exposes host-owned warm/cold runner lifecycle controls. -const PLUGIN_VERSION = "0.1.7"; +// 0.1.8 declares the default provider acquisition budget to the host. +const PLUGIN_VERSION = "0.1.8"; const manifest: PaperclipPluginManifestV1 = { id: PLUGIN_ID, @@ -31,6 +33,7 @@ const manifest: PaperclipPluginManifestV1 = { environmentDrivers: [ { driverKey: "daytona", + defaultAcquireTimeoutMs: DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS, kind: "sandbox_provider", displayName: "Daytona Sandbox", description: @@ -139,8 +142,8 @@ const manifest: PaperclipPluginManifestV1 = { }, timeoutMs: { type: "number", - description: "Timeout for Daytona create/start/stop/execute operations in milliseconds.", - default: 300000, + description: "Timeout for Daytona operations in milliseconds. Fresh lease acquisition shares one budget across creation, setup, and inline cleanup.", + default: DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS, }, livenessTimeoutMs: { type: "number", diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index 86b4fbfd65..358f4f5974 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -135,6 +135,7 @@ describe("Daytona sandbox provider plugin", () => { expect(plugin.definition.onEnvironmentStartInteractiveSetup).toBeTypeOf("function"); expect(plugin.definition.onEnvironmentCaptureTemplate).toBeTypeOf("function"); expect(manifest.environmentDrivers?.[0]).toMatchObject({ + defaultAcquireTimeoutMs: 300_000, supportsInteractiveSetup: true, interactiveSetupConnectionTypes: ["ssh"], supportsTemplateCapture: true, @@ -249,9 +250,9 @@ describe("Daytona sandbox provider plugin", () => { it("bumps the plugin version so the server reconciles the stored manifest", () => { // The bundled-plugin boot reconcile refreshes the stored manifest for an - // existing install only when the version changes. The duplex capability needs + // existing install only when the version changes. The acquisition budget needs // the bump to reach an existing install. - expect(manifest.version).toBe("0.1.7"); + expect(manifest.version).toBe("0.1.8"); }); it.each([false, true])("closes duplex routes on lease release even when bridge drain hangs: %s", async (hangDrain) => { @@ -524,6 +525,85 @@ describe("Daytona sandbox provider plugin", () => { }); }); + describe("fresh acquisition deadline", () => { + const params = { + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", + config: { image: "node:20", reuseLease: false }, + }; + + beforeEach(() => { + process.env.DAYTONA_API_KEY = "host-key"; + vi.useFakeTimers(); + }); + afterEach(() => { vi.useRealTimers(); }); + + it("allows a slow create and setup that finish inside the total budget", async () => { + const sandbox = createMockSandbox(); + mockCreate.mockImplementation(() => new Promise(resolve => setTimeout(() => resolve(sandbox), 280_000))); + sandbox.process.executeCommand.mockImplementation(() => new Promise(resolve => setTimeout(() => resolve({ result: "bash" }), 15_000))); + const pending = plugin.definition.onEnvironmentAcquireLease!(params); + await vi.advanceTimersByTimeAsync(295_000); + expect(await pending).toMatchObject({ providerLeaseId: sandbox.id }); + expect(vi.getTimerCount()).toBe(0); + }); + + it("journals ownership before the host deadline when setup outlasts creation", async () => { + const sandbox = createMockSandbox(); + mockCreate.mockImplementation(() => new Promise(resolve => setTimeout(() => resolve(sandbox), 280_000))); + sandbox.process.executeCommand.mockImplementation(() => new Promise(resolve => setTimeout(() => resolve({ result: "bash" }), 55_000))); + const pending = plugin.definition.onEnvironmentAcquireLease!(params).catch(error => error); + await vi.advanceTimersByTimeAsync(300_000); + const cleanup = readEnvironmentCreationCleanupError(await pending); + expect(cleanup).toMatchObject({ companyId: params.companyId, environmentId: params.environmentId, + runId: params.runId, observedProviderLeaseId: sandbox.id, providerLeaseId: mockCreate.mock.calls[0][0].name }); + await vi.advanceTimersByTimeAsync(35_000); + expect(sandbox.setTtl).not.toHaveBeenCalled(); + expect(sandbox.fs.uploadFile).not.toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + }); + + it("records an uncertain create and rejects its late result without starting setup", async () => { + const sandbox = createMockSandbox(); + mockCreate.mockImplementation(() => new Promise(resolve => setTimeout(() => resolve(sandbox), 310_000))); + const pending = plugin.definition.onEnvironmentAcquireLease!(params).catch(error => error); + await vi.advanceTimersByTimeAsync(300_000); + const cleanup = readEnvironmentCreationCleanupError(await pending); + expect(cleanup?.providerLeaseId).toBe(mockCreate.mock.calls[0][0].name); + expect(cleanup?.observedProviderLeaseId).toBeUndefined(); + await vi.advanceTimersByTimeAsync(10_000); + expect(sandbox.getWorkDir).not.toHaveBeenCalled(); + expect(sandbox.process.executeCommand).not.toHaveBeenCalled(); + expect(sandbox.delete).not.toHaveBeenCalled(); + }); + + it("keeps a failed setup's ownership when inline deletion does not finish", async () => { + const sandbox = createMockSandbox(); + mockCreate.mockResolvedValue(sandbox); + sandbox.getWorkDir.mockRejectedValue(new Error("workspace unavailable")); + sandbox.delete.mockImplementation(() => new Promise(() => {})); + const pending = plugin.definition.onEnvironmentAcquireLease!({ ...params, + config: { ...params.config, timeoutMs: 2_000 }, + }).catch(error => error); + await vi.advanceTimersByTimeAsync(2_000); + expect(readEnvironmentCreationCleanupError(await pending)).toMatchObject({ + observedProviderLeaseId: sandbox.id, companyId: params.companyId, runId: params.runId, + }); + expect(sandbox.delete).toHaveBeenCalledOnce(); + }); + + it("keeps ownership when setup and its inline deletion both reject", async () => { + const sandbox = createMockSandbox(); + mockCreate.mockResolvedValue(sandbox); + sandbox.getWorkDir.mockRejectedValue(new Error("workspace unavailable")); + sandbox.delete.mockRejectedValue(new Error("delete unavailable")); + const error = await plugin.definition.onEnvironmentAcquireLease!(params).catch(error => error); + expect(readEnvironmentCreationCleanupError(error)).toMatchObject({ + observedProviderLeaseId: sandbox.id, companyId: params.companyId, runId: params.runId, + }); + expect(vi.getTimerCount()).toBe(0); + }); + }); + describe("failed sandbox creation cleanup", () => { const params = { driverKey: "daytona", companyId: "company-1", environmentId: "env-1", runId: "run-1", diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 8e1b19866f..ac8465254b 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -45,6 +45,7 @@ import type { PluginSyncOperation, } from "@paperclipai/plugin-sdk"; import { performSyncIn, performSyncOut, withProviderSpan } from "./file-sync.js"; +import { DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS } from "./manifest.js"; // The Claude `setup-token` login pseudo-terminal (PTY) session for this provider. // The session runs the login command on a real pseudo-terminal, streams the @@ -275,7 +276,7 @@ function parseOptionalNumber(value: unknown): number | null { } function parseDriverConfig(raw: Record): DaytonaDriverConfig { - const timeoutMs = Number(raw.timeoutMs ?? 300_000); + const timeoutMs = Number(raw.timeoutMs ?? DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS); const livenessTimeoutMs = Number(raw.livenessTimeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS); return { apiKey: parseOptionalString(raw.apiKey), @@ -284,7 +285,7 @@ function parseDriverConfig(raw: Record): DaytonaDriverConfig { snapshot: parseOptionalString(raw.snapshot), image: parseOptionalString(raw.image), language: parseOptionalString(raw.language), - timeoutMs: Number.isFinite(timeoutMs) ? Math.trunc(timeoutMs) : 300_000, + timeoutMs: Number.isFinite(timeoutMs) ? Math.trunc(timeoutMs) : DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS, livenessTimeoutMs: Number.isFinite(livenessTimeoutMs) ? Math.trunc(livenessTimeoutMs) : DEFAULT_LIVENESS_TIMEOUT_MS, cpu: parseOptionalNumber(raw.cpu), memory: parseOptionalNumber(raw.memory), @@ -483,7 +484,7 @@ async function drainSandboxBeforeTermination(sandbox: Sandbox, scope: SandboxSco } async function terminateAtProvider(scope: SandboxScope, operation: string, action: () => Promise) { - const timeoutMs = scope.config.timeoutMs > 0 ? scope.config.timeoutMs : 300_000; + const timeoutMs = scope.config.timeoutMs > 0 ? scope.config.timeoutMs : DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS; return withLivenessTimeout(operation, timeoutMs + LIVENESS_START_TIMEOUT_MARGIN_MS, action); } @@ -936,7 +937,10 @@ function resolveSyncRemoteDir(lease: { metadata?: Record | null async function createSandbox( params: PluginEnvironmentAcquireLeaseParams | PluginEnvironmentProbeParams | PluginEnvironmentStartInteractiveSetupParams, config: DaytonaDriverConfig, - options: { purpose?: string } = {}, + options: { + purpose?: string; + onCreateAttempt?: (cleanup: PluginEnvironmentCreationCleanup) => void; + } = {}, ): Promise { const resourceRequestError = validateRuntimeResourceRequest(config); if (resourceRequestError) { @@ -958,16 +962,17 @@ async function createSandbox( // The SDK mutates params.labels (for example, code-toolbox-language). // Preserve our immutable ownership snapshot for validation and retry. const createParams = { ...buildCreateParams(config, { ...labels }), name }; + const cleanup: PluginEnvironmentCreationCleanup = { + providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, + ...("runId" in params ? { runId: params.runId } : {}), + attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), + }; + options.onCreateAttempt?.(cleanup); try { return await client.create(createParams, { timeout: toTimeoutSeconds(config.timeoutMs), }); } catch (createError) { - const cleanup: PluginEnvironmentCreationCleanup = { - providerLeaseId: name, companyId: params.companyId, environmentId: params.environmentId, - ...("runId" in params ? { runId: params.runId } : {}), - attemptId, labels, accountFingerprint: sandboxAccountDiscriminator(config), - }; try { // A not-found lookup after an uncertain create is not a deletion receipt: // the provider may still materialize the request. Keep the name in the @@ -2223,60 +2228,118 @@ const plugin = definePlugin({ params: PluginEnvironmentAcquireLeaseParams, ): Promise { const config = parseDriverConfig(params.config); - const sandbox = await createSandbox(params, config); - try { - const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); - const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(config.timeoutMs)); - // Configure a provider-side destroy time at or before a caller deadline, so - // an abandoned sandbox self-destroys even if Paperclip is down. The lease - // carries the real provider expiry (or none) as evidence of the bound. - const expiresAt = await configureSandboxExpiry({ - sandbox, - requestedExpiresAt: params.requestedExpiresAt, - nowMs: Date.now(), + // One budget covers creation, setup, and inline cleanup. The host leaves + // 30 seconds beyond this deadline to receive/journal the cleanup record. + const budgetMs = config.timeoutMs > 0 ? config.timeoutMs : DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS; + const deadline = Date.now() + budgetMs; + let expired = false; + let phase = "create"; + let cleanup: PluginEnvironmentCreationCleanup | undefined; + const timeoutFailure = () => { + expired = true; + const cause = new Error(`Daytona lease acquisition exceeded ${budgetMs} ms during ${phase}`); + return cleanup + ? new PluginEnvironmentCreationCleanupError([cause], + "Daytona lease acquisition timed out; allocation cleanup is pending", + { ...cleanup, labels: { ...cleanup.labels } }) + : cause; + }; + const assertActive = () => { + if (expired || Date.now() >= deadline) throw timeoutFailure(); + }; + const acquire = async (): Promise => { + const sandbox = await createSandbox(params, config, { + onCreateAttempt: (attempt) => { cleanup = attempt; }, }); - const workspaceSentinel = await writeWorkspaceSentinel({ - sandbox, - remoteCwd, - params, - config, - timeoutSeconds: toTimeoutSeconds(config.timeoutMs), - }); - sandboxHandleLeaseAdmissionStates.open({ - driverKey: params.driverKey, - companyId: params.companyId, - environmentId: params.environmentId, - providerLeaseId: sandbox.id, - config, - }); - // Seed the handle cache with the fresh handle under the exact scope that - // `onEnvironmentRealizeWorkspace` reads (providerLeaseId === sandbox.id). - // Realize then reuses this handle instead of paying a real `client.get`. - sandboxHandleCache.seed( - { + try { + assertActive(); + if (cleanup) cleanup.observedProviderLeaseId = sandbox.id; + phase = "workspace"; + const remoteCwd = await resolveSandboxWorkingDirectory(sandbox); + assertActive(); + phase = "shell"; + const shellCommand = await detectSandboxShellCommand(sandbox, toTimeoutSeconds(Math.max(1, deadline - Date.now()))); + assertActive(); + // Configure a provider-side destroy time at or before a caller deadline, so + // an abandoned sandbox self-destroys even if Paperclip is down. The lease + // carries the real provider expiry (or none) as evidence of the bound. + phase = "expiry"; + const expiresAt = await configureSandboxExpiry({ + sandbox, + requestedExpiresAt: params.requestedExpiresAt, + nowMs: Date.now(), + }); + assertActive(); + phase = "sentinel"; + const workspaceSentinel = await writeWorkspaceSentinel({ + sandbox, + remoteCwd, + params, + config, + timeoutSeconds: toTimeoutSeconds(Math.max(1, deadline - Date.now())), + }); + assertActive(); + sandboxHandleLeaseAdmissionStates.open({ driverKey: params.driverKey, companyId: params.companyId, environmentId: params.environmentId, providerLeaseId: sandbox.id, config, - }, - sandbox, - ); - return { - providerLeaseId: sandbox.id, - expiresAt, - metadata: leaseMetadata({ - config, + }); + // Seed the handle cache with the fresh handle under the exact scope that + // `onEnvironmentRealizeWorkspace` reads (providerLeaseId === sandbox.id). + // Realize then reuses this handle instead of paying a real `client.get`. + sandboxHandleCache.seed( + { + driverKey: params.driverKey, + companyId: params.companyId, + environmentId: params.environmentId, + providerLeaseId: sandbox.id, + config, + }, sandbox, - shellCommand, - remoteCwd, - resumedLease: false, - workspaceSentinel, + ); + return { + providerLeaseId: sandbox.id, + expiresAt, + metadata: leaseMetadata({ + config, + sandbox, + shellCommand, + remoteCwd, + resumedLease: false, + workspaceSentinel, + }), + }; + } catch (error) { + // After timeout the host owns the durable cleanup record. A late SDK + // completion must not admit this lease or start another setup phase. + if (!expired) { + phase = "cleanup"; + try { + await sandbox.delete(toTimeoutSeconds(Math.max(1, deadline - Date.now()))); + } catch (cleanupError) { + if (cleanup) { + throw new PluginEnvironmentCreationCleanupError([error, cleanupError], + "Daytona lease setup failed; allocation cleanup is pending", + { ...cleanup, labels: { ...cleanup.labels } }); + } + throw error; + } + } + throw error; + } + }; + let timer: ReturnType | undefined; + try { + return await Promise.race([ + acquire(), + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(timeoutFailure()), budgetMs); }), - }; - } catch (error) { - await sandbox.delete(toTimeoutSeconds(config.timeoutMs)).catch(() => undefined); - throw error; + ]); + } finally { + clearTimeout(timer); } }, diff --git a/packages/shared/src/types/plugin.ts b/packages/shared/src/types/plugin.ts index 7465f693f0..68662753a3 100644 --- a/packages/shared/src/types/plugin.ts +++ b/packages/shared/src/types/plugin.ts @@ -205,6 +205,13 @@ export interface PluginEnvironmentDriverDeclaration { displayName: string; /** Optional description for operator-facing docs or UI affordances. */ description?: string; + /** + * Default provider budget for a fresh lease acquisition, in milliseconds. + * The host adds RPC overhead. A valid explicit config.timeoutMs overrides + * this default; bridgeRequestTimeoutMs can extend the resulting budget. + * Omit to retain the worker's normal RPC timeout. This is not lease lifetime. + */ + defaultAcquireTimeoutMs?: number; /** * Sandbox providers must opt in before the host retains and resumes provider * leases across runs. Providers without this flag keep per-run acquire/release diff --git a/packages/shared/src/validators/plugin.test.ts b/packages/shared/src/validators/plugin.test.ts index ff2e7a9935..043a048b1e 100644 --- a/packages/shared/src/validators/plugin.test.ts +++ b/packages/shared/src/validators/plugin.test.ts @@ -276,6 +276,24 @@ describe("plugin UI slot validators", () => { }); describe("sandbox provider capability declaration validators", () => { + it("preserves a declared acquisition budget and keeps it optional", () => { + const parsed = pluginManifestV1Schema.parse( + buildSandboxProviderManifest({ defaultAcquireTimeoutMs: 300_000 }), + ); + expect(parsed.environmentDrivers?.[0]?.defaultAcquireTimeoutMs).toBe(300_000); + const legacy = pluginManifestV1Schema.parse(buildSandboxProviderManifest({})); + expect(legacy.environmentDrivers?.[0]?.defaultAcquireTimeoutMs).toBeUndefined(); + }); + + it.each([0, -1, 1.5, Infinity, NaN, "300000", 86_400_001])( + "rejects an invalid acquisition budget: %s", + (defaultAcquireTimeoutMs) => { + expect(pluginManifestV1Schema.safeParse( + buildSandboxProviderManifest({ defaultAcquireTimeoutMs }), + ).success).toBe(false); + }, + ); + it("test_manifest_accepts_sandbox_capabilities_and_rejects_unknown_capability_keys", () => { const parsed = pluginManifestV1Schema.parse( buildSandboxProviderManifest({ diff --git a/packages/shared/src/validators/plugin.ts b/packages/shared/src/validators/plugin.ts index 6b0ebe6e40..55a915a069 100644 --- a/packages/shared/src/validators/plugin.ts +++ b/packages/shared/src/validators/plugin.ts @@ -184,6 +184,7 @@ export const pluginEnvironmentDriverDeclarationSchema = z.object({ kind: z.enum(["environment_driver", "sandbox_provider"]).optional(), displayName: z.string().min(1).max(100), description: z.string().max(500).optional(), + defaultAcquireTimeoutMs: z.number().int().positive().max(86_400_000).optional(), supportsReusableLeases: z.boolean().optional(), sandboxCapabilities: sandboxProviderCapabilitiesSchema.optional(), supportsInteractiveSetup: z.boolean().optional(), diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 285d01c4db..bffb022301 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -4685,14 +4685,20 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(checks).toBe(readyAfterChecks); }); - it("extends plugin-backed sandbox lease RPC timeouts from provider config", async () => { + it.each([ + { label: "explicit provider and bridge budgets", config: { timeoutMs: 1_234, bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 70_000 }, + { label: "declared acquisition default", config: {}, declared: 300_000, expected: 330_000 }, + { label: "explicit shorter provider budget", config: { timeoutMs: 5_000 }, declared: 300_000, expected: 35_000 }, + { label: "bridge budget does not shorten acquisition default", config: { bridgeRequestTimeoutMs: 40_000 }, declared: 300_000, expected: 330_000 }, + { label: "bridge extends the acquisition default", config: { bridgeRequestTimeoutMs: 400_000 }, declared: 300_000, expected: 430_000 }, + { label: "undeclared provider retains worker default", config: {}, declared: undefined, expected: undefined }, + ])("uses $label for plugin-backed sandbox acquisition", async ({ config, declared, expected }) => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", - timeoutMs: 1_234, - bridgeRequestTimeoutMs: 40_000, + ...config, reuseLease: false, }; const environment = { @@ -4728,7 +4734,10 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { driverKey: "fake-plugin", kind: "sandbox_provider", displayName: "Fake Plugin", - configSchema: { type: "object" }, + defaultAcquireTimeoutMs: declared, + // A timeoutMs schema default can mean lease lifetime (for example + // E2B). Only the dedicated declaration sets the host RPC budget. + configSchema: { type: "object", properties: { timeoutMs: { type: "number", default: 3_600_000 } } }, }, ], }, @@ -4746,8 +4755,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { metadata: { provider: "fake-plugin", image: "fake:test", - timeoutMs: 1_234, - bridgeRequestTimeoutMs: 40_000, + ...config, reuseLease: false, }, }; @@ -4774,12 +4782,11 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { driverKey: "fake-plugin", config: { image: "fake:test", - timeoutMs: 1_234, - bridgeRequestTimeoutMs: 40_000, + ...config, reuseLease: false, }, }), - 70_000, + expected, ); }); @@ -7068,7 +7075,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { })); }); - it("delegates plugin environment leases through the plugin worker manager", async () => { + it.each([undefined, 300_000])("delegates plugin environment leases with acquisition budget %s", async (defaultAcquireTimeoutMs) => { const pluginId = randomUUID(); const expiresAt = new Date(Date.now() + 60_000).toISOString(); const workerManager = { @@ -7129,6 +7136,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { { driverKey: "fake-plugin", displayName: "Fake plugin", + defaultAcquireTimeoutMs, configSchema: { type: "object" }, }, ], @@ -7158,7 +7166,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { adapterType: undefined, runId, workspaceMode: undefined, - }); + }, ...(defaultAcquireTimeoutMs === undefined ? [] : [330_000])); expect(acquired.lease.providerLeaseId).toBe("plugin-lease-1"); expect(acquired.lease.expiresAt?.toISOString()).toBe(expiresAt); expect(acquired.lease.metadata).toMatchObject({ diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index e5956c1e39..cb484cc5ba 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -485,9 +485,16 @@ export interface EnvironmentDriverReleaseInput { status: Extract; } -function resolvePluginSandboxRpcTimeoutMs(config: Record): number | undefined { +function resolvePluginSandboxRpcTimeoutMs( + config: Record, + defaultTimeoutMs?: number, +): number | undefined { + const configuredTimeoutMs = typeof config.timeoutMs === "number" && + Number.isFinite(config.timeoutMs) && config.timeoutMs > 0 + ? config.timeoutMs + : defaultTimeoutMs; const timeoutCandidates = [ - typeof config.timeoutMs === "number" ? config.timeoutMs : undefined, + configuredTimeoutMs, typeof config.bridgeRequestTimeoutMs === "number" ? config.bridgeRequestTimeoutMs : undefined, ] .filter((value): value is number => typeof value === "number" && Number.isFinite(value) && value > 0) @@ -2084,7 +2091,10 @@ function createSandboxEnvironmentDriver( ? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) } : {}), }, - resolvePluginSandboxRpcTimeoutMs(workerConfig), + resolvePluginSandboxRpcTimeoutMs( + workerConfig, + pluginProvider.resolved.driver.defaultAcquireTimeoutMs, + ), ); } catch (error) { const cleanup = readEnvironmentCreationCleanupError(error); @@ -3390,7 +3400,7 @@ function createPluginEnvironmentDriver( if (!workerManager.isRunning(plugin.id)) { throw new Error(`Plugin environment driver "${pluginDriverProviderKey(config)}" has no running worker.`); } - return { plugin }; + return { plugin, driver }; } async function resolvePluginDriverForRelease(input: EnvironmentDriverReleaseInput) { @@ -3459,7 +3469,12 @@ function createPluginEnvironmentDriver( if (parsed.driver !== "plugin") { throw new Error(`Expected plugin environment config for driver "${input.environment.driver}".`); } - const { plugin } = await resolvePluginDriver(parsed.config); + const { plugin, driver } = await resolvePluginDriver(parsed.config); + const rpcTimeoutMs = resolvePluginSandboxRpcTimeoutMs( + parsed.config.driverConfig, + driver.defaultAcquireTimeoutMs, + ); + const timeoutOverride: [number?] = rpcTimeoutMs === undefined ? [] : [rpcTimeoutMs]; const providerLease = await workerManager.call(plugin.id, "environmentAcquireLease", { driverKey: parsed.config.driverKey, companyId: input.companyId, @@ -3477,7 +3492,7 @@ function createPluginEnvironmentDriver( ...(requestedExpiresAtParam(input.requestedExpiresAt) !== undefined ? { requestedExpiresAt: requestedExpiresAtParam(input.requestedExpiresAt) } : {}), - } as PluginEnvironmentAcquireLeaseParams); + } as PluginEnvironmentAcquireLeaseParams, ...timeoutOverride); return await environmentsSvc.acquireLease({ companyId: input.companyId,