diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 46492a0df5f..5593fd17f38 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1563,6 +1563,13 @@ configuration, and controller-owned credential or instruction resources use verified suspension and saved-history resume. This recovery can restart a provider; it does not suspend every healthy warm turn. +Plugin sandbox reuse during acquisition uses the configured startup timeout or +the provider's declared acquisition default, with the host RPC margin. Resume +attempts and transient retries share that deadline. Providers without either +timeout retain the one-minute host default. A resume failure still preserves +the recorded sandbox unless the provider proves it lost and replacement passes +the existing backup checks. + For the native runner, warm mode requests a reusable sandbox lease **before** lease acquisition. The environment's explicit runner lifecycle overrides the agent default; an inherited lifecycle uses the current agent setting on each diff --git a/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md b/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md index c3988f98f11..5c8eb927d2a 100644 --- a/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md +++ b/doc/plugins/SANDBOX_PROVIDER_CAPABILITIES.md @@ -28,10 +28,16 @@ Drivers that omit the declaration keep the existing fallback. The host does not infer this budget from config-schema defaults: a field named `timeoutMs` can describe sandbox lifetime instead of the time needed to acquire it. -This declaration does not change provider config, lease expiry, the resume -deadline, or other lifecycle calls. Providers must still enforce their operation -timeouts and report uncertain allocations for cleanup. A bundled plugin must bump -its manifest version when adding the field so existing installations receive it. +For reusable sandbox acquisition, the host also uses this budget for +`environmentResumeLease`, with the same explicit-config precedence, bridge +extension, and RPC margin. Resume attempts and transient retries share one +deadline. Without a configured or declared budget, resume keeps its one-minute +host default. Generic plugin-driver resume keeps its existing behavior. + +This declaration does not change provider config, lease expiry, or other +lifecycle calls. Providers must still enforce their operation timeouts and +report uncertain allocations for cleanup. A bundled plugin must bump its +manifest version when adding the field so existing installations receive it. ## How the host resolves an effective capability diff --git a/maps/feature-map/execution-environments.md b/maps/feature-map/execution-environments.md index 00d59b7dcb8..6beb90d4a26 100644 --- a/maps/feature-map/execution-environments.md +++ b/maps/feature-map/execution-environments.md @@ -77,6 +77,12 @@ execution. Remote-driver probes and a real bounded run need their own evidence. Automated: [execution target](../../server/src/__tests__/environment-execution-target.test.ts) and [custom images](../../server/src/__tests__/environment-custom-images-service.test.ts) cover contracts; [SSH live test](../../server/src/__tests__/environment-live-ssh.test.ts) requires its own external setup. +The [environment runtime regression](../../server/src/__tests__/environment-runtime.test.ts) +also covers sandbox reuse after a start longer than one minute. Acquisition +uses the configured or provider-declared startup budget for resume and shares +one deadline across retries. Failure preserves the recorded sandbox. This +controlled timing check does not establish live provider acceptance. + Manual: Run a bounded command in the intended target, inspect its output and workspace, and verify execution did not silently fall back to the server host. For custom-image changes, build/probe the disposable image before assigning it. Inspect cleanup and shutdown after the run. ## Gotchas diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 1385e9167e6..d85b6d8a19e 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -567,13 +567,16 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { }; } - async function seedReusablePluginSandboxLease(adapterType: string | null = null) { + async function seedReusablePluginSandboxLease( + adapterType: string | null = null, + options: { timeoutMs?: number | null; defaultAcquireTimeoutMs?: number } = {}, + ) { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); const providerConfig = { provider: "fake-plugin", image: "fake:test", - timeoutMs: 1234, + ...(options.timeoutMs === null ? {} : { timeoutMs: options.timeoutMs ?? 1234 }), reuseLease: true, }; const environment = { @@ -598,6 +601,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { kind: "sandbox_provider", displayName: "Fake Plugin", supportsReusableLeases: true, + defaultAcquireTimeoutMs: options.defaultAcquireTimeoutMs, configSchema: { type: "object", properties: { @@ -645,10 +649,7 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { pluginId, pluginKey: "acme.reusable-sandbox-provider", sandboxProviderPlugin: true, - provider: "fake-plugin", - image: "fake:test", - timeoutMs: 1234, - reuseLease: true, + ...providerConfig, reusableSandboxLease: { version: 1, companyId, @@ -4618,6 +4619,54 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { ); }); + it.each([ + { label: "declared provider startup budget", timeoutMs: null, defaultAcquireTimeoutMs: 300_000, expectedRpcTimeoutMs: 330_000 }, + { label: "explicit startup budget", timeoutMs: 120_000, defaultAcquireTimeoutMs: 300_000, expectedRpcTimeoutMs: 150_000 }, + ])("resumes the same sandbox after a slow start within its $label", async (options) => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner", options); + let signalResumeStarted!: () => void; + const resumeStarted = new Promise(resolve => { signalResumeStarted = resolve; }); + const call = vi.fn(async (_pluginId: string, method: string, _params: unknown, rpcTimeoutMs: number) => { + if (method !== "environmentResumeLease") throw new Error(`Unexpected plugin method: ${method}`); + vi.useFakeTimers({ toFake: ["Date", "setTimeout", "clearTimeout"] }); + const response = new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + reject(new Error("Plugin RPC timed out")); + }, rpcTimeoutMs); + setTimeout(() => { + clearTimeout(timeout); + vi.useRealTimers(); + resolve({ providerLeaseId: seeded.reusableLease.providerLeaseId, metadata: { remoteCwd: "/workspace" } }); + }, 61_000); + }); + signalResumeStarted(); + return await response; + }); + const runtime = environmentRuntimeService(db, { pluginWorkerManager: { + isRunning: () => true, call, + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + } as unknown as PluginWorkerManager }); + const now = vi.spyOn(Date, "now").mockReturnValue(Date.now()); + const acquisition = runtime.acquireRunLease({ + companyId: seeded.companyId, environment: seeded.environment, + issueId: null, agentId: seeded.agentId, adapterType: "paperclip_runner", heartbeatRunId: seeded.runId, + persistedExecutionWorkspace: { id: seeded.executionWorkspaceId, mode: "shared_workspace" }, + }).then(value => ({ value, error: null }), error => ({ value: null, error })); + try { + await resumeStarted; + await vi.advanceTimersByTimeAsync(61_000); + } finally { + vi.useRealTimers(); + now.mockRestore(); + } + const result = await acquisition; + expect(result.error).toBeNull(); + expect(result.value?.lease).toMatchObject({ id: seeded.reusableLease.id, providerLeaseId: seeded.reusableLease.providerLeaseId }); + expect(call).toHaveBeenCalledOnce(); + expect(call.mock.calls[0]?.[3]).toBe(options.expectedRpcTimeoutMs); + expect(await environmentService(db).getLeaseById(seeded.reusableLease.id)).toMatchObject({ status: "active", heartbeatRunId: seeded.runId }); + }); + it("preserves the exact lease when plugin-backed sandbox resume throws", async () => { const pluginId = randomUUID(); const { companyId, agentId, environment: baseEnvironment, runId } = await seedEnvironment(); diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index a7c2f0098af..e737779a283 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -2018,9 +2018,12 @@ function createSandboxEnvironmentDriver( }); } try { - const resumeDeadline = Date.now() + 60_000; const configuredResumeTimeoutMs = - resolvePluginSandboxRpcTimeoutMs(workerConfig) ?? 60_000; + resolvePluginSandboxRpcTimeoutMs( + workerConfig, + pluginProvider.resolved.driver.defaultAcquireTimeoutMs, + ) ?? 60_000; + const resumeDeadline = Date.now() + configuredResumeTimeoutMs; let retryDelayMs = 250; let resumed: PluginEnvironmentLease; while (true) {