diff --git a/doc/plugins/PLUGIN_SPEC.md b/doc/plugins/PLUGIN_SPEC.md index 715a0476d5a..e6d96c59d44 100644 --- a/doc/plugins/PLUGIN_SPEC.md +++ b/doc/plugins/PLUGIN_SPEC.md @@ -538,7 +538,9 @@ expected current status, so a disable or uninstall during a retry wins. Server shutdown cancels pending retries and waits for running ones. Runs that need the plugin see a worker that is not running, which setup retries within its bounded budget. After the last retry, the plugin becomes `error` with that same prefix. A run that -needs the plugin then re-activates it on demand, at most once a minute. Any +needs the plugin then re-activates it on demand, at most once a minute. A run that resumes its exact sandbox lease after a +restart waits up to 60 seconds for a re-activating sandbox provider's worker +before it reports the worker unavailable. Any other activation failure, such as a bad manifest, a worker that exits during `initialize`, or a version mismatch, makes the plugin `error` at once. The next server boot retries it once. diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 509646dc29b..57fae63cda3 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -4036,13 +4036,30 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { const call = vi.fn(); const requestActivationTimeoutRecovery = vi.fn(() => true); const workerManager = { isRunning: () => false, call, getWorker: () => undefined, requestActivationTimeoutRecovery } as unknown as PluginWorkerManager; - const error = await environmentRuntimeService(db, { pluginWorkerManager: workerManager }) + const error = await environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerResumeWaitMs: 50, pluginWorkerReadyPollMs: 5 }) .resumeRunLease({ environment: seeded.environment, lease }).catch(caught => caught); expect(error).toBeInstanceOf(PluginWorkerUnavailableError); expect(call).not.toHaveBeenCalled(); expect(requestActivationTimeoutRecovery).toHaveBeenCalledTimes(status === "error" ? 1 : 0); }); + it("resumes the exact lease once a re-activating plugin's worker comes back within the wait", async () => { + const seeded = await seedReusablePluginSandboxLease("paperclip_runner"); + await environmentService(db).releaseLease(seeded.reusableLease.id, "released", { cleanupStatus: "success" }); + const lease = (await environmentService(db).getLeaseById(seeded.reusableLease.id))!; + let checks = 0; + const call = vi.fn(async (_id: string, method: string) => { + if (method !== "environmentResumeLease") throw new Error("Only exact resume is permitted"); + return { providerLeaseId: lease.providerLeaseId, metadata: { remoteCwd: "/workspace" } }; + }); + const workerManager = { isRunning: () => ++checks > 3, call, + getWorker: () => ({ supportedMethods: ["environmentResumeLease", "environmentReleaseLease", "environmentDestroyLease"] }), + } as unknown as PluginWorkerManager; + await expect(environmentRuntimeService(db, { pluginWorkerManager: workerManager, pluginWorkerResumeWaitMs: 5_000, pluginWorkerReadyPollMs: 5 }) + .resumeRunLease({ environment: seeded.environment, lease })).resolves.toMatchObject({ providerLeaseId: lease.providerLeaseId }); + expect(call).toHaveBeenCalledOnce(); + }); + it.each([ { timeoutMs: 300_000, rpcTimeoutMs: 330_000 }, { timeoutMs: 1234, rpcTimeoutMs: 31_234 }, diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index a567bcffcc8..62294857264 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -782,6 +782,7 @@ export type ProviderResourceDisposition = | "destroy"; const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS = 5_000; +const DEFAULT_PLUGIN_SANDBOX_WORKER_RESUME_WAIT_MS = 60_000; const DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS = 100; const DEFAULT_DEFERRED_ORPHAN_CLEANUP_BUFFER_LIMIT = 256; @@ -1338,6 +1339,7 @@ function createSandboxEnvironmentDriver( readOnlyInspection?: boolean; pluginWorkerReadyTimeoutMs?: number; pluginWorkerReadyPollMs?: number; + pluginWorkerResumeWaitMs?: number; deferredOrphanCleanupBufferLimit?: number; orphanCleanupSpool?: SandboxOrphanCleanupSpool; orphanCleanupSpoolDir?: string; @@ -1346,6 +1348,7 @@ function createSandboxEnvironmentDriver( const pluginWorkerManager = options.pluginWorkerManager; const pluginWorkerReadyTimeoutMs = options.pluginWorkerReadyTimeoutMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_TIMEOUT_MS; const pluginWorkerReadyPollMs = options.pluginWorkerReadyPollMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_READY_POLL_MS; + const pluginWorkerResumeWaitMs = options.pluginWorkerResumeWaitMs ?? DEFAULT_PLUGIN_SANDBOX_WORKER_RESUME_WAIT_MS; // The buffer holds at least one orphan, so a zero or negative override never // drops every buffered record. const deferredOrphanCleanupBufferLimit = Math.max( @@ -2702,7 +2705,9 @@ function createSandboxEnvironmentDriver( && !pluginWorkerManager.isRunning(pluginId)) { const plugin = await resolvePluginSandboxProviderDriverById({ db, pluginId, driverKey: providerKey }); if (plugin && (plugin.plugin.status === "ready" || pluginWorkerManager.requestActivationTimeoutRecovery?.(plugin.plugin))) { - throw new PluginWorkerUnavailableError({ pluginId, workerStatus: pluginWorkerManager.getWorker(pluginId)?.status ?? null, + const deadline = Date.now() + Math.max(0, pluginWorkerResumeWaitMs); + while (!pluginWorkerManager.isRunning(pluginId) && Date.now() < deadline) await delay(Math.max(1, pluginWorkerReadyPollMs)); + if (!pluginWorkerManager.isRunning(pluginId)) throw new PluginWorkerUnavailableError({ pluginId, workerStatus: pluginWorkerManager.getWorker(pluginId)?.status ?? null, message: `Sandbox provider "${providerKey}" is installed via plugin "${plugin.plugin.pluginKey}", but its worker is not running.` }); } } @@ -3786,6 +3791,7 @@ export function environmentRuntimeService( readOnlyInspection?: boolean; pluginWorkerReadyTimeoutMs?: number; pluginWorkerReadyPollMs?: number; + pluginWorkerResumeWaitMs?: number; deferredOrphanCleanupBufferLimit?: number; orphanCleanupSpool?: SandboxOrphanCleanupSpool; orphanCleanupSpoolDir?: string; @@ -3803,6 +3809,7 @@ export function environmentRuntimeService( readOnlyInspection: options.readOnlyInspection, pluginWorkerReadyTimeoutMs: options.pluginWorkerReadyTimeoutMs, pluginWorkerReadyPollMs: options.pluginWorkerReadyPollMs, + pluginWorkerResumeWaitMs: options.pluginWorkerResumeWaitMs, deferredOrphanCleanupBufferLimit: options.deferredOrphanCleanupBufferLimit, orphanCleanupSpool: options.orphanCleanupSpool, orphanCleanupSpoolDir: options.orphanCleanupSpoolDir,