diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index cdaefa0e505..105d0abc607 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1288,7 +1288,20 @@ not part of lease identity or metadata. Daytona derives its effective on resume (a bounded, best-effort update). Its cached-handle freshness keeps using the configured interval until the provider confirms the new one. An explicit `0` (auto-stop disabled), a longer configured interval, per-turn, and non-reusable -sandboxes are unchanged, and idle sandboxes still auto-stop. +sandboxes are unchanged by the warm floor, and idle sandboxes still auto-stop. + +A remote runner also keeps working while its controller is offline, and the host +reattaches it within the remote runner reconnect grace (7 minutes). Daytona counts +only SDK and lifecycle calls as activity, so with no controller its idle auto-stop +runs during the outage; a shorter interval stops the sandbox and kills the runner +while the host would still reattach it. For every `paperclip_runner` run on a +sandbox environment the host therefore also forwards `runnerReconnectGraceMs`, +on acquisition and on recovery resume, under the same provider-only rules. Daytona +keeps the effective `autoStopInterval` at least the grace plus 5 minutes (12 +minutes today), applies it at creation, and raises an older sandbox's shorter +interval on recovery resume. An explicit `0` and a longer configured interval are +unchanged, released sandboxes are still stopped or destroyed by cleanup, and +leaked sandboxes still auto-stop after the raised interval. When an admitted continuation reuses a sandbox after verified cleanup, an unpublished preparation uses a directory bound to its writer epoch and lease. diff --git a/packages/plugins/sandbox-providers/daytona/src/manifest.ts b/packages/plugins/sandbox-providers/daytona/src/manifest.ts index 4f830fe1b3b..2494e18866e 100644 --- a/packages/plugins/sandbox-providers/daytona/src/manifest.ts +++ b/packages/plugins/sandbox-providers/daytona/src/manifest.ts @@ -160,7 +160,7 @@ const manifest: PaperclipPluginManifestV1 = { autoStopInterval: { type: "number", description: - "Daytona auto-stop interval in minutes. `0` disables auto-stop. Defaults to 15 when unset. A reusable sandbox that keeps a warm runner never auto-stops before the warm idle window plus 5 minutes.", + "Daytona auto-stop interval in minutes. `0` disables auto-stop. Defaults to 15 when unset. A reusable sandbox that keeps a warm runner never auto-stops before the warm idle window plus 5 minutes, and a sandbox running a Paperclip runner never auto-stops before the host's runner reconnect grace plus 5 minutes.", default: 15, }, autoArchiveInterval: { diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts index e3d412acfe1..bfa76665615 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.test.ts @@ -1045,6 +1045,57 @@ describe("Daytona sandbox provider plugin", () => { } }); + // RC-54: a per-turn, non-reusable sandbox running a remote runner must not + // auto-stop during a controller outage the host still treats as + // reattachable. The host forwards its reconnect grace for runner leases. + it.each([ + [{ reuseLease: false, runnerLifecycleMode: "per_turn", autoStopInterval: 5 }], + [{ reuseLease: false, autoStopInterval: 2 }], + [{ reuseLease: true, runnerLifecycleMode: "per_turn", autoStopInterval: 5 }], + [{ reuseLease: false }], + ])("keeps a runner sandbox alive through the host reconnect grace plus a margin (%o)", async (config) => { + const autoStopMs = (await acquire({ ...config, runnerReconnectGraceMs: 420_000 })) * 60_000; + expect(autoStopMs).toBeGreaterThanOrEqual(420_000 + marginMs); + }); + + it("combines the reconnect grace floor with the warm floor and explicit settings", async () => { + expect(await acquire({ reuseLease: true, warmRunnerIdleTimeoutMs: 900_000, runnerReconnectGraceMs: 420_000 })) + .toBe(Math.ceil(900_000 / 60_000) + WARM_IDLE_AUTO_STOP_MARGIN_MINUTES); + expect(await acquire({ reuseLease: false, runnerReconnectGraceMs: 420_000, autoStopInterval: 60 })).toBe(60); + expect(await acquire({ reuseLease: false, runnerReconnectGraceMs: 420_000, autoStopInterval: 0 })).toBe(0); + // Without a host-forwarded grace (not a runner lease), the configured value stands. + expect(await acquire({ reuseLease: false, autoStopInterval: 5 })).toBe(5); + }); + + it("raises the auto-stop of an existing runner sandbox on recovery resume", async () => { + // The stopped sandbox from the outage restarts on resume and keeps its + // original 5-minute auto-stop unless the provider raises it. + process.env.DAYTONA_API_KEY = "host-key"; + const sandbox = createMockSandbox({ id: "sandbox-runner", state: "stopped" }); + (sandbox as { autoStopInterval: number }).autoStopInterval = 5; + sandbox.process.executeCommand.mockImplementation(async (command: string) => ({ exitCode: 0, + result: command.startsWith("cat ") ? JSON.stringify({ token: "sentinel-token" }) : "bash" })); + mockGet.mockResolvedValue(sandbox); + await plugin.definition.onEnvironmentResumeLease!({ + driverKey: "daytona", companyId: "company-1", environmentId: "env-1", providerLeaseId: sandbox.id, + config: { reuseLease: false, runnerLifecycleMode: "per_turn", autoStopInterval: 5, runnerReconnectGraceMs: 420_000 }, + leaseMetadata: { workspaceSentinel: { + path: "/home/daytona/paperclip-workspace/.paperclip-runtime/reusable-sandbox-lease.json", + token: "sentinel-token", result: "written" } }, + }); + expect(sandbox.setAutostopInterval).toHaveBeenCalledTimes(1); + const [minutes] = sandbox.setAutostopInterval.mock.calls[0] as [number]; + expect(minutes * 60_000).toBeGreaterThanOrEqual(420_000 + marginMs); + }); + + it("keeps the host reconnect grace out of the saved provider config", async () => { + const result = await plugin.definition.onEnvironmentValidateConfig!({ driverKey: "daytona", + config: { apiKey: "key", image: "node:20", runnerReconnectGraceMs: 420_000, warmRunnerIdleTimeoutMs: 900_000, reuseLease: true } }); + expect(result.ok).toBe(true); + expect(result.normalizedConfig).not.toHaveProperty("runnerReconnectGraceMs"); + expect(result.normalizedConfig).not.toHaveProperty("warmIdleTimeoutMs"); + }); + it("does not change auto-stop on resume of a per-turn reusable sandbox", async () => { const sandbox = await resume({ reuseLease: true, runnerLifecycleMode: "per_turn" }); expect(sandbox.setAutostopInterval).not.toHaveBeenCalled(); diff --git a/packages/plugins/sandbox-providers/daytona/src/plugin.ts b/packages/plugins/sandbox-providers/daytona/src/plugin.ts index 931dff2d7aa..efd2f96292a 100644 --- a/packages/plugins/sandbox-providers/daytona/src/plugin.ts +++ b/packages/plugins/sandbox-providers/daytona/src/plugin.ts @@ -200,6 +200,13 @@ interface DaytonaDriverConfig { * normalized provider config; it only raises the effective auto-stop. */ warmIdleTimeoutMs: number | null; + /** + * Host-owned remote runner reconnect grace (ms): how long the host still + * reattaches a live remote runner after a controller outage. Null when the + * lease hosts no remote runner. Never persisted in the normalized provider + * config; it only raises the effective auto-stop. + */ + runnerReconnectGraceMs: number | null; } type WorkspaceSentinelResult = { @@ -256,17 +263,37 @@ const DEFAULT_AUTO_STOP_INTERVAL_MINUTES = 15; export const DEFAULT_WARM_RUNNER_IDLE_TIMEOUT_MS = 300_000; export const WARM_IDLE_AUTO_STOP_MARGIN_MINUTES = 5; +// A remote runner keeps working while its controller is offline, but Daytona +// counts only SDK and lifecycle events as activity. With no controller there are +// none, so an auto-stop shorter than the host's reconnect grace stops the +// sandbox (and kills the runner) while the host would still reattach it. The +// host forwards its grace for runner leases; the provider backstop then fires +// only after the grace plus this margin. Idle sandboxes still stop. + +/** + * Host-owned floors (ms) the provider auto-stop must outlive: the warm runner + * idle window and the remote runner reconnect grace. + */ +function hostAutoStopFloorsMs( + config: Pick, +): number[] { + return [config.warmIdleTimeoutMs, config.runnerReconnectGraceMs] + .filter((value): value is number => value != null && value > 0); +} + /** * Effective Daytona auto-stop (minutes) for a sandbox. An explicit 0 (disabled) - * is preserved. A warm reusable lease never auto-stops before the host's warm - * idle window plus {@link WARM_IDLE_AUTO_STOP_MARGIN_MINUTES}. + * is preserved. A sandbox never auto-stops before a host-owned window (the warm + * idle window of a warm reusable lease, or the remote runner reconnect grace) + * plus {@link WARM_IDLE_AUTO_STOP_MARGIN_MINUTES}. */ export function effectiveAutoStopIntervalMinutes( - config: Pick, + config: Pick, ): number | null { const configured = config.autoStopInterval; - if (configured === 0 || config.warmIdleTimeoutMs == null) return configured; - const floor = Math.ceil(config.warmIdleTimeoutMs / 60_000) + WARM_IDLE_AUTO_STOP_MARGIN_MINUTES; + const floorsMs = hostAutoStopFloorsMs(config); + if (configured === 0 || floorsMs.length === 0) return configured; + const floor = Math.ceil(Math.max(...floorsMs) / 60_000) + WARM_IDLE_AUTO_STOP_MARGIN_MINUTES; return Math.max(configured ?? DEFAULT_AUTO_STOP_INTERVAL_MINUTES, floor); } @@ -347,6 +374,11 @@ function parseOptionalNumber(value: unknown): number | null { return Number.isFinite(parsed) ? parsed : null; } +function parsePositiveInteger(value: unknown): number | null { + const parsed = parseOptionalInteger(value); + return parsed != null && parsed > 0 ? parsed : null; +} + function parseDriverConfig(raw: Record): DaytonaDriverConfig { const timeoutMs = Number(raw.timeoutMs ?? DEFAULT_DAYTONA_OPERATION_TIMEOUT_MS); const livenessTimeoutMs = Number(raw.livenessTimeoutMs ?? DEFAULT_LIVENESS_TIMEOUT_MS); @@ -370,6 +402,7 @@ function parseDriverConfig(raw: Record): DaytonaDriverConfig { reuseLease: raw.reuseLease === true, archiveOnRelease: raw.archiveOnRelease === true, warmIdleTimeoutMs: parseWarmIdleTimeoutMs(raw), + runnerReconnectGraceMs: parsePositiveInteger(raw.runnerReconnectGraceMs), }; } @@ -2744,8 +2777,10 @@ const plugin = definePlugin(trackDaytonaHooks({ return { ok: false, errors }; } - // The warm idle window is host-owned; keep it out of the saved provider config. - const { warmIdleTimeoutMs: _warmIdleTimeoutMs, ...normalizedConfig } = config; + // The warm idle window and reconnect grace are host-owned; keep them out of + // the saved provider config. + const { warmIdleTimeoutMs: _warmIdleTimeoutMs, runnerReconnectGraceMs: _runnerReconnectGraceMs, + ...normalizedConfig } = config; return { ok: true, normalizedConfig, @@ -3028,13 +3063,14 @@ const plugin = definePlugin(trackDaytonaHooks({ } const resumedFromState = sandbox.state ?? null; await ensureSandboxStarted(sandbox, toTimeoutSeconds(config.timeoutMs), scope); - // A sandbox created before the warm floor (or under a shorter warm - // window) keeps its old auto-stop. Raise it on resume so the provider - // backstop cannot race the host's warm idle window. Best effort: the - // host's bounded resume retry still covers a refused resume. + // A sandbox created before a host floor (or under a shorter warm window + // or reconnect grace) keeps its old auto-stop. Raise it on resume so the + // provider backstop cannot race the host's warm idle window or a later + // controller outage. Best effort: the host's bounded resume retry still + // covers a refused resume. const autoStopInterval = effectiveAutoStopIntervalMinutes(config); const currentAutoStop = sandbox.autoStopInterval; - if (autoStopInterval != null && autoStopInterval > 0 && config.warmIdleTimeoutMs != null + if (autoStopInterval != null && autoStopInterval > 0 && hostAutoStopFloorsMs(config).length > 0 && typeof currentAutoStop === "number" && currentAutoStop > 0 && currentAutoStop < autoStopInterval) { await withLivenessTimeout("sandbox.setAutostopInterval", Math.max(1, Math.min(config.livenessTimeoutMs > 0 ? config.livenessTimeoutMs : AUTO_STOP_UPDATE_TIMEOUT_MS, AUTO_STOP_UPDATE_TIMEOUT_MS)), diff --git a/server/src/__tests__/environment-run-orchestrator.test.ts b/server/src/__tests__/environment-run-orchestrator.test.ts index 08d4816572e..38a2e1f1383 100644 --- a/server/src/__tests__/environment-run-orchestrator.test.ts +++ b/server/src/__tests__/environment-run-orchestrator.test.ts @@ -69,6 +69,7 @@ import { environmentRunOrchestrator, EnvironmentRunError, } from "../services/environment-run-orchestrator.ts"; +import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../services/native-runtime/native-runner-recovery-contract.js"; import type { Environment, EnvironmentLease, ExecutionWorkspace } from "@paperclipai/shared"; import type { RealizedExecutionWorkspace } from "../services/workspace-runtime.ts"; import type { EnvironmentRuntimeService } from "../services/environment-runtime.ts"; @@ -868,6 +869,26 @@ describe("native runner lifecycle changes before lease acquisition", () => { }); expect(acquireRunLease.mock.calls[0][0].environment.config.reuseLease).toBe(expectedReuse); }); + + // RC-54: a per-turn remote runner sandbox must outlive a controller outage + // the host still recovers, so acquisition forwards the reconnect grace. + it.each([ + ["paperclip_runner", "sandbox", REMOTE_RUNNER_RECONNECT_GRACE_MS], + ["claude_local", "sandbox", undefined], + ["paperclip_runner", "local", undefined], + ] as const)("forwards the remote runner reconnect grace for %s on %s", async (adapterType, driver, expected) => { + const environment = { ...makeEnvironment(driver), config: driver === "sandbox" + ? { provider: "daytona", reuseLease: false, runnerLifecycleMode: "per_turn" } : {} }; + mockGetEnvironment.mockResolvedValue(environment); + const acquireRunLease = vi.fn(async (input) => ({ environment: input.environment, lease: makeLease(), leaseContext: { executionWorkspaceId: null, executionWorkspaceMode: null } })); + const orchestrator = environmentRunOrchestrator({} as never, { environmentRuntime: makeMockRuntime({ acquireRunLease }) }); + await orchestrator.acquireForRun({ + companyId: "company-1", selectedEnvironmentId: "env-1", localEnvironmentId: "local", adapterType, + adapterConfig: {}, issueId: "task-1", heartbeatRunId: "run-1", agentId: "agent-1", + persistedExecutionWorkspace: null, executionWorkspaceSettings: null, + }); + expect(acquireRunLease.mock.calls[0][0].runnerReconnectGraceMs).toBe(expected); + }); }); @@ -917,8 +938,11 @@ describe("live remote runner lease reattachment", () => { const result = await environmentRunOrchestrator({} as never, { environmentRuntime: runtime }).acquireForRun(input); expect(mockGetLease).toHaveBeenCalledWith("lease-1"); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); + // RC-54: the reattach resume carries the host reconnect grace, so the + // provider raises an older sandbox's shorter idle auto-stop. expect(runtime.resumeRunLease).toHaveBeenCalledWith({ environment: result.environment, lease: savedLease, requestedCwd: "/remote/workspace", + runnerReconnectGraceMs: REMOTE_RUNNER_RECONNECT_GRACE_MS, }); expect(result.lease).toEqual(savedLease); expect(result.leaseContext.executionWorkspaceId).toBe("ew-1"); @@ -933,7 +957,7 @@ describe("live remote runner lease reattachment", () => { }); expect(runtime.resumeRunLease).toHaveBeenCalledWith({ environment: result.environment, lease: savedLease, requestedCwd: "/remote/workspace", - warmRunnerIdleTimeoutMs: 1_800_000, + warmRunnerIdleTimeoutMs: 1_800_000, runnerReconnectGraceMs: REMOTE_RUNNER_RECONNECT_GRACE_MS, }); expect(runtime.acquireRunLease).not.toHaveBeenCalled(); }); diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index d92eb32aaa4..08441642ecc 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -416,6 +416,29 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(acquired.lease.metadata).not.toHaveProperty("warmRunnerIdleTimeoutMs"); }); + it("forwards the run's remote runner reconnect grace to the provider worker without changing lease identity", async () => { + const pluginId = randomUUID(); + const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); + const config = { provider: "daytona", image: "node:20", timeoutMs: 1234, autoStopInterval: 5 }; + await insertSandboxProviderPlugin(pluginId, { environmentDrivers: [{ driverKey: "daytona", kind: "sandbox_provider", displayName: "Daytona", configSchema: { type: "object" } }] }); + await environmentService(db).update(baseEnvironment.id, { driver: "sandbox", config }); + const environment: Environment = { ...baseEnvironment, driver: "sandbox", config }; + const worker = createFreshPluginLeaseWorker(pluginId); + vi.mocked(worker.call).mockImplementation(async (_id, method) => { + if (method !== "environmentAcquireLease") throw new Error(`Unexpected ${method}`); + return { providerLeaseId: "runner-lease", metadata: { provider: "daytona", image: "node:20", remoteCwd: "/workspace" } }; + }); + const service = environmentRuntimeService(db, { pluginWorkerManager: worker }); + const acquired = await service.acquireRunLease({ companyId, environment, configuredEnvironment: environment, + issueId: null, heartbeatRunId: runId, persistedExecutionWorkspace: null, + adapterType: "paperclip_runner", runnerReconnectGraceMs: 420_000 }); + expect(worker.call).toHaveBeenCalledWith(pluginId, "environmentAcquireLease", expect.objectContaining({ + config: expect.objectContaining({ autoStopInterval: 5, runnerReconnectGraceMs: 420_000 }), + }), expect.any(Number)); + expect(acquired.lease.metadata).not.toHaveProperty("runnerReconnectGraceMs"); + expect((await environmentService(db).getById(environment.id))?.config).not.toHaveProperty("runnerReconnectGraceMs"); + }); + it.each([["paperclip_runner", false, false], ["paperclip_runner", true, false], ["paperclip_runner", true, undefined], ["claude_local", false, false], [null, false, false]] as const)("pins qualified snapshots only for native execution (%s, warm=%s, savedReuse=%s)", async (adapterType, warm, savedReuse) => { const pluginId = randomUUID(); const { companyId, environment: baseEnvironment, runId } = await seedEnvironment(); diff --git a/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts b/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts index dcc3e62418a..88d1448a4f4 100644 --- a/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts +++ b/server/src/__tests__/heartbeat-native-remote-failure-retention.test.ts @@ -16,6 +16,7 @@ import { instanceSettingsService } from "../services/instance-settings.js"; import { buildNativeExecutionInput } from "../services/native-runtime/native-execution-input.js"; import { nativeRuntimeContextFixture } from "../services/native-runtime/runtime-context.test-fixture.js"; import { registerNativeRunnerCheckpoint } from "../services/native-runtime/native-runner-checkpoint.js"; +import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "../services/native-runtime/native-runner-recovery-contract.js"; import { describeEmbeddedPostgres, nativeRestartFixture } from "./helpers/native-restart.js"; describeEmbeddedPostgres("heartbeat native remote preparation failure retention", () => { @@ -169,6 +170,8 @@ describeEmbeddedPostgres("heartbeat native remote preparation failure retention" executionWorkspaceId: workspaceId, heartbeatRunId: f.runId, providerLeaseId, status: "active", leasePolicy: "ephemeral" }), requestedCwd: remoteCwd, + // The reattach resume tells the provider the reconnect grace (RC-54). + runnerReconnectGraceMs: REMOTE_RUNNER_RECONNECT_GRACE_MS, }); expect(acquireAllocation).not.toHaveBeenCalled(); expect(resumed).toHaveBeenCalledOnce(); diff --git a/server/src/services/environment-run-orchestrator.ts b/server/src/services/environment-run-orchestrator.ts index 0f03cc2f343..4e3f725cfa5 100644 --- a/server/src/services/environment-run-orchestrator.ts +++ b/server/src/services/environment-run-orchestrator.ts @@ -32,7 +32,11 @@ import type { ExecutionWorkspaceConfig, IssueExecutionWorkspaceSettings, } from "@paperclipai/shared"; -import { resolveRunnerEnvironmentForRun, resolveWarmRunnerIdleTimeoutMsForRun } from "./runner-environment-lifecycle.js"; +import { + resolveRunnerEnvironmentForRun, + resolveRunnerReconnectGraceMsForRun, + resolveWarmRunnerIdleTimeoutMsForRun, +} from "./runner-environment-lifecycle.js"; import { environmentService } from "./environments.js"; import { environmentRuntimeService, @@ -221,6 +225,7 @@ export function environmentRunOrchestrator( runnerTaskProjectIds?: readonly string[]; adapterType: string | null; warmRunnerIdleTimeoutMs?: number; + runnerReconnectGraceMs?: number; }): Promise { try { return await environmentRuntime.acquireRunLease(input); @@ -328,6 +333,9 @@ export function environmentRunOrchestrator( const warmRunnerIdleTimeoutMs = resolveWarmRunnerIdleTimeoutMsForRun( selectedEnvironment, input.adapterType, input.adapterConfig, input.admittedLifecycleMode, ); + const runnerReconnectGraceMs = input.adapterType + ? resolveRunnerReconnectGraceMsForRun(selectedEnvironment, input.adapterType) + : undefined; const acquisitionInput = { companyId: input.companyId, environment, @@ -340,6 +348,7 @@ export function environmentRunOrchestrator( runnerTaskProjectIds: input.runnerTaskProjectIds, adapterType: input.adapterType ?? null, warmRunnerIdleTimeoutMs, + runnerReconnectGraceMs, }; let leaseRecord: EnvironmentRuntimeLeaseRecord; if (input.reattachRemoteLease) { @@ -370,6 +379,7 @@ export function environmentRunOrchestrator( const resumed = await environmentRuntime.resumeRunLease({ environment, lease, requestedCwd: expected.remoteCwd, ...(warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs } : {}), + ...(runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs } : {}), }); if (!resumed || resumed.providerLeaseId !== expected.providerLeaseId || resumed.metadata?.expired === true) { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index 149923a2ca1..4fc672b52dc 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -500,6 +500,13 @@ export interface EnvironmentDriverAcquireInput { * its idle auto-stop strictly after the host's warm idle window. */ warmRunnerIdleTimeoutMs?: number; + /** + * The run's remote runner reconnect grace when the run places a remote runner + * on the sandbox. Forwarded to the provider worker only (never to lease + * identity or metadata), so a provider idle auto-stop cannot stop the sandbox + * during a controller outage the host still recovers. + */ + runnerReconnectGraceMs?: number; /** * Re-check the environment company binding inside the lease insert * transaction. The login acquire paths set this so a managed reconciliation @@ -557,6 +564,8 @@ export interface EnvironmentDriverResumeInput extends EnvironmentDriverLeaseInpu requestedCwd?: string; /** The run's warm runner idle window, forwarded to the provider worker only. */ warmRunnerIdleTimeoutMs?: number; + /** The run's remote runner reconnect grace, forwarded to the provider worker only. */ + runnerReconnectGraceMs?: number; } export interface EnvironmentDriverRealizeWorkspaceInput extends EnvironmentDriverLeaseInput { @@ -1661,6 +1670,8 @@ function createSandboxEnvironmentDriver( ...stripSandboxProviderEnvelope(parsed.config), // Provider-only view of the run's warm idle window (see the input doc). ...(input.warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs: input.warmRunnerIdleTimeoutMs } : {}), + // Provider-only view of the run's remote runner reconnect grace. + ...(input.runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs: input.runnerReconnectGraceMs } : {}), }; const storedConfig = pinnedSnapshotId ? { ...storedParsed.config, snapshot: pinnedSnapshotId } : storedParsed.config; if (pinnedSnapshotId) delete storedConfig.image; @@ -2597,6 +2608,7 @@ function createSandboxEnvironmentDriver( }) as SandboxEnvironmentConfig), // Provider-only view of the run's warm idle window (see the input doc). ...(input.warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs: input.warmRunnerIdleTimeoutMs } : {}), + ...(input.runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs: input.runnerReconnectGraceMs } : {}), }; const resumed = await pluginWorkerManager.call(pluginId, "environmentResumeLease", { driverKey: providerKey, companyId: input.lease.companyId, environmentId: input.environment.id, @@ -3837,6 +3849,8 @@ export function environmentRuntimeService( requestedExpiresAt?: Date | null; /** The run's warm runner idle window, forwarded to the provider worker only. */ warmRunnerIdleTimeoutMs?: number; + /** The run's remote runner reconnect grace, forwarded to the provider worker only. */ + runnerReconnectGraceMs?: number; /** * Re-check the environment company binding inside the lease insert * transaction. The login acquire paths set this to reject a foreign-company @@ -3869,6 +3883,7 @@ export function environmentRuntimeService( requestedExpiresAt: input.requestedExpiresAt ?? null, assertCompanyBinding: input.assertCompanyBinding, ...(input.warmRunnerIdleTimeoutMs !== undefined ? { warmRunnerIdleTimeoutMs: input.warmRunnerIdleTimeoutMs } : {}), + ...(input.runnerReconnectGraceMs !== undefined ? { runnerReconnectGraceMs: input.runnerReconnectGraceMs } : {}), }); return { diff --git a/server/src/services/runner-environment-lifecycle.ts b/server/src/services/runner-environment-lifecycle.ts index 61d1ad023d3..16b174d8afe 100644 --- a/server/src/services/runner-environment-lifecycle.ts +++ b/server/src/services/runner-environment-lifecycle.ts @@ -1,5 +1,6 @@ import { resolvePaperclipRunnerIdleTimeoutMs } from "@paperclipai/adapter-utils"; import type { Environment } from "@paperclipai/shared"; +import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runtime/native-runner-recovery-contract.js"; function resolveRunnerLifecycleModeForRun( config: Record, @@ -58,3 +59,19 @@ export function resolveWarmRunnerIdleTimeoutMsForRun( if (resolveRunnerLifecycleModeForRun(config, adapterConfig, admittedLifecycleMode) !== "warm") return undefined; return resolvePaperclipRunnerIdleTimeoutMs(config.runnerIdleTimeoutMs ?? adapterConfig.idleTimeoutMs); } + +/** + * The remote runner reconnect grace (ms) a sandbox-provider lease must outlive + * for this run, or undefined when the run places no remote runner on a sandbox. + * A remote runner keeps working while the controller is offline, and the host + * reattaches it within this grace. The host forwards it to the provider so a + * provider idle auto-stop (for example Daytona's `autoStopInterval`) cannot stop + * the sandbox, and kill the runner, during an outage the host still recovers. + */ +export function resolveRunnerReconnectGraceMsForRun( + environment: Pick, + adapterType: string, +): number | undefined { + if (adapterType !== "paperclip_runner" || environment.driver !== "sandbox") return undefined; + return REMOTE_RUNNER_RECONNECT_GRACE_MS; +}