diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 03909f22e00..c5801b8d8ec 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1639,6 +1639,14 @@ the configured interval until the provider confirms the new one. An explicit `0` (auto-stop disabled), a longer configured interval, per-turn, and non-reusable sandboxes are unchanged by the warm floor, and idle sandboxes still auto-stop. +The host's own warm-window stop of a retained sandbox waits 2 minutes past the +warm window, so the host's idle close of the warm runner (suspend and its proof +read) finishes before the sandbox stops. When an idle close still fails and every +lease row for that sandbox is no longer active or retained, the host has given the +sandbox up: the warm owner is released and the next run on the task prepares its +instructions fresh. A follow-up whose predecessor's warm instruction copy cannot be +handed off retires that owner and prepares fresh instead of failing. + A remote runner also keeps working while its controller is offline, and the host reattaches it within the remote runner reconnect grace (7 minutes). Daytona counts only SDK and lifecycle calls as activity, so with no controller its idle auto-stop diff --git a/server/src/__tests__/warm-sandbox-expiry.test.ts b/server/src/__tests__/warm-sandbox-expiry.test.ts index 0100bc54c37..0097287b743 100644 --- a/server/src/__tests__/warm-sandbox-expiry.test.ts +++ b/server/src/__tests__/warm-sandbox-expiry.test.ts @@ -8,6 +8,7 @@ import { heartbeatService, type HeartbeatEnvironmentRuntime } from "../services/ import { nextWarmSandboxStopAt, stopWarmSandboxesPastWindow, + WARM_SANDBOX_IDLE_CLOSE_MARGIN_MS, WARM_SANDBOX_IDLE_TIMEOUT_KEY, warmSandboxWindowMetadata, } from "../services/warm-sandbox-expiry.js"; @@ -81,14 +82,14 @@ suite("warm sandbox expiry", () => { expect(await stopWarmSandboxesPastWindow(db)).toEqual({ stopping: 0 }); expect(await row(leaseId)).toEqual(before); - const windowEnd = before.lastUsedAt.getTime() + 5 * MINUTE; + const windowEnd = before.lastUsedAt.getTime() + 5 * MINUTE + WARM_SANDBOX_IDLE_CLOSE_MARGIN_MS; expect(await nextWarmSandboxStopAt(db)).toBe(windowEnd); // The cleanup worker sleeps until exactly that deadline: no interval scan. expect(await nextEnvironmentCleanupAt(db)).toBe(windowEnd); }); it("stops a sandbox once its warm window has passed and keeps it for the next turn", async () => { - const leaseId = await warmRetained({ releasedAgoMs: 6 * MINUTE, windowMs: 5 * MINUTE }); + const leaseId = await warmRetained({ releasedAgoMs: 8 * MINUTE, windowMs: 5 * MINUTE }); expect(await stopWarmSandboxesPastWindow(db)).toEqual({ stopping: 1 }); const stopping = await row(leaseId); @@ -99,8 +100,21 @@ suite("warm sandbox expiry", () => { expect(await nextWarmSandboxStopAt(db)).toBeNull(); }); + it("leaves the host's idle close of the warm runner time to finish before stopping its sandbox", async () => { + const closing = await warmRetained({ releasedAgoMs: 5 * MINUTE + 3_000, windowMs: 5 * MINUTE }); + const before = await row(closing); + + expect(await stopWarmSandboxesPastWindow(db)).toEqual({ stopping: 0 }); + expect(await row(closing)).toEqual(before); + expect(await nextWarmSandboxStopAt(db)).toBeGreaterThanOrEqual(before.lastUsedAt.getTime() + 5 * MINUTE + 90_000); + + await db.update(environmentLeases).set({ lastUsedAt: new Date(Date.now() - 5 * MINUTE - WARM_SANDBOX_IDLE_CLOSE_MARGIN_MS - 1_000) }) + .where(eq(environmentLeases.id, closing)); + expect(await stopWarmSandboxesPastWindow(db)).toEqual({ stopping: 1 }); + }); + it("stops the sandbox at the provider through the cleanup sweep", async () => { - const leaseId = await warmRetained({ releasedAgoMs: 6 * MINUTE, windowMs: 5 * MINUTE }); + const leaseId = await warmRetained({ releasedAgoMs: 8 * MINUTE, windowMs: 5 * MINUTE }); const teardown = vi.fn(async ({ lease }: { lease: { providerLeaseId: string | null } }) => ({ providerLeaseId: lease.providerLeaseId, state: "stopped" })); @@ -118,7 +132,7 @@ suite("warm sandbox expiry", () => { it("keeps a sandbox past its window while a live run holds it", async () => { const providerLeaseId = `sandbox://fake/${randomUUID()}`; - const leaseId = await warmRetained({ releasedAgoMs: 6 * MINUTE, providerLeaseId }); + const leaseId = await warmRetained({ releasedAgoMs: 8 * MINUTE, providerLeaseId }); const liveRun = await run("running"); await db.insert(environmentLeases).values({ companyId, environmentId, issueId, heartbeatRunId: liveRun, status: "active", leasePolicy: "reuse_by_environment", provider: "fake", providerLeaseId, @@ -132,7 +146,7 @@ suite("warm sandbox expiry", () => { }); it("keeps a sandbox past its window while a run on the task is starting", async () => { - const leaseId = await warmRetained({ releasedAgoMs: 6 * MINUTE }); + const leaseId = await warmRetained({ releasedAgoMs: 8 * MINUTE }); await run("running"); const before = await row(leaseId); @@ -141,7 +155,7 @@ suite("warm sandbox expiry", () => { }); it("keeps a sandbox whose own run has not ended", async () => { - const leaseId = await warmRetained({ releasedAgoMs: 6 * MINUTE, runStatus: "running" }); + const leaseId = await warmRetained({ releasedAgoMs: 8 * MINUTE, runStatus: "running" }); expect(await stopWarmSandboxesPastWindow(db)).toEqual({ stopping: 0 }); expect((await row(leaseId)).status).toBe("retained"); @@ -150,7 +164,7 @@ suite("warm sandbox expiry", () => { it("leaves a plugin Runner task lease to its own cleanup", async () => { // Its release path does not use the stop-only contract, so expiry must not // hand it over: the plugin could free the allocation and the task's files. - const leaseId = await warmRetained({ releasedAgoMs: 6 * MINUTE }); + const leaseId = await warmRetained({ releasedAgoMs: 8 * MINUTE }); await db.update(environmentLeases).set({ metadata: { driver: "plugin", pluginId: "fake-plugin", runnerTaskLease: true, [WARM_SANDBOX_IDLE_TIMEOUT_KEY]: 5 * MINUTE } }).where(eq(environmentLeases.id, leaseId)); const before = await row(leaseId); @@ -178,7 +192,7 @@ suite("warm sandbox expiry", () => { expect(await row(leaked)).toMatchObject({ status: "released", cleanupStatus: "success" }); expect((await row(recent)).status).toBe("retained"); const recentRow = await row(recent); - expect(await nextWarmSandboxStopAt(db)).toBe(recentRow.lastUsedAt.getTime() + 24 * HOUR); + expect(await nextWarmSandboxStopAt(db)).toBe(recentRow.lastUsedAt.getTime() + 24 * HOUR + WARM_SANDBOX_IDLE_CLOSE_MARGIN_MS); }); it("records the run's warm window on the lease only for a warm run", () => { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 7ddf5de47d2..c08975a60e1 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -11503,6 +11503,51 @@ describe("native warm session supervision", () => { } }); + it.each([ + { lease: "pending_cleanup", forgotten: true }, + { lease: "released", forgotten: true }, + { lease: "retained", forgotten: false }, + ])("forgets a warm owner whose idle close failed only once the host gave up its sandbox: $lease", async ({ lease, forgotten }) => { + const identity = `warm-idle-close-sandbox-${lease}`; + const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, + session: { ...execution.session, normalizedSessionId: identity, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 50 } }, + } as NativeExecutionInputV1; + let allowClose = false; + const session = { close: vi.fn(async () => { if (!allowClose) throw new Error("runner_harness_state_mismatch: provider_state_unreadable"); }) }; + state.execute.mockReset().mockImplementation(async (options) => { + await options.onSession?.(session); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: identity, normalizedSessionId: identity, providerSessionId: identity, + driverKind: "test", driverVersion: "1", nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + const copy = (runId: string) => ({ runId, root: `/tmp/${identity}`, preparationKey: "same", hasChanges: async () => false, + collectStopped: async () => undefined, retirementFailed: async () => undefined }); + let leaseRows: Array<{ status: string }> = []; + const base = leaseDb(warmExecution); + const db = { ...base, select: (fields?: unknown) => ({ from: (table: unknown) => table === environmentLeases + ? { where: async () => leaseRows } : base.select(fields as never).from(table as never) }) } as unknown as Db; + try { + await executePaperclipNativeSession({ db, execution: warmExecution, runnerInstanceId: identity, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", environmentId: identity, remoteCwd: `/tmp/${identity}`, + sandboxLeaseAcquisition: { providerLeaseId: `${identity}-sandbox` } } as never, + instructionWorkingCopy: copy(identity) }); + leaseRows = [{ status: lease }]; + await vi.waitFor(() => expect(session.close).toHaveBeenCalledTimes(1)); + const followUpRunId = `${identity}-follow-up`; + const adopt = vi.fn(async () => null); + allowClose = true; + await expect(claimWarmNativeInstructionCopy({ priorExecution: warmExecution, + companyId: warmExecution.binding.companyId, agentId: warmExecution.binding.agentId, + executionWorkspaceId: warmExecution.binding.executionWorkspaceId, workspace: warmExecution.workspace, + environmentId: identity, runId: followUpRunId, preparationKey: "same", adopt })).resolves.toBeNull(); + expect(adopt).toHaveBeenCalledTimes(forgotten ? 0 : 1); + expect(session.close).toHaveBeenCalledTimes(forgotten ? 1 : 2); + } finally { + allowClose = true; + await closeWarmNativeSessionsForEnvironment({ environmentId: identity, reason: "test cleanup" }); + } + }); + it("admits the follow-up's execution only after the previous owner's final cleanup hands its scopes back", async () => { const identity = "warm-release-scope-handback"; const warmExecution = { ...execution, binding: { ...execution.binding, runId: identity, executionWorkspaceId: `${identity}-workspace` }, @@ -11895,8 +11940,8 @@ describe("native warm session supervision", () => { const claiming = claimWarmNativeInstructionCopy({ priorExecution: first, companyId: first.binding.companyId, agentId: first.binding.agentId, executionWorkspaceId: first.binding.executionWorkspaceId, workspace: first.workspace, environmentId: identity, runId: current.runId, preparationKey: current.preparationKey, forceRetirement: scenario === "reset", adopt }); - if (["unavailable", "auth", "wrong-capability", "retirement-failure"].includes(scenario)) { - await expect(claiming).rejects.toThrow(({ unavailable: "handoff_unavailable", auth: "authorization denied", + if (["auth", "wrong-capability", "retirement-failure"].includes(scenario)) { + await expect(claiming).rejects.toThrow(({ auth: "authorization denied", "wrong-capability": "handoff_mismatch", "retirement-failure": "stop unconfirmed" } as Record)[scenario]); } else expect(await claiming).toBeNull(); expect(priorCopy.hasChanges).not.toHaveBeenCalled(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 279c8c81ed0..fe7843e2a5d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -716,6 +716,7 @@ type WarmNativeSession = { pluginTaskId?: string; /** The sandbox that holds this owner's Runner, when it runs in one. */ providerLeaseId?: string; + sandboxLeaseDb?: Db; /** A successor's retirement of this owner failed; it stays registered for its own sandbox. */ retirementUnproven?: boolean; observabilityCompatibilityKey?: string; @@ -1002,7 +1003,7 @@ export async function claimWarmNativeInstructionCopy(input: { const adopted = await input.adopt(entry.instructionWorkingCopy.runId); if (!adopted) { await retire("warm instruction materialization cannot be handed off"); - throw new Error("native_instruction_materialization_handoff_unavailable"); + return null; } if (adopted.runId !== input.runId || adopted.preparationKey !== input.preparationKey) throw new Error("native_instruction_materialization_handoff_mismatch"); entry.instructionWorkingCopy = adopted; @@ -1088,6 +1089,16 @@ export async function closeWarmNativeSessionsForEnvironment(input: { return closeIdleWarmNativeSessions(input); } +async function warmSandboxGivenUpByHost(entry: WarmNativeSession): Promise { + if (!entry.sandboxLeaseDb || !entry.providerLeaseId || !entry.environmentId) return false; + const leases = await entry.sandboxLeaseDb.select({ status: environmentLeases.status }).from(environmentLeases).where(and( + eq(environmentLeases.companyId, entry.companyId), + eq(environmentLeases.environmentId, entry.environmentId), + eq(environmentLeases.providerLeaseId, entry.providerLeaseId), + )); + return leases.length > 0 && leases.every(lease => lease.status !== "active" && lease.status !== "retained"); +} + async function warmSandboxDestroyed(db: Db, entry: WarmNativeSession): Promise { if (!entry.providerLeaseId || !entry.environmentId) return false; const leases = await db.select().from(environmentLeases).where(and( @@ -8777,6 +8788,16 @@ function armWarmNativeSessionIdleTimer( logger.warn({ runId: current.runId, issueId: current.issueId, environmentId: current.environmentId, providerLeaseId: current.providerLeaseId ?? null, closeFailure: describeWarmCloseFailure(error) }, released ? "idle warm native session detach failed" : "idle warm native session close failed"); + if (released) return; + return warmSandboxGivenUpByHost(current).then(givenUp => { + if (!givenUp) return; + logger.warn({ runId: current.runId, issueId: current.issueId, environmentId: current.environmentId, + providerLeaseId: current.providerLeaseId ?? null }, + "warm native session's sandbox was stopped by the host; released its owner"); + forgetWarmNativeSession(sessionId, current); + }, proofError => { + logger.warn({ err: proofError, runId: current.runId }, "could not read the warm session's sandbox lease"); + }); }) .finally(() => { current.busy = false; @@ -12602,6 +12623,7 @@ async function executePaperclipNativeSessionAttempt( pluginTaskId: usesPluginTask && input.runnerExecutionTarget?.kind === "remote" && input.runnerExecutionTarget.transport === "sandbox" ? input.runnerExecutionTarget.sandboxLeaseAcquisition?.providerLeaseId : undefined, providerLeaseId: sandboxProviderLeaseId(input.runnerExecutionTarget) ?? undefined, + sandboxLeaseDb: input.db, runId: input.execution.binding.runId, issueId: input.execution.binding.issueId, instructionWorkingCopy: input.instructionWorkingCopy?.checkpointWarm ? undefined : input.instructionWorkingCopy, diff --git a/server/src/services/warm-sandbox-expiry.ts b/server/src/services/warm-sandbox-expiry.ts index 2f4ab6171d5..dcbfb598c18 100644 --- a/server/src/services/warm-sandbox-expiry.ts +++ b/server/src/services/warm-sandbox-expiry.ts @@ -40,6 +40,8 @@ export const WARM_SANDBOX_IDLE_TIMEOUT_KEY = "warmSandboxIdleTimeoutMs"; /** Marker on the stop-and-retain intent: the warm window ended. */ export const WARM_WINDOW_EXPIRED_KEY = "warmWindowExpired"; +export const WARM_SANDBOX_IDLE_CLOSE_MARGIN_MS = 2 * 60_000; + const STOP_PAGE_SIZE = 25; function positiveWindowSql(value: ReturnType) { @@ -52,7 +54,7 @@ export function warmWindowEndsAtSql() { const window = sql`least(coalesce( ${positiveWindowSql(sql`${environmentLeases.metadata}->${WARM_SANDBOX_IDLE_TIMEOUT_KEY}`)}, ${PAPERCLIP_RUNNER_IDLE_TIMEOUT_MAX_MS}), ${PAPERCLIP_RUNNER_IDLE_TIMEOUT_MAX_MS})`; - return sql`(extract(epoch from ${environmentLeases.lastUsedAt}) * 1000 + ${window})`; + return sql`(extract(epoch from ${environmentLeases.lastUsedAt}) * 1000 + ${window} + ${WARM_SANDBOX_IDLE_CLOSE_MARGIN_MS})`; } /**