From 46cbfc67a3bd317ba2844a557b26c8b8fda080b9 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 11 Oct 2026 08:56:36 -0500 Subject: [PATCH] Checkpoint warm native sessions before stopping their computer owners Close the exact run for stop-and-retain and default stopping paths, retain failed explicit cleanup fences, and settle Stop requests arriving during or after warm retention. Prevent stale warm returns from overwriting a released or rebound lease. Co-authored-by: Codex Co-Authored-By: Paperclip --- .../src/__tests__/environment-runtime.test.ts | 28 ++++ server/src/services/environment-runtime.ts | 22 ++- .../services/heartbeat/run-control.test.ts | 37 +++++ server/src/services/heartbeat/run-control.ts | 12 +- .../native-session-executor.test.ts | 126 +++++++++++++++++- .../native-runtime/native-session-executor.ts | 45 +++++-- 6 files changed, 246 insertions(+), 24 deletions(-) diff --git a/server/src/__tests__/environment-runtime.test.ts b/server/src/__tests__/environment-runtime.test.ts index 9c456d8e83a..b8fa88b86ed 100644 --- a/server/src/__tests__/environment-runtime.test.ts +++ b/server/src/__tests__/environment-runtime.test.ts @@ -671,6 +671,34 @@ describeEmbeddedPostgres("environmentRuntimeService", () => { expect(workerManager.call).not.toHaveBeenCalled(); }); + it.each(["released", "rebound"])("does not publish stale warm retention after the lease was %s", async (change) => { + const { runId, reusableLease } = await seedReusablePluginSandboxLease(); + const realFactory = environmentsModule.environmentService; + const receipt = { proof: "exact-owner-stopped" }; + let changed = false; + const factory = vi.spyOn(environmentsModule, "environmentService").mockImplementation(database => { + const real = realFactory(database); + return { ...real, getById: async (id: string) => { + const result = await real.getById(id); + if (!changed) { + changed = true; + if (change === "released") await real.releaseLease(reusableLease.id, "released", { cleanupStatus: "success", remoteExecutionTermination: receipt }); + else await database.update(environmentLeases).set({ providerLeaseId: "new-owner", metadata: { computerOwner: { ownerId: "new-owner", generation: 2 } } }).where(eq(environmentLeases.id, reusableLease.id)); + } + return result; + } }; + }); + try { + const released = await environmentRuntimeService(db).releaseRunLeases(runId, "released", undefined, "keep_running"); + expect(released).toEqual([]); + const actual = await realFactory(db).getLeaseById(reusableLease.id); + if (change === "released") { + expect(actual).toMatchObject({ status: "released", metadata: { remoteExecutionTermination: receipt } }); + expect(actual?.releasedAt).not.toBeNull(); + } else expect(actual).toMatchObject({ status: "active", providerLeaseId: "new-owner", metadata: { computerOwner: { generation: 2 } } }); + } finally { factory.mockRestore(); } + }); + it("stops a reusable sandbox and keeps its lease resumable", async () => { const { pluginId, runId, reusableLease } = await seedReusablePluginSandboxLease(); const workerManager = { diff --git a/server/src/services/environment-runtime.ts b/server/src/services/environment-runtime.ts index b3a93ca67dc..ac917b8a4fc 100644 --- a/server/src/services/environment-runtime.ts +++ b/server/src/services/environment-runtime.ts @@ -4121,11 +4121,23 @@ export function environmentRuntimeService( providerResourceDisposition === "keep_running" && leaseSnapshot.leasePolicy === "reuse_by_environment" ) { - const lease = await environmentsSvc.releaseLease( - leaseRow.id, - "retained", - { cleanupStatus: "success" }, - ); + // A concurrent stop or owner replacement outranks this stale warm + // return. Never erase its termination receipt or restore retention. + const now = new Date(); + const [retained] = await db.update(environmentLeases).set({ + status: "retained", cleanupStatus: "success", releasedAt: null, + lastUsedAt: now, updatedAt: now, + metadata: sql`${environmentLeases.metadata} - 'remoteExecutionTermination'`, + }).where(and( + eq(environmentLeases.id, leaseRow.id), + eq(environmentLeases.companyId, leaseRow.companyId), + eq(environmentLeases.heartbeatRunId, heartbeatRunId), + eq(environmentLeases.status, "active"), + sql`${environmentLeases.environmentId} is not distinct from ${leaseRow.environmentId}`, + sql`${environmentLeases.providerLeaseId} is not distinct from ${leaseRow.providerLeaseId}`, + sql`${environmentLeases.metadata} is not distinct from ${leaseRow.metadata === null ? null : JSON.stringify(leaseRow.metadata)}::jsonb`, + )).returning(); + const lease = retained ? toEnvironmentLeaseSnapshot(retained) : null; if (lease) { released.push({ environment, diff --git a/server/src/services/heartbeat/run-control.test.ts b/server/src/services/heartbeat/run-control.test.ts index bac6f5a93f1..eb7a701dcf5 100644 --- a/server/src/services/heartbeat/run-control.test.ts +++ b/server/src/services/heartbeat/run-control.test.ts @@ -148,6 +148,43 @@ describe("heartbeat run-control module boundary", () => { expect(database.access).not.toHaveBeenCalled(); }); + it.each(["stop_and_retain", undefined] as const)("closes the exact warm run before stopping its lease: %s", async (disposition) => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run); + let finishClose!: () => void; + deps.options.closeWarmNativeSessionsForRun.mockImplementation(() => new Promise(resolve => { + finishClose = () => resolve({ closed: 1, busy: 0, failed: 0 }); + })); + const release = createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ + runId: "stopping", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: disposition, + }); + await vi.waitFor(() => expect(deps.options.closeWarmNativeSessionsForRun).toHaveBeenCalledWith(expect.objectContaining({ runId: "stopping" }))); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + finishClose(); + await release; + expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledOnce(); + }); + + it.each(["busy", "failed"] as const)("defers stop-and-retain while exact warm cleanup is %s", async (outcome) => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run); + deps.options.closeWarmNativeSessionsForRun.mockResolvedValue({ closed: 0, busy: outcome === "busy" ? 1 : 0, failed: outcome === "failed" ? 1 : 0 }); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "stopping", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "stop_and_retain" }); + expect(deps.envOrchestrator.releaseForRun).not.toHaveBeenCalled(); + expect(database.access).not.toHaveBeenCalled(); + }); + + it("leaves successful keep-running ownership warm", async () => { + const database = guardedDatabase(); + const deps = callbacks(database.db); + deps.getRun.mockResolvedValue({ status: "succeeded", runtimeMode: "legacy" } as Run); + await createHeartbeatRunControl(database.db, deps).releaseEnvironmentLeasesForRun({ runId: "warm", companyId: "company", agentId: "agent", status: "succeeded", providerResourceDisposition: "keep_running" }); + expect(deps.options.closeWarmNativeSessionsForRun).not.toHaveBeenCalled(); + expect(deps.envOrchestrator.releaseForRun).toHaveBeenCalledWith(expect.objectContaining({ providerResourceDisposition: "keep_running" })); + }); + it("defers destruction when a warm session remains busy", async () => { const database = guardedDatabase(); const deps = callbacks(database.db); diff --git a/server/src/services/heartbeat/run-control.ts b/server/src/services/heartbeat/run-control.ts index 4586f29bb8c..8bef5e55cd7 100644 --- a/server/src/services/heartbeat/run-control.ts +++ b/server/src/services/heartbeat/run-control.ts @@ -289,24 +289,26 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont input.providerResourceDisposition, status, ); - if (providerResourceDisposition === "destroy") { + if (providerResourceDisposition !== "keep_running") { const closeResult = await ( options.closeWarmNativeSessionsForRun ?? closeWarmNativeSessionsForRun )({ runId: input.runId, - reason: "terminal heartbeat run destroyed its environment lease", + reason: providerResourceDisposition === "destroy" + ? "terminal heartbeat run destroyed its environment lease" + : "terminal heartbeat run stopped its environment lease", }).catch((err) => { logger.warn( { err, runId: input.runId }, - "failed to close warm native sessions before environment lease destruction", + "failed to close warm native sessions before environment lease stop", ); return { closed: 0, busy: 0, failed: 1 }; }); // A failed remote checkpoint cannot veto destruction of the isolated // sandbox after the run stopped. Provider destruction supplies the exit // proof; it does not turn the interrupted checkpoint into a success. - const remoteLeases = closeResult.failed > 0 && leaseOwnerRun && + const remoteLeases = providerResourceDisposition === "destroy" && closeResult.failed > 0 && leaseOwnerRun && ["cancelled", "failed", "timed_out", "interrupted"].includes(leaseOwnerRun.status) ? await db.select({ provider: environmentLeases.provider }).from(environmentLeases).where(and( eq(environmentLeases.companyId, input.companyId), @@ -318,7 +320,7 @@ export function createHeartbeatRunControl(db: Db, dependencies: HeartbeatRunCont if (closeResult.busy > 0 || (closeResult.failed > 0 && !canDestroyRemote)) { logger.warn( { runId: input.runId, warmNativeSessions: closeResult }, - "deferred environment lease destruction until warm native sessions close", + "deferred environment lease stop until warm native sessions close", ); return; } diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 21ed59e9596..bb9e4b32fa6 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -7876,6 +7876,116 @@ describe("native warm session supervision", () => { await closeWarmNativeSessionsForRun({ runId: owners[1]!.runId, reason: "fixture cleanup" }); }); + it.each(["provider active", "retaining"])("settles a requested computer Stop before physical retirement when %s", async (when) => { + const name = `computer-stop-${when}`; + const current = { ...execution, binding: { ...execution.binding, runId: name, agentId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1; + const order: string[] = []; + const close = vi.fn(async () => { order.push("checkpoint settled"); }); + const stop = async () => { + expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "stop exact run" })).toMatchObject({ busy: 1, closed: 0 }); + expect(close).not.toHaveBeenCalled(); + }; + const retainWarm = vi.fn(async () => { if (when === "retaining") await stop(); }); + const retire = vi.fn(async (options?: { beforeStop?: () => Promise }) => { + await options?.beforeStop?.(); order.push("physical retirement"); return true; + }); + const target = { kind: "remote", transport: "computer", environmentId: name, leaseId: name, + remoteCwd: `/home/user/${name}`, listenerPort: 45111, + resourceAuthority: { kind: "computer-owner", computerId: name, ownerId: name, generation: 1 }, + fileAuthority: { kind: "remote-persistent", placementId: name, root: `/home/user/${name}`, agentHome: `/home/user/${name}/home` }, + retainWarm, retire, inspectProcess: async () => ({ running: true, claim: {} }), launch: async () => ({}), + computerTool: { command: "cua-driver", args: ["mcp"] } }; + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + if (when === "provider active") await stop(); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: name, + normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + try { + await executePaperclipNativeSession({ db: leaseDb(current, {}, {}, [], {}, "running", [], { provider: "boat", providerLeaseId: name }), + execution: current, runnerInstanceId: name, runnerExecutionTarget: target as never }); + expect(order).toEqual(["checkpoint settled", "physical retirement"]); + expect(close).toHaveBeenCalledOnce(); + expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat stop" })).toEqual({ closed: 0, busy: 0, failed: 0 }); + } finally { await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" }); } + }); + + it.each([false, true])("drains a Stop after warm release before clearing execution scope; close fails=%s", async (fails) => { + const name = `late-warm-stop-${fails}`; + const current = { ...execution, binding: { ...execution.binding, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1; + const close = vi.fn(async () => { if (fails) throw new Error("checkpoint not settled"); }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "completed" }, terminal: { runTerminalState: "succeeded" }, turnId: name, + normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + let requested = false; + let stopResult: Awaited> | undefined; + let closedBeforeScopeExit = false; + try { + const result = await executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name, useRunnerd: true, + onEvent: async event => { + if (event.eventType === "run.performance.span" && event.payload?.span === "task.run.measured" && !requested) { + requested = true; + stopResult = await closeWarmNativeSessionsForRun({ runId: name, reason: "late exact stop" }); + closedBeforeScopeExit = close.mock.calls.length > 0; + } + } }); + expect(requested).toBe(true); + expect(stopResult).toMatchObject({ busy: 1 }); + expect(closedBeforeScopeExit).toBe(false); + expect(result.exitCode).toBe(0); + expect(close).toHaveBeenCalledOnce(); + expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat exact stop" })) + .toEqual({ closed: 0, busy: 0, failed: fails ? 1 : 0 }); + } finally { + close.mockResolvedValue(undefined); + await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" }); + } + }); + + it.each(["failed turn", "environment", "idle timeout"])("keeps a late explicit Stop fenced when an implicit %s close fails", async (kind) => { + const name = `explicit-during-close-${kind}`; + const current = { ...execution, binding: { ...execution.binding, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1; + let entered!: () => void, rejectClose!: (error: Error) => void; + const started = new Promise(resolve => { entered = resolve; }); + const pendingClose = new Promise((_resolve, reject) => { rejectClose = reject; }); + const failure = new Error("implicit checkpoint failed"); + const close = vi.fn(async () => { entered(); await pendingClose; }); + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "settled" }, terminal: { runTerminalState: kind === "failed turn" ? "failed" : "succeeded" }, turnId: name, + normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + if (kind === "idle timeout") vi.useFakeTimers(); + const running = executePaperclipNativeSession({ db: leaseDb(current), execution: current, runnerInstanceId: name, + runnerExecutionTarget: { kind: "local", environmentId: name } }).catch(error => error); + const closing = kind === "environment" + ? running.then(() => closeWarmNativeSessionsForEnvironment({ environmentId: name, reason: "implicit environment cleanup" })) + : running; + try { + if (kind === "idle timeout") { await running; await vi.advanceTimersByTimeAsync(60_000); } + await started; + expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "explicit Stop during close" })).toMatchObject({ busy: 1 }); + rejectClose(failure); + await closing; + if (kind === "idle timeout") await vi.advanceTimersByTimeAsync(0); + expect(await closeWarmNativeSessionsForRun({ runId: name, reason: "repeat explicit Stop" })).toEqual({ closed: 0, busy: 0, failed: 1 }); + } finally { + rejectClose(failure); + await closing; + close.mockResolvedValue(undefined); + await closeWarmNativeSessionsForRun({ runId: name, reason: "fixture cleanup" }); + if (kind === "idle timeout") vi.useRealTimers(); + } + }); + it.each(["model", "credential", "tools", "read-only"])("replaces a computer warm runner after %s configuration admits a fresh owner", async (change) => { const name = `computer-config-change-${change}`; const current = { ...execution, binding: { ...execution.binding, runId: `${name}-one`, agentId: name, executionWorkspaceId: name }, @@ -8513,12 +8623,12 @@ describe("native warm session supervision", () => { ...execution, binding: { ...execution.binding, - runId: "run-warm-restart-release", - executionWorkspaceId: "workspace-warm-restart-release", + runId: `run-warm-restart-release-${checkpointFails}`, + executionWorkspaceId: `workspace-warm-restart-release-${checkpointFails}`, }, session: { ...execution.session, - normalizedSessionId: "session-warm-restart-release", + normalizedSessionId: `session-warm-restart-release-${checkpointFails}`, lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 }, }, } as NativeExecutionInputV1; @@ -8544,7 +8654,8 @@ describe("native warm session supervision", () => { const running = executePaperclipNativeSession({ db: leaseDb(warmExecution), execution: warmExecution, - runnerInstanceId: "runner-warm-restart-release", + runnerInstanceId: `runner-warm-restart-release-${checkpointFails}`, + useRunnerd: true, }).then((result) => { settled = true; return result; }); try { await vi.waitFor(() => expect(close).toHaveBeenCalledExactlyOnceWith({ reason: "controller restart" })); @@ -8554,7 +8665,12 @@ describe("native warm session supervision", () => { if (checkpointFails) await expect(running).rejects.toBe(checkpointError); else await running; } - await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: 0 }); + expect(close).toHaveBeenCalledOnce(); + await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toEqual({ closed: 0, busy: 0, failed: checkpointFails ? 1 : 0 }); + if (checkpointFails) { + close.mockResolvedValue(undefined); + await expect(closeIdleWarmNativeSessionsForRestart()).resolves.toMatchObject({ closed: 1 }); + } }); it("preserves the active turn when a warm checkpoint resumes the same run", async () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 5327794d333..d482219dd06 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -483,6 +483,8 @@ type WarmNativeSession = { environmentId: string | null; busy: boolean; closeOnReleaseReason?: string; + closeAfterExecution?: boolean; + explicitCloseRequested?: boolean; idleTimer: ReturnType | null; lastActivityAt: string; }; @@ -504,6 +506,7 @@ class ComputerOwnerSupersededError extends Error { } export async function closeWarmNativeSession(entry: WarmNativeSession, reason: string, preserveInstructionsForRunId?: string) { + entry.closeAfterExecution = false; let closeFailure: unknown; let closeCompleted = false; let closeState: WarmRetirementDiagnostic["closeState"] = "not_started"; @@ -709,7 +712,7 @@ export async function closeWarmNativeSessionsForEnvironment(input: { /** Cancel one attempt without closing other agents on its computer. */ export async function closeWarmNativeSessionsForRun(input: { runId: string; reason: string }) { - return closeIdleWarmNativeSessions(input); + return closeIdleWarmNativeSessions({ ...input, closeBusyOnRelease: true }); } /** Suspend idle owners and persist their remote backup before a controller @@ -737,11 +740,15 @@ async function closeIdleWarmNativeSessions(input: { if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) { continue; } + if (input.closeBusyOnRelease) entry.explicitCloseRequested = true; if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) { // A busy turn can complete while another idle session is checkpointing. // Fence that entry now so its eventual release cannot leave a new idle // owner behind after the shutdown sweep has already passed it. - if (input.closeBusyOnRelease) entry.closeOnReleaseReason = input.reason; + if (input.closeBusyOnRelease) { + entry.closeOnReleaseReason = input.reason; + if (executingRunnerdSessionScopes.has(sessionId)) entry.closeAfterExecution = true; + } busy += 1; continue; } @@ -759,7 +766,8 @@ async function closeIdleWarmNativeSessions(input: { if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId); closed += 1; } catch { - if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId); + // A later requested Stop must not mistake failed cleanup for no owner. + if (!entry.explicitCloseRequested && !entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId); failed += 1; } finally { entry.busy = false; @@ -6362,10 +6370,10 @@ async function releaseWarmNativeSession( // A late completion or cleanup callback from an older execution must never // release a replacement that has since claimed the same logical session. if (!entry || entry.ownerToken !== ownerToken) return; - entry.busy = false; + entry.busy = true; entry.lastActivityAt = new Date().toISOString(); if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); - if (failed || entry.closeOnReleaseReason !== undefined) { + const closeRequested = async () => { const requireCleanClose = entry.closeOnReleaseReason !== undefined; entry.closeOnReleaseReason ??= "warm native session failed"; entry.busy = true; @@ -6373,13 +6381,25 @@ async function releaseWarmNativeSession( await closeWarmNativeSession(entry, entry.closeOnReleaseReason); if (warmNativeSessions.get(sessionId) === entry) warmNativeSessions.delete(sessionId); } catch (error) { - if (!entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId); - if (requireCleanClose) throw error; + // An explicit Stop can arrive while an implicit close is pending. + // Consult the current request, not only the pre-await close reason. + if (!requireCleanClose && !entry.explicitCloseRequested && !entry.instructionWorkingCopy?.runId) warmNativeSessions.delete(sessionId); + if (requireCleanClose || entry.explicitCloseRequested) throw error; } finally { entry.busy = false; } + }; + if (failed || entry.closeOnReleaseReason !== undefined) { + await closeRequested(); return; } // Durable shared-computer ownership must be retained before completion is acknowledged. - await entry.computerTarget?.retainWarm(idleTimeoutMs); + try { await entry.computerTarget?.retainWarm(idleTimeoutMs); } + finally { entry.busy = false; } + // Stop may arrive while durable retention is being written. It still owns + // protocol close before any physical retirement, even after a successful turn. + if (entry.closeOnReleaseReason !== undefined) { + await closeRequested(); + return; + } entry.idleTimer = setTimeout(() => { const current = warmNativeSessions.get(sessionId); if (current !== entry || current.busy) return; @@ -6395,7 +6415,7 @@ async function releaseWarmNativeSession( current.idleTimer = null; return; } - if (!current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); + if (!current.explicitCloseRequested && !current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); }) .finally(() => { current.busy = false; }); }, idleTimeoutMs); @@ -7877,6 +7897,13 @@ export async function executePaperclipNativeSession(input: { input.execution.binding.runId ) { executingRunnerdSessionScopes.delete(sessionScopeId); + const entry = warmNativeSessions.get(sessionScopeId); + // A stop arriving after warm release but before this scope clears cannot + // close in parallel with execution. Drain that exact request now, retaining + // failed cleanup without reclassifying the committed provider result. + if (entry?.runId === runId && entry.closeAfterExecution && entry.closeOnReleaseReason !== undefined) { + await closeWarmNativeSessionsForRun({ runId, reason: entry.closeOnReleaseReason }); + } } try { await cleanupStagedAttachments();