From d5b19756214ff9d2d348e26ddcaa97c187907e7d Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Thu, 15 Oct 2026 08:09:38 -0700 Subject: [PATCH] fix(native): serialize warm idle expiry with turn admission (#16432) The warm idle timer no longer closes a warm native session while a continuation has reserved or is preparing the same session scope; it re-arms instead. An idle close that has already started is registered in closingWarmNativeSessions so turn admission waits for the checkpoint, matching the other closers. Co-Authored-By: Paperclip Co-Authored-By: Claude Opus 5.5 --- .../native-session-executor.test.ts | 80 +++++++++++++++++++ .../native-runtime/native-session-executor.ts | 24 +++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index f90b4ffeb9d..fac9f420114 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -7906,6 +7906,86 @@ describe("native warm session supervision", () => { } }); + it.each(["idle expiry first", "turn first"])("serializes idle expiry and turn admission: %s", async (order) => { + const stateBase = await mkdtemp(join(tmpdir(), "paperclip-idle-race-")); + const previousPaperclipHome = process.env.PAPERCLIP_HOME; + process.env.PAPERCLIP_HOME = stateBase; + let finishClose!: () => void; + const closing = new Promise((resolve) => { finishClose = resolve; }); + const close = vi.fn(() => closing); + const idleTimeoutMs = 200; + try { + const first = { + ...execution, + binding: { + ...execution.binding, + runId: "run-idle-race-first", + executionWorkspaceId: "workspace-idle-race", + }, + session: { + ...execution.session, + normalizedSessionId: "session-idle-race", + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs }, + }, + } as NativeExecutionInputV1; + const result = { + result: { summary: "completed" }, + terminal: { runTerminalState: "succeeded" }, + turnId: "turn-idle-race", + normalizedSessionId: first.session.normalizedSessionId, + providerSessionId: "provider-idle-race", + driverKind: "test", + driverVersion: "1", + nativeEventCount: 1, + highestContiguousSourceSeq: 1, + usage: null, + }; + state.execute.mockReset().mockImplementationOnce(async (options) => { + await options.onSession?.({ close }); + return result; + }); + await executePaperclipNativeSession({ + db: leaseDb(first), execution: first, + runnerInstanceId: "runner-idle-race", useRunnerd: true, + }); + const second = { ...first, binding: { ...first.binding, runId: "run-idle-race-second" } }; + const reachedAdmission = new Error("test reached post-checkpoint admission"); + state.stageNativeRunnerWakeAttachments.mockClear(); + state.stageNativeRunnerWakeAttachments.mockImplementationOnce(async () => { + if (order === "turn first") { + // The idle window elapses while the continuation still prepares. + await new Promise((resolve) => setTimeout(resolve, idleTimeoutMs * 2)); + expect(close).not.toHaveBeenCalled(); + } + throw reachedAdmission; + }); + if (order === "idle expiry first") { + await vi.waitFor(() => expect(close).toHaveBeenCalledOnce(), { timeout: 2_000 }); + } + const continuation = executePaperclipNativeSession({ + db: leaseDb(second), execution: second, + runnerInstanceId: "runner-idle-race", useRunnerd: true, + }).catch((error) => error); + if (order === "idle expiry first") { + await new Promise((resolve) => setTimeout(resolve, 50)); + expect(state.stageNativeRunnerWakeAttachments).not.toHaveBeenCalled(); + finishClose(); + } + expect(await continuation).toBe(reachedAdmission); + expect(state.stageNativeRunnerWakeAttachments).toHaveBeenCalledOnce(); + // A deferred idle owner still retires once the scope is released. + await vi.waitFor(() => expect(close).toHaveBeenCalledExactlyOnceWith({ + reason: "warm native session idle timeout", + }), { timeout: 2_000 }); + } finally { + finishClose(); + await closeIdleWarmNativeSessionsForRestart(); + if (previousPaperclipHome === undefined) delete process.env.PAPERCLIP_HOME; + else process.env.PAPERCLIP_HOME = previousPaperclipHome; + await rm(stateBase, { recursive: true, force: true }); + } + }); + it.each([false, true])("checkpoints a busy warm session on release after the restart sweep: checkpoint fails=%s", async (checkpointFails) => { const checkpointError = new Error("restart checkpoint failed"); let finishCheckpoint!: () => void; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 24d5198972f..04dcb09e8af 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -6162,15 +6162,35 @@ async function releaseWarmNativeSession( } finally { entry.busy = false; } return; } + armWarmNativeSessionIdleTimer(sessionId, entry, idleTimeoutMs); +} + +function armWarmNativeSessionIdleTimer( + sessionId: string, + entry: WarmNativeSession, + idleTimeoutMs: number, +): void { entry.idleTimer = setTimeout(() => { const current = warmNativeSessions.get(sessionId); if (current !== entry || current.busy) return; + if (current.preparingRunId || executingRunnerdSessionScopes.has(sessionId)) { + // A continuation has reserved this scope and is still preparing. It + // verifies this idle owner as the prior run's authority; suspending it + // underneath that check makes the ready state look indeterminate. + armWarmNativeSessionIdleTimer(sessionId, current, idleTimeoutMs); + return; + } current.busy = true; current.closeOnReleaseReason = "warm native session idle timeout"; - void closeWarmNativeSession(current, current.closeOnReleaseReason) + const closing = closeWarmNativeSession(current, current.closeOnReleaseReason) .then(() => { if (warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); }) .catch(() => { if (!current.instructionWorkingCopy?.runId && warmNativeSessions.get(sessionId) === current) warmNativeSessions.delete(sessionId); }) - .finally(() => { current.busy = false; }); + .finally(() => { + current.busy = false; + if (closingWarmNativeSessions.get(sessionId) === closing) closingWarmNativeSessions.delete(sessionId); + }); + // Admission waits for this checkpoint like a restart sweep's close. + closingWarmNativeSessions.set(sessionId, closing); }, idleTimeoutMs); entry.idleTimer.unref(); }