diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 3216e40173b..31cbd5b5823 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -355,6 +355,31 @@ describe("persistent agent directories", () => { expect(worker.nextWakeAt()).toBeNull(); } finally { release(); await foreground; removes.mockRestore(); await worker.stop(); } }); + it("keeps a cleanup pending without an error while another server holds the agent directory lock", async () => { + const copy = await run(); + await db.update(agentInstructionWorkingCopies).set({ location: `remote:${randomUUID()}` }) + .where(eq(agentInstructionWorkingCopies.runId, copy.runId)); + await stopped(copy, new Date(Date.now() - 1000)); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + const held = await db.$client.reserve(); + const lockKey = `agent-directory:${companyId}:${agentId}`; + await held`select pg_advisory_lock(hashtextextended(${lockKey}, 0))`; + const errors: unknown[] = [], worker = coordinator(() => true, () => true, error => errors.push(error)); + try { + const before = Date.now(); + await copies.registerDirectoryCleanupWork(worker, () => true, async () => true).ready; + await expect.poll(() => idleWorkSnapshot().active).toBe(0); + expect(errors).toEqual([]); + const pending = (await copies.get(companyId, copy.runId))!; + expect(pending.receipt?.cleanupPending).not.toBe(false); + expect(pending.nextAttemptAt!.getTime()).toBeGreaterThan(before); + } finally { + await worker.stop(); + await held`select pg_advisory_unlock(hashtextextended(${lockKey}, 0))`; + held.release(); + vi.unstubAllEnvs(); + } + }); it("reactivates from the real worktree setting without polling disabled policy", async () => { const copy = await run(); await stopped(copy); const runtimeEnv = { PAPERCLIP_IN_WORKTREE: "true", PAPERCLIP_INSTANCE_ID: "sleep41-directory-activation" }; diff --git a/server/src/services/agent-instruction-working-copies.ts b/server/src/services/agent-instruction-working-copies.ts index f588c06284f..b2ab372e9b2 100644 --- a/server/src/services/agent-instruction-working-copies.ts +++ b/server/src/services/agent-instruction-working-copies.ts @@ -5,7 +5,7 @@ import { notifyInstructionCommit, type InstructionCommitScope } from "./instruct import { DELIVERY_QUEUES } from "./delivery-work-notifications.js"; import type { createDeliveryWorkCoordinator } from "./delivery-work-coordinator.js"; import { registerScopedDeadlineWork } from "./scoped-deadline-work.js"; -import { agentDirectoryWorkingCopyService, isAgentDirectoryCopy } from "./agent-directory-working-copies.js"; +import { AgentDirectoryOperationBusyError, agentDirectoryWorkingCopyService, isAgentDirectoryCopy } from "./agent-directory-working-copies.js"; import { createHash } from "node:crypto"; import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; @@ -564,7 +564,8 @@ export function agentInstructionWorkingCopyService(db: Db, options: { environmen if (claimed) { try { await directories.release(claimed); } catch (error) { - await deferCleanup(); + const deferred = await deferCleanup(); + if (error instanceof AgentDirectoryOperationBusyError) return directoryCleanupEntry(deferred); throw error; } } diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index c08975a60e1..81aba3bf4c1 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -13262,6 +13262,50 @@ describe("native warm session supervision", () => { expect(onGoalCheckpoint).toHaveBeenCalledOnce(); }); + it("closes idle warm sessions of different agents in parallel and one at a time within an agent", async () => { + await closeIdleWarmNativeSessionsForRestart(); + const owners = [["agent-close-a", "first"], ["agent-close-a", "second"], ["agent-close-b", "only"]] as const; + const finishers = new Map void>(); + let released = false; + const closes = new Map Promise>>>(); + try { + for (const [agentId, label] of owners) { + const name = `parallel-close-${agentId}-${label}`; + const close = vi.fn(() => released ? Promise.resolve() : new Promise(resolve => { finishers.set(name, resolve); })); + closes.set(name, close); + const warmExecution = { ...execution, binding: { ...execution.binding, agentId, issueId: name, runId: name, executionWorkspaceId: name }, + session: { ...execution.session, normalizedSessionId: name, + lifecyclePolicy: { mode: "warm" as const, idleTimeoutMs: 60_000 } } } as NativeExecutionInputV1; + state.execute.mockReset().mockImplementationOnce(async options => { + await options.onSession?.({ close }); + return { result: { summary: "Completed" }, terminal: { runTerminalState: "succeeded" }, + turnId: name, normalizedSessionId: name, providerSessionId: name, driverKind: "test", driverVersion: "1", + nativeEventCount: 1, highestContiguousSourceSeq: 1, usage: null }; + }); + await executePaperclipNativeSession({ db: leaseDb(warmExecution), execution: warmExecution, runnerInstanceId: name }); + } + const first = closes.get("parallel-close-agent-close-a-first")!; + const second = closes.get("parallel-close-agent-close-a-second")!; + const other = closes.get("parallel-close-agent-close-b-only")!; + const closing = closeIdleWarmNativeSessionsForRestart(); + await vi.waitFor(() => { + expect(first).toHaveBeenCalledOnce(); + expect(other).toHaveBeenCalledOnce(); + }); + await new Promise(resolve => setTimeout(resolve, 50)); + expect(second).not.toHaveBeenCalled(); + finishers.get("parallel-close-agent-close-a-first")!(); + await vi.waitFor(() => expect(second).toHaveBeenCalledOnce()); + finishers.get("parallel-close-agent-close-a-second")!(); + finishers.get("parallel-close-agent-close-b-only")!(); + expect(await closing).toEqual({ closed: 3, busy: 0, failed: 0, failures: [] }); + } finally { + released = true; + for (const finish of finishers.values()) finish(); + await closeIdleWarmNativeSessionsForRestart(); + } + }); + it("blocks idle sleep for a released local warm session until its close checkpoint finishes", async () => { await closeIdleWarmNativeSessionsForRestart(); const { reconcilePersistedRuntimeServicesOnStartup, readWorkspaceRuntimeIdleWork } = await import("../workspace-runtime.js"); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 54a3b50501d..3f33a94c005 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -747,6 +747,7 @@ type WarmNativeSession = { }; const WARM_CLOSE_FAILURE_DETAIL_MAX = 2_000; +const WARM_NATIVE_SESSION_CLOSE_CONCURRENCY = 4; export function describeWarmCloseFailure(error: unknown): string { const describe = (value: unknown): string => { @@ -1141,11 +1142,8 @@ async function closeIdleWarmNativeSessions(input: { let busy = 0; let failed = 0; const failures: WarmNativeSessionCloseFailure[] = []; - for (const [sessionId, entry] of [...warmNativeSessions]) { - if (input.canRetire && !input.canRetire()) break; - if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) { - continue; - } + let stopped = false; + const closeEntry = async (sessionId: string, entry: WarmNativeSession) => { if (entry.busy || entry.preparingRunId || executingRunnerdSessionScopes.has(sessionId) || (input.canRetire && (entry.instructionPreparationRunId || closingWarmNativeSessions.has(sessionId)))) { // A busy turn can complete while another idle session is checkpointing. @@ -1153,10 +1151,10 @@ async function closeIdleWarmNativeSessions(input: { // owner behind after the shutdown sweep has already passed it. if (input.closeBusyOnRelease) entry.closeOnReleaseReason = input.reason; busy += 1; - continue; + return; } // The host owns retention after release; a quiet local handle needs no retirement for sleep. - if (input.canRetire && entry.released && !entry.closeOnReleaseReason) continue; + if (input.canRetire && entry.released && !entry.closeOnReleaseReason) return; const detach = input.detachReleased === true && entry.released === true && !entry.closeOnReleaseReason; if (entry.idleTimer !== null) clearTimeout(entry.idleTimer); // Keep a fenced owner until both retirement and stopped collection succeed. @@ -1179,7 +1177,7 @@ async function closeIdleWarmNativeSessions(input: { if (detach) entry.closeOnReleaseReason = "released Runner local detach is unconfirmed"; if (!detach && error instanceof AgentDirectoryOperationBusyError) { busy += 1; - continue; + return; } const destroyed = !detach && input.forceStopUnsettled && input.db ? await warmSandboxDestroyed(input.db, entry).catch(proofError => { @@ -1193,7 +1191,7 @@ async function closeIdleWarmNativeSessions(input: { "warm native session's sandbox was already destroyed; released its owner"); forgetWarmNativeSession(sessionId, entry); closed += 1; - continue; + return; } failed += 1; const forced = forcedStopFailure === undefined ? "" : `; forced Runner stop failed: ${describeWarmCloseFailure(forcedStopFailure)}`; @@ -1207,8 +1205,28 @@ async function closeIdleWarmNativeSessions(input: { } releaseIdleSessionToolAuthority(sessionId); } - if (input.canRetire && !input.canRetire()) break; + }; + const agentQueues = new Map(); + for (const [sessionId, entry] of [...warmNativeSessions]) { + if (input.environmentId !== undefined && entry.environmentId !== input.environmentId) continue; + const agentKey = `${entry.companyId}:${entry.agentId}`; + const queue = agentQueues.get(agentKey); + if (queue) queue.push([sessionId, entry]); else agentQueues.set(agentKey, [[sessionId, entry]]); } + const pendingAgents = [...agentQueues.values()]; + const closeAgentQueues = async () => { + for (let queue = pendingAgents.shift(); queue; queue = pendingAgents.shift()) { + for (const [sessionId, entry] of queue) { + if (stopped || (input.canRetire && !input.canRetire())) { + stopped = true; + return; + } + await closeEntry(sessionId, entry); + } + } + }; + const concurrency = input.canRetire ? 1 : WARM_NATIVE_SESSION_CLOSE_CONCURRENCY; + await Promise.all(Array.from({ length: Math.min(concurrency, pendingAgents.length) }, closeAgentQueues)); if (failures.length > 0) logger.warn({ reason: input.reason, failures }, "idle warm native sessions did not close"); return { closed, busy, failed, failures }; }