diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 5a179465d7c..bea7c10ddec 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1812,6 +1812,17 @@ by the deploy, and the task then lost its session to a fresh one. A Stop, any identity or authority refusal, any other recovery failure, and a protocol integrity fault still stop it. +The failed attempt's close keeps the session quarantined in the server process, +and later close retries only replay that failed close, so they never settle it. +The same run's reattachment to that Runner (in the same remote sandbox) takes +over the quarantine instead of waiting for it: it lets any close attempt still +in flight settle, then retires the attempt's cleanup and adopts the Runner. +Before this, the reattachment was refused with `native_session_cleanup_quarantined` +(`prior session cleanup remains incomplete` while retries were scheduled, or +`the previous session is still shutting down` once the close was +operator-required), each refusal spent a same-run attempt, and the task needed +Retry. Other runs of the task and other sandboxes stay gated as before. + A kept Runner can answer its WebSocket pings but no recovery command. On staging, a reattached Codex Runner authenticated and restored its session, then left `session.snapshot` unanswered (`pending ahead 0; last runner event seq 338 diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 0ff940f210e..ec48e18204a 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -11132,3 +11132,137 @@ describe("remote Runner retention after a failed run", () => { expect(retainedOn(close).every(retained => retained === false)).toBe(true); }); }); + +describe("same-run reattachment to a Runner the failed attempt retained", () => { + const capabilities = { resume: true, typedEvents: true, steering: false, interruption: false, structuredResult: true }; + const controlPlane = (): ControlPlanePort => ({ + openRun: async () => {}, checkpointSession: async () => {}, + appendEvent: async () => ({ cursor: 0, highestContiguousSourceSeq: 0, disposition: "committed" }), + replayEvents: async () => ({ events: [], highestContiguousSourceSeq: 0 }), completeRun: vi.fn(async () => {}), + }); + const attemptSession = (runIdentity: NativeRunIdentity, failure: string, close: () => Promise): NativeSession => ({ + identity: () => runIdentity, + capabilities: async () => capabilities, + async *events() { throw new Error(failure); }, + startTurn: async () => ({ turnId: "retained-turn" }), + result: async () => null, + async snapshot() { return { backendKind: "mock", sessionId: runIdentity.sessionId, identity: runIdentity, + providerSessionId: "retained-provider", cursor: null, activeTurnId: null, pendingRuntimeRequests: [], lineage: [] }; }, + close: vi.fn(close), + }); + const setup = (name: string) => { + const runIdentity = { ...identity, companyId: `${name}-company`, runId: `${name}-run` }; + const scopedInput = { ...input, binding: { ...input.binding, companyId: runIdentity.companyId, runId: runIdentity.runId } }; + const block = { companyId: runIdentity.companyId, issueId: scopedInput.binding.issueId }; + const reattached = attemptSession(runIdentity, "reattached attempt reached the Runner", async () => {}); + const options = (session: NativeSession, extra: Partial = {}): ExecuteNativeSessionOptions => ({ + input: scopedInput, + backend: { descriptor: async () => ({ kind: "runner", name: `${name}-backend`, version: "1", capabilities }), + openSession: vi.fn(async () => session) }, + controlPlane: controlPlane(), runnerInstanceId: `${name}-runner`, controlPlaneInstanceId: "control", + requireSessionCloseBeforeReturn: true, remoteCleanupScope: `${name}-sandbox`, ...extra, + }); + return { runIdentity, scopedInput, block, reattached, options }; + }; + + it("admits the run's own reattachment after a close that left the remote Runner alive and unsettled", async () => { + const { runIdentity, scopedInput, block, reattached, options } = setup("retained-unsettled"); + const retained = new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, runnerRetained: true }); + const failed = attemptSession(runIdentity, "runnerd did not report the provider turn identity", async () => { throw retained; }); + await expect(executeNativeSession(options(failed))).rejects.toMatchObject({ + message: "runnerd did not report the provider turn identity", cleanupError: retained }); + expect(nativeSessionCleanupBlock(block)).toBe("operator_required"); + + const otherRun = options(reattached, { retainedRunnerReattachment: true, + input: { ...scopedInput, binding: { ...scopedInput.binding, runId: "another-run" } } }); + await expect(executeNativeSession(otherRun)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); + + const reattachment = options(reattached, { retainedRunnerReattachment: true }); + await expect(executeNativeSession(reattachment)).rejects.toThrow("reattached attempt reached the Runner"); + expect(reattachment.backend.openSession).toHaveBeenCalledOnce(); + expect(failed.close).toHaveBeenCalledOnce(); + expect(nativeSessionCleanupBlock(block)).toBeNull(); + }); + + it("keeps refusing a run that is not reattaching to its own retained remote Runner", async () => { + const remote = setup("retained-without-takeover"); + const retained = new NativeSessionCloseUnrecoverableError({ runnerSuspended: false, runnerRetained: true }); + const failed = attemptSession(remote.runIdentity, "transport interrupted", async () => { throw retained; }); + await expect(executeNativeSession(remote.options(failed))).rejects.toThrow("transport interrupted"); + const fresh = remote.options(remote.reattached); + await expect(executeNativeSession(fresh)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); + expect(fresh.backend.openSession).not.toHaveBeenCalled(); + expect(nativeSessionCleanupBlock(remote.block)).toBe("operator_required"); + completeTerminatedRemoteNativeSessionCleanup({ companyId: remote.runIdentity.companyId, runId: remote.runIdentity.runId, + remoteCleanupScope: "retained-without-takeover-sandbox" }); + + const local = setup("retained-local"); + const localFailed = attemptSession(local.runIdentity, "local transport interrupted", async () => { throw retained; }); + await expect(executeNativeSession(local.options(localFailed, { remoteCleanupScope: undefined }))) + .rejects.toThrow("local transport interrupted"); + const localReattachment = local.options(local.reattached, { remoteCleanupScope: undefined, retainedRunnerReattachment: true }); + await expect(executeNativeSession(localReattachment)).rejects.toBeInstanceOf(NativeSessionCleanupQuarantinedError); + expect(localReattachment.backend.openSession).not.toHaveBeenCalled(); + completeTerminatedLocalNativeSessionCleanup({ companyId: local.runIdentity.companyId, runId: local.runIdentity.runId, + runnerInstanceId: "retained-local-runner" }); + }); + + it("admits the run's own reattachment instead of waiting out close retries that replay one failed close", async () => { + vi.useFakeTimers(); + try { + const { runIdentity, block, reattached, options } = setup("retained-replayed-close"); + const drainTimedOut = new Error("Controller journal drain timed out; preserve the session before replacing its writer."); + const failed = attemptSession(runIdentity, "runnerd did not report the provider turn identity", async () => { throw drainTimedOut; }); + const first = expect(executeNativeSession(options(failed))).rejects.toThrow("runnerd did not report the provider turn identity"); + await vi.advanceTimersByTimeAsync(10_000); + await first; + expect(nativeSessionCleanupBlock(block)).toBe("pending"); + + const closesBeforeReattachment = vi.mocked(failed.close).mock.calls.length; + const reattachment = options(reattached, { retainedRunnerReattachment: true }); + const admitted = expect(executeNativeSession(reattachment)).rejects.toThrow("reattached attempt reached the Runner"); + await vi.advanceTimersByTimeAsync(2_000); + await admitted; + expect(reattachment.backend.openSession).toHaveBeenCalledOnce(); + expect(nativeSessionCleanupBlock(block)).toBeNull(); + await vi.advanceTimersByTimeAsync(600_000); + expect(vi.mocked(failed.close).mock.calls.length).toBe(closesBeforeReattachment); + } finally { + vi.useRealTimers(); + } + }); + + it("lets a close that a deploy interrupted settle before the reattachment takes the Runner over", async () => { + vi.useFakeTimers(); + try { + const { runIdentity, block, reattached, options } = setup("retained-deploy-mid-cleanup"); + const interrupted = new Error("runner transport closed by deploy"); + let rejectClose!: (error: Error) => void; + const inFlightClose = new Promise((_resolve, reject) => { rejectClose = reject; }); + void inFlightClose.catch(() => undefined); + const failed = attemptSession(runIdentity, "server restarting for deploy", () => inFlightClose); + const first = executeNativeSession(options(failed, { requireSessionCloseBeforeReturn: false })); + void first.catch(() => undefined); + await vi.waitFor(() => expect(failed.close).toHaveBeenCalledOnce()); + expect(nativeSessionCleanupBlock(block)).toBe("pending"); + + const reattachment = options(reattached, { retainedRunnerReattachment: true }); + let settled = false; + const admitted = executeNativeSession(reattachment).catch((error: unknown) => { settled = true; return error; }); + await vi.advanceTimersByTimeAsync(1_000); + expect(settled).toBe(false); + expect(reattachment.backend.openSession).not.toHaveBeenCalled(); + + rejectClose(interrupted); + await vi.advanceTimersByTimeAsync(10_000); + await expect(admitted).resolves.toMatchObject({ message: "reattached attempt reached the Runner" }); + expect(reattachment.backend.openSession).toHaveBeenCalledOnce(); + expect(nativeSessionCleanupBlock(block)).toBeNull(); + const closesAfterTakeover = vi.mocked(failed.close).mock.calls.length; + await vi.advanceTimersByTimeAsync(600_000); + expect(vi.mocked(failed.close).mock.calls.length).toBe(closesAfterTakeover); + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 2f930321671..182f8e8b13f 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -329,6 +329,11 @@ export interface ExecuteNativeSessionOptions { /** Trusted provider resource identity: independent remote sandboxes must not * inherit each other's process-cleanup gates. Omit for local backends. */ remoteCleanupScope?: string; + /** This execution is the same run reattaching to the remote Runner that its + * earlier attempt's close left alive for exactly this reattachment. It takes + * over that attempt's retained cleanup in `remoteCleanupScope` instead of + * waiting for a close that can no longer settle. Other runs stay gated. */ + retainedRunnerReattachment?: boolean; /** Operation bound; explicit values also preserve the legacy turn bound. */ timeoutMs?: number; /** Total turn duration. Zero or no configured bound allows long-running work. */ @@ -1016,9 +1021,42 @@ function scheduleQuarantinedSessionCleanup( cleanup.timer.unref?.(); } +function quarantinedCleanupRunId(cleanup: QuarantinedSessionCleanup): string | null { + try { + return cleanup.session.identity().runId; + } catch { + return null; + } +} + +/** The same run's reattachment owns the remote Runner its earlier attempt + * retained, so it retires that attempt's cleanup in the same remote scope. + * Removal stops later close retries; an in-flight attempt stays an admission + * owner until it settles, so the old and new controllers never overlap. */ +function takeOverRetainedRunnerCleanups( + cleanupDomain: NativeSessionCleanupDomain, + runId: string, +): void { + if (JSON.parse(cleanupDomain.key).length !== 4) return; + for (const cleanup of [...quarantinedSessionCleanups]) { + if ( + cleanup.domain.key !== cleanupDomain.key || + !nativeSessionCleanupScopeBlocks(cleanup.domain, cleanupDomain) || + quarantinedCleanupRunId(cleanup) !== runId + ) + continue; + if (cleanup.timer) clearTimeout(cleanup.timer); + cleanup.timer = null; + quarantinedSessionCleanups.delete(cleanup); + if (!cleanup.attempt && !cleanup.recovery) + announceCleanupSettled(cleanup.domain); + } +} + async function retryQuarantinedSessionCleanups( cleanupDomain: NativeSessionCleanupDomain, signal?: AbortSignal, + reattachingRunId?: string, ): Promise { // A close attempt becomes admission-visible as soon as the runtime retains // its exact cleanup owner. It may not have rejected yet, so it may not have @@ -1030,6 +1068,8 @@ async function retryQuarantinedSessionCleanups( let observedOwnerPhases = 0; while (true) { signal?.throwIfAborted(); + if (reattachingRunId) + takeOverRetainedRunnerCleanups(cleanupDomain, reattachingRunId); if ( [...quarantinedSessionCleanups].some( (cleanup) => @@ -2353,7 +2393,13 @@ export async function executeNativeSession( ]), ], }; - await retryQuarantinedSessionCleanups(cleanupDomain, options.signal); + await retryQuarantinedSessionCleanups( + cleanupDomain, + options.signal, + options.retainedRunnerReattachment && options.remoteCleanupScope + ? input.binding.runId + : undefined, + ); if ("runtimeContext" in input) { const capabilities = descriptor.runtimeContextCapabilities; const unsupported = (["instructions", "skills", "mcp"] as const).filter( diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index afc631c3413..fbf92cb97cd 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -12662,6 +12662,7 @@ describe("native warm session supervision", () => { expect(state.stageNativeRunnerWakeAttachments).not.toHaveBeenCalled(); expect(cancel).toHaveBeenCalledTimes(invalidated ? 0 : 1); expect(submit).not.toHaveBeenCalled(); + expect(state.execute.mock.calls.at(-1)?.[0].retainedRunnerReattachment).toBe(true); } finally { state.stageNativeRunnerWakeAttachments.mockReset().mockImplementation(originalStaging); close.mockImplementation(async () => undefined); @@ -12965,6 +12966,7 @@ describe("native warm session supervision", () => { expect(state.execute).toHaveBeenCalledTimes(2); expect(state.execute.mock.calls.at(-1)?.[0].existingSession).toBeUndefined(); expect(close).toHaveBeenCalledTimes(closes); + expect(state.execute.mock.calls.at(-1)?.[0].retainedRunnerReattachment).toBe(false); }); it.each(["companyId", "agentId", "issueId", "executionWorkspaceId"] as const)( diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 122c1c5f4a1..b00c43c10be 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -12353,6 +12353,8 @@ async function executePaperclipNativeSessionAttempt( input: runnerExecution, startup: executionContext && (!usesPluginTask || executionContext.pluginSupported) ? executionContext : undefined, remoteCleanupScope: remoteCleanupLease ? remoteLeaseCleanupScope(remoteCleanupLease) : undefined, + retainedRunnerReattachment: input.restartRecovery?.kind === "reattach_remote_runner" + && input.restartRecovery.runId === input.execution.binding.runId, turnTimeoutMs: input.turnTimeoutMs, backend: input.backend ??