diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index bea7c10ddec..0e802f1d336 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -6016,6 +6016,25 @@ cannot. Process verification and the final state read share one recovery deadline. A timeout, command failure, transport failure or invalid saved JSON on that final read keeps its closed recovery stage and reason in the diagnostic. +A Runner that no controller reaches before its connection lease expires, or +before its reconnect grace or deadline ends, stops its provider, saves its state +with lifecycle `recoverable_failure` and `lease_expired_requires_bootstrap`, +`transport_reconnect_grace_exceeded` or `transport_reconnect_deadline_exceeded`, +and exits. Only a new process with a fresh bootstrap ticket can reconnect, and +that process continues from the same cursor. A reattachment that finds this exit +re-reads the state after the process is proven gone. When it has the same six +identity fields, the same lifecycle and code, and no pending terminal delivery +or warm transition, the reattachment launches a Runner from that state, even +mid-turn and with events still in the outbox. Sandbox preparation reads that +state once more and keeps it in place, as it does for an adopted live Runner; +it does not require a `suspended` state or an idle provider checkpoint for this +exit. The new Runner restores the +conversation: it continues the turn, or reports the lost turn as recoverable +after it proves that the old provider process ended. This path admits the same +attempts as a reattachment, including the extra attempt of an unanswered-Runner +replacement. On staging, a deploy that kept a reattachment from reconnecting for +more than the lease ended these runs with `runner_remote_process_identity_unavailable`. + Reattachment keeps the complete durable PRP binding, including its turn and item attachment IDs. A healthy warm handoff can create new attachment IDs within the retained Runner. Replacing that run's controller must preserve those IDs; deriving diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index fbf92cb97cd..113c0e724f6 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1146,6 +1146,34 @@ describe("remote controller restart adoption", () => { target: exitedRunner(state) as never, identity, runId: "run", normalizedSessionId: "session", providerAttempt }).catch(caught => caught); expect(readRemoteRunnerRecoveryError(error)).toMatchObject({ stage: "process_verification", reason: "process_not_running" }); }); + describe("a reattachment whose Runner exited on its own and asked for a fresh bootstrap", () => { + const leaseExpired = { lifecycle: "recoverable_failure", recoverableFailure: "lease_expired_requires_bootstrap", + outbox: [{ id: 412 }], pendingTerminalDelivery: null, pendingProviderCleanup: null, warmTransition: null }; + const unanswered = { ...claim, replaceUnansweredRunner: true as const }; + it.each([ + ["a lease expiry on attempt 2", claim, leaseExpired, 2], + ["a lease expiry on the last attempt", claim, leaseExpired, 3], + ["a lease expiry on the admitted replacement attempt", unanswered, leaseExpired, 4], + ["an elapsed reconnect deadline", claim, { ...leaseExpired, recoverableFailure: "transport_reconnect_deadline_exceeded" }, 2], + ["an elapsed reconnect grace", claim, { ...leaseExpired, recoverableFailure: "transport_reconnect_grace_exceeded" }, 2], + ] as const)("launches a new Runner over the saved state after %s", async (_case, recoveryClaim, state, providerAttempt) => { + await expect(verifyRemoteRunnerOrSettledExit({ claim: recoveryClaim, target: exitedRunner(state) as never, identity, + runId: "run", normalizedSessionId: "session", providerAttempt })).resolves.toBeUndefined(); + }); + it.each([ + ["an unknown recoverable failure", claim, leaseExpired, { ...leaseExpired, recoverableFailure: "provider_failed" }, 2], + ["a warm transition", claim, leaseExpired, { ...leaseExpired, warmTransition: { transitionId: "warm" } }, 2], + ["a pending terminal delivery", claim, leaseExpired, { ...leaseExpired, pendingTerminalDelivery: { eventId: "event" } }, 2], + ["a state that changed after the exit", claim, leaseExpired, { ...leaseExpired, lifecycle: "turn_active" }, 2], + ["a different Runner after the exit", claim, leaseExpired, { ...leaseExpired, runnerInstanceId: "other" }, 2], + ["an attempt beyond the budget", claim, leaseExpired, leaseExpired, 4], + ["an attempt beyond the replacement budget", unanswered, leaseExpired, leaseExpired, 5], + ] as const)("keeps the reattachment fail-closed for %s", async (_case, recoveryClaim, state, afterExit, providerAttempt) => { + const error = await verifyRemoteRunnerOrSettledExit({ claim: recoveryClaim, target: exitedRunner(state, afterExit) as never, + identity, runId: "run", normalizedSessionId: "session", providerAttempt }).catch(caught => caught); + expect(readRemoteRunnerRecoveryError(error)).toMatchObject({ stage: "process_verification", reason: "process_not_running" }); + }); + }); describe("a same-run reattachment whose adopted Runner never answered its recovery read", () => { const unanswered = { ...claim, replaceUnansweredRunner: true as const }; const midTurn = { lifecycle: "turn_active", outbox: [{ id: 338 }], pendingTerminalDelivery: null, pendingProviderCleanup: null }; @@ -17767,6 +17795,63 @@ describe("runnerd provider runtime wiring", () => { || request.args.includes("paperclip-runner-claim-unstarted-session"))).toBe(false); }); + it.each([ + ["keeps", "lease_expired_requires_bootstrap"], + ["refuses", "provider_failed"], + ] as const)("%s the saved mid-turn state of a remote Runner that exited for %s when the reattachment prepares its sandbox", async (outcome, recoverableFailure) => { + const current = scopedNativeExecution(`fresh-bootstrap-relaunch-${recoverableFailure}`); + const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, + runnerInstanceId: "lease-expired-runner", environmentLeaseId: "lease-expired-lease", + turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId); + await mkdir(join(root, "control-plane"), { recursive: true }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), JSON.stringify(durableControlPlaneState(identity))); + const runnerState = { ...identity, lifecycle: "recoverable_failure", recoverableFailure, outbox: [{ id: 412 }], + pendingTerminalDelivery: null, pendingProviderCleanup: null, warmTransition: null }; + const providerState = { schema: "paperclip.runner.codex-provider-state.v1", lifecycle: "session_open", + threadId: "lease-expired-thread", providerSessionId: null, activeProviderTurnId: "provider-turn-1", + config: { provider: "codex", driver: "codex_app_server" } }; + const execute = vi.fn(async (request: { args: string[] }) => { + const script = request.args[1] ?? ""; + if (request.args[2] === "paperclip-runner-recovery-check") return { exitCode: 45, timedOut: false, stderr: "", stdout: "" }; + const stdout = request.args[0] === "--build-metadata" ? JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + }) : request.args[0] === "--version" ? "codex-cli 0.160.0" + : script.includes("base64") && script.includes("runner-state.json") ? Buffer.from(JSON.stringify(runnerState)).toString("base64") + : script.includes("base64") ? Buffer.from(JSON.stringify(providerState)).toString("base64") + : script.includes("cat --") ? `nonce\n123\n2026-09-19T10:00:00.000Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : ""; + return { exitCode: 0, timedOut: false, stderr: "", stdout }; + }); + const syncIn = vi.fn(async () => undefined); + state.createBackend.mockClear(); state.createTransport.mockClear(); + const created = createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: identity.runnerInstanceId, + runnerIngressAuthorized: true, + controllerPersistence: { bootId: "successor", runnerArtifact: { version: "admitted-version", digest: `sha256:${"a".repeat(64)}` } }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", + environmentId: "environment", leaseId: identity.environmentLeaseId, providerKey: "daytona", runner: { execute, syncIn }, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never, + restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId, leaseOwner: "controller", + controllerGeneration: 2, providerAttempt: 2, restartKind: "hard", recoveryRequestId: null, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } }, + }); + if (outcome === "refuses") { + await expect(created).rejects.toThrow("runner_remote_process_identity_unavailable"); + return; + } + await created; + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise }; + expect(options).toMatchObject({ prpIdentity: identity }); + expect((options as { adoptExistingRunner?: unknown }).adoptExistingRunner).toBeUndefined(); + await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + expect(syncIn).not.toHaveBeenCalled(); + expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-claim-unstarted-session") + || request.args.some(argument => argument.includes(".continuity-break-") || argument.includes("rm -")))).toBe(false); + }); + it.each(["claude", "pi"] as const)("relays the assigned gateway when a deploy lands mid-run and the same ACPX %s run reattaches its surviving runner", async agent => { const base = scopedNativeExecution(`relay-mid-run-${agent}`); const current = { ...base, provider: { kind: "acpx", agent, model: null }, diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 59fb391c5e6..3b6d3662c47 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -291,7 +291,8 @@ import { reportRunFailure } from "../run-failure-report.js"; import { REMOTE_RECOVERY_MARKER_READ_SCRIPT, REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, REMOTE_RUNNER_EXIT_EVIDENCE_SCRIPT, RemoteRunnerRecoveryError, attachRemoteRunnerRecoveryDiagnostic, boundRemoteRunnerExitEvidence, - describeExitedRemoteRunner, describeExitedRemoteRunnerReason, readRemoteRunnerRecoveryError, remoteRecoveryProbeReason, + describeExitedRemoteRunner, describeExitedRemoteRunnerReason, exitedRunnerRequestedFreshBootstrap, readRemoteRunnerRecoveryError, + remoteRecoveryProbeReason, runnerStateRequestsFreshBootstrap, } from "./remote-runner-recovery-diagnostics.js"; import { isPluginRpcTimeout, readRemoteRunnerRecoveryProbe, REMOTE_RUNNER_PLUGIN_STALL_BUDGET_MS, REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS, remoteRunnerPluginStall, type RemoteRunnerPluginStall } from "./remote-runner-recovery-probe.js"; import { describeOwnFailedAttachment, inspectRemoteWarmSessionRecovery, isSettledRemoteWarmControlState, summarizePostTerminalEvents, type RemoteWarmSessionRecovery, unsettledControllerCommands } from "./remote-warm-session-recovery.js"; @@ -15372,14 +15373,18 @@ export async function verifyRemoteRunnerOrSettledExit( } catch (error) { const diagnostic = readRemoteRunnerRecoveryError(error); const exitedBeforeSessionOpen = input.claim.exitedBeforeSessionOpen === true; - if (input.providerAttempt > NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT || diagnostic?.stage !== "process_verification" || + const freshBootstrap = diagnostic !== null && exitedRunnerRequestedFreshBootstrap(diagnostic); + const attemptLimit = freshBootstrap && input.claim.replaceUnansweredRunner === true + ? NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + 1 : NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT; + if (input.providerAttempt > attemptLimit || diagnostic?.stage !== "process_verification" || diagnostic.reason !== "process_not_running" || - (!exitedBeforeSessionOpen && (diagnostic.runnerLifecycle !== "suspended" || diagnostic.runnerSettled !== true)) || + (!exitedBeforeSessionOpen && !freshBootstrap && + (diagnostic.runnerLifecycle !== "suspended" || diagnostic.runnerSettled !== true)) || target?.kind !== "remote" || target.transport !== "sandbox" || !target.runner) throw error; const state = await readRemoteRunnerState({ runner: target.runner, stateDirectory: remoteRunnerStateDirectory(target.remoteCwd, input.normalizedSessionId), recoveryDeadline, stall }); if (REMOTE_RUNNER_IDENTITY_FIELDS.some(field => state[field] !== input.identity[field])) throw error; - if (exitedBeforeSessionOpen ? !runnerStateAllowsFreshStart(state) + if (freshBootstrap ? !runnerStateRequestsFreshBootstrap(state) : exitedBeforeSessionOpen ? !runnerStateAllowsFreshStart(state) : state.lifecycle !== "suspended" || !describeExitedRemoteRunner(state).runnerSettled) throw error; return undefined; } @@ -17177,6 +17182,13 @@ async function createRunnerdBackendWithinSessionClaim( }; }; + const retainsFreshBootstrapRunnerState = async (): Promise => { + if (!remoteCommandRunner || !remoteStateDirectory || !durableIdentity) return false; + const runnerState = await readRemoteRunnerState({ runner: remoteCommandRunner, stateDirectory: remoteStateDirectory }); + return runnerStateRequestsFreshBootstrap(runnerState) + && REMOTE_RUNNER_IDENTITY_FIELDS.every(field => runnerState[field] === durableIdentity[field]); + }; + const claimUntouchedRemoteSession = async (): Promise => { if (!remoteCommandRunner || !remoteRuntimeRoot || !remoteSessionRoot) return false; const controllerState = runnerControlPlane?.store.state; @@ -17470,6 +17482,10 @@ async function createRunnerdBackendWithinSessionClaim( throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }); } }); + } else if (sandboxLeaseAcquisition?.outcome === "resumed" && !adoptedProcess && !stagedRecovery && + input.restartRecovery?.kind === "reattach_remote_runner" && + await measureNativeRunnerSpan(input.trace, "sandbox.lease.resume", retainsFreshBootstrapRunnerState)) { + await input.onLog?.("stdout", "[paperclip-runner] Starting a new runner from the saved state of the runner whose connection lease ended.\n"); } else if (sandboxLeaseAcquisition?.outcome === "resumed") { const reuseStartedAtMs = Date.now(); const state = await measureNativeRunnerSpan( diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts index c35cee2a9db..8cc70adc72d 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts @@ -90,6 +90,29 @@ export function describeExitedRemoteRunner(state: Record): Pick }; } +/** + * The Runner exits with these codes after it stops its provider and saves its + * state, because only a new process with a fresh bootstrap ticket can + * reconnect. The replacement continues from the same cursor. + */ +const FRESH_BOOTSTRAP_RECOVERABLE_FAILURES = new Set([ + "lease_expired_requires_bootstrap", "transport_reconnect_deadline_exceeded", "transport_reconnect_grace_exceeded", +]); + +export function exitedRunnerRequestedFreshBootstrap( + diagnostic: Pick, +): boolean { + return diagnostic.runnerLifecycle === "recoverable_failure" + && FRESH_BOOTSTRAP_RECOVERABLE_FAILURES.has(diagnostic.runnerRecoverableFailure ?? ""); +} + +/** The saved state of an exited Runner that asked for a fresh bootstrap, re-read after its exit. */ +export function runnerStateRequestsFreshBootstrap(state: Record): boolean { + return exitedRunnerRequestedFreshBootstrap({ ...describeExitedRemoteRunner(state), + ...describeExitedRemoteRunnerReason(state, text => text) }) + && state.pendingTerminalDelivery == null && state.warmTransition == null; +} + const transportDiagnostics = new WeakMap(); export function attachRemoteRunnerRecoveryDiagnostic(error: unknown, diagnostic: RemoteRunnerRecoveryDiagnostic): unknown {