diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 77562d0644d..03909f22e00 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1860,6 +1860,22 @@ mid-turn can leave provider or tool processes that the fence cannot find, so it keeps the fail-closed path below, and the task continues in a fresh session from the saved work. +A slow answer is not a dead Runner. After a release swap the new server can reach +a live Runner late, while the Runner is still replaying the events it saved with +no controller attached; its first command then times out although the Runner is +working. So a `replaceUnansweredRunner` attempt first checks whether the live +Runner is making progress. It adopts the Runner, launching no second one and +signalling nothing, when the Runner's saved event sequence is past the +controller's last acknowledgement, or when its saved state (event and command +sequences, generation, lifecycle) changes within a bounded wait of about ten +seconds. Only a Runner that made no progress in that wait is replaced as above: +the relaunch's launch fence stops it and its process lineage with proof, and the +new Runner restores its own journal. This replacement also proceeds on a resumed +sandbox lease, where the in-place reuse check would otherwise refuse the live +Runner's unsuspended state (`runner_harness_state_mismatch: resumed_not_suspended`) +and leave it running. A Runner that exits during the wait follows the exited-Runner +rules: a settled suspension restores, a mid-turn exit stays fail-closed. A state read that fails is no evidence either way: the first successful read sets the baseline, and a Runner whose state could not be read at all during the wait is adopted, not replaced. A Runner whose identity changes during the wait fails closed. A settled exit during the wait restores on every attempt that admitted the replacement, including the extra fourth attempt. + On every `reattach_remote_runner` attempt, a turn the Runner reports as `provider_turn_lost_on_restore` with `recoverable: true` continues once in the same conversation, as on a local dead-runner resume. The Runner reports that cause only diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 26553995dbf..3083c864618 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1644,6 +1644,150 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); .toBe(unanswered ? true : undefined); }); + describe("a same-run reattachment claimed to replace a Runner that left its recovery read unanswered", () => { + async function replacementClaim(label: string) { + const fixture = await seedRun(`remote-unanswered-progress-${label}`); + await fixture.db.update(heartbeatRuns).set({ processPid: null, runnerProfileJson: { + nativeWorkspaceSync: remotePreparedWorkspace(), + sessionCheckpoint: { identity: { companyId, agentId, issueId: fixture.issueId, runId: fixture.runId, + sessionId: fixture.native.normalizedSessionId }, providerSessionId: "provider-session" }, + } }).where(eq(heartbeatRuns.id, fixture.runId)); + await fixture.db.update(nativeRunFinalizations).set({ attempt: 1, failureDetail: { + message: "native_session_recovery_failed: Error: runner_ingress_unavailable: PRP command runner.drain timed out (runner connected; pending ahead 0; last runner event seq 338 16s ago)", + recoveryMode: "exact_checkpoint_resume", checkpointExists: true, adoptedRunnerUnanswered: true, + } }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "graceful", runIds: [fixture.runId] }); + if (claim?.kind !== "reattach_remote_runner" || claim.replaceUnansweredRunner !== true) { + throw new Error(`expected a replacement claim, got ${JSON.stringify(claim)}`); + } + const identity = { runId: fixture.runId, normalizedSessionId: fixture.native.normalizedSessionId, + runnerInstanceId: "unanswered-runner", environmentLeaseId: "lease", turnId: "turn", itemId: "item" }; + return { claim, identity }; + } + function sandbox(providerLeaseId: string, identity: Record, runnerState: (read: number) => Record | null, + alive: (check: number) => boolean) { + let reads = 0, checks = 0; + const execute = vi.fn(async (request: { args: string[] }) => { + const script = request.args[1] ?? ""; + if (request.args[2] === "paperclip-runner-recovery-check") { + return { stdout: "", stderr: "", exitCode: alive(checks++) ? 0 : 45, timedOut: false }; + } + if (script.includes("base64")) { + const state = runnerState(reads++); + if (state === null) return { stdout: "", stderr: "state read failed", exitCode: 1, timedOut: false }; + return { stdout: Buffer.from(JSON.stringify({ ...identity, outbox: [], pendingTerminalDelivery: null, pendingProviderCleanup: null, + ...state })).toString("base64"), stderr: "", exitCode: 0, timedOut: false }; + } + if (script.includes("cat --")) { + return { stdout: `nonce\n4242\n2026-11-13T09:00:00.000Z\n${String(identity.runnerInstanceId)}\nlinux:abcd-1234:100\n`, + stderr: "", exitCode: 0, timedOut: false }; + } + return { stdout: "", stderr: "", exitCode: 0, timedOut: false }; + }); + const target = { kind: "remote" as const, transport: "sandbox" as const, remoteCwd: "/workspace", providerKey: "daytona", + leaseId: "lease", sandboxLeaseAcquisition: { outcome: "resumed" as const, providerLeaseId }, runner: { execute } }; + const signalled = () => execute.mock.calls.some(([request]) => /kill -/.test(request.args.join(" "))); + return { target, signalled }; + } + async function reattach(label: string, runnerState: (read: number) => Record | null, + alive: (check: number) => boolean = () => true, controllerAckedSourceSeq: number | null = 338, providerAttempt?: number) { + const { claim, identity } = await replacementClaim(label); + if (claim.remote.kind === "plugin_task") throw new Error("expected a sandbox claim"); + const { target, signalled } = sandbox(claim.remote.providerLeaseId, identity, runnerState, alive); + const onReplace = vi.fn(); + const outcome = await verifyRemoteRunnerOrSettledExit({ claim, target: target as never, identity, runId: identity.runId, + normalizedSessionId: identity.normalizedSessionId, providerAttempt: providerAttempt ?? claim.providerAttempt, + unansweredRunnerProgress: { controllerAckedSourceSeq, onReplace, windowMs: 200, pollMs: 20 } }).catch(error => error as Error); + return { outcome, onReplace, signalled }; + } + + it("adopts a live Runner that emitted events past the controller's last acknowledgement", async () => { + const { outcome, onReplace, signalled } = await reattach("emitted", + () => ({ lifecycle: "turn_active", nextSourceSeq: 2345, ackedSourceSeq: 338, lastControllerCommandSeq: 40 })); + expect(outcome).toMatchObject({ pid: 4242 }); + expect(onReplace).not.toHaveBeenCalled(); + expect(signalled()).toBe(false); + }); + + it("adopts a live Runner whose saved state advances within the bounded wait", async () => { + const { outcome, onReplace, signalled } = await reattach("advancing", + read => ({ lifecycle: "turn_active", nextSourceSeq: 339 + read, ackedSourceSeq: 338, lastControllerCommandSeq: 40 }), + () => true, null); + expect(outcome).toMatchObject({ pid: 4242 }); + expect(onReplace).not.toHaveBeenCalled(); + expect(signalled()).toBe(false); + }); + + it("hands a live Runner that made no progress to the relaunch, whose launch fence stops it, without signalling it here", async () => { + const { outcome, onReplace, signalled } = await reattach("wedged", + () => ({ lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 })); + expect(outcome).toBeUndefined(); + expect(onReplace).toHaveBeenCalledOnce(); + expect(signalled()).toBe(false); + }); + + it("restores from the journal of a Runner that suspended and exited during the wait", async () => { + const { outcome, onReplace } = await reattach("suspended-exit", + read => read === 0 ? { lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 } + : { lifecycle: "suspended", nextSourceSeq: 339, ackedSourceSeq: 339, lastControllerCommandSeq: 40 }, + check => check === 0); + expect(outcome).toBeUndefined(); + expect(onReplace).not.toHaveBeenCalled(); + }); + + it("adopts a live Runner whose first state read failed and whose later reads advance", async () => { + const { outcome, onReplace, signalled } = await reattach("first-read-failed", + read => read === 1 ? null : { lifecycle: "turn_active", nextSourceSeq: 339 + read, ackedSourceSeq: 338, lastControllerCommandSeq: 40 }, + () => true, null); + expect(outcome).toMatchObject({ pid: 4242 }); + expect(onReplace).not.toHaveBeenCalled(); + expect(signalled()).toBe(false); + }); + + it("adopts a live Runner whose first readable state is already past the controller's acknowledgement", async () => { + const { outcome, onReplace } = await reattach("first-read-failed-emitted", + read => read === 0 ? { lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 } + : read === 1 ? null : { lifecycle: "turn_active", nextSourceSeq: 2345, ackedSourceSeq: 338, lastControllerCommandSeq: 40 }); + expect(outcome).toMatchObject({ pid: 4242 }); + expect(onReplace).not.toHaveBeenCalled(); + }); + + it("does not replace a live Runner whose state could not be read at all during the wait", async () => { + const { outcome, onReplace, signalled } = await reattach("unreadable", + read => read === 0 ? { lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 } : null); + expect(outcome).toMatchObject({ pid: 4242 }); + expect(onReplace).not.toHaveBeenCalled(); + expect(signalled()).toBe(false); + }); + + it("fails closed when the Runner's identity changes during the wait", async () => { + const { outcome, onReplace } = await reattach("identity-changed", + read => read === 0 ? { lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 } + : { lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40, runnerInstanceId: "another-runner" }); + expect(outcome).toBeInstanceOf(Error); + expect((outcome as Error).message).toContain("runner_remote_recovery_identity_mismatch"); + expect(onReplace).not.toHaveBeenCalled(); + }); + + it("restores from the journal of a Runner that suspended and exited during the wait on the admitted fourth attempt", async () => { + const { outcome, onReplace } = await reattach("suspended-exit-fourth", + read => read === 0 ? { lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 } + : { lifecycle: "suspended", nextSourceSeq: 339, ackedSourceSeq: 339, lastControllerCommandSeq: 40 }, + check => check === 0, 338, 4); + expect(outcome).toBeUndefined(); + expect(onReplace).not.toHaveBeenCalled(); + }); + + it("stays fail-closed for a Runner that exited mid-turn during the wait", async () => { + const { outcome, onReplace } = await reattach("midturn-exit", + () => ({ lifecycle: "turn_active", nextSourceSeq: 339, ackedSourceSeq: 338, lastControllerCommandSeq: 40 }), + check => check === 0); + expect(outcome).toBeInstanceOf(Error); + expect(onReplace).not.toHaveBeenCalled(); + }); + }); + it("keeps a restart recovery's attempt credit while the agent directory is busy, then resumes at attempt 3", async () => { const fixture = await seedRun("remote-busy-directory"); await fixture.db.update(heartbeatRuns).set({ diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index eb678727efe..7ddf5de47d2 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -18306,6 +18306,50 @@ describe("runnerd provider runtime wiring", () => { expect(existsSync(join(root, "control-plane", "control-plane-state.json"))).toBe(true); }); + it.each(["progressing", "wedged"] as const)("reattaches a live %s Runner whose recovery read went unanswered on a resumed sandbox lease", async mode => { + const current = scopedNativeExecution(`unanswered-live-runner-${mode}`); + const identity = { runId: current.binding.runId, normalizedSessionId: current.session.normalizedSessionId!, + runnerInstanceId: "unanswered-codex", environmentLeaseId: "unanswered-lease", + turnId: `turn_attachment_${randomUUID()}`, itemId: `item_attachment_${randomUUID()}` }; + const root = await initializeScopedRunnerRoot(current, identity.runnerInstanceId); + await mkdir(join(root, "control-plane"), { recursive: true }); + await writeFile(join(root, "control-plane", "control-plane-state.json"), + JSON.stringify({ ...durableControlPlaneState(identity), ackedSourceSeq: 2204 })); + const execute = vi.fn(async (request: { args: string[] }) => { + const script = request.args[1] ?? ""; + return { exitCode: 0, timedOut: false, stderr: "", + stdout: request.args[0] === "--build-metadata" ? JSON.stringify({ + schema: "paperclip-runner/runnerd-build-metadata/v1", binaryName: "paperclip-runnerd", + packageName: "@paperclipai/paperclip-runner", binaryContractVersion: 2, + durableSessionCapabilities: ["unlimited_runtime", "connection_lease_renewal"], prpTransportModes: ["listen_ws"], + }) : request.args[0] === "--version" ? "codex-cli 0.160.0" : script.includes("base64") + ? Buffer.from(JSON.stringify({ ...identity, lifecycle: "turn_active", outbox: [{ id: 2205 }], pendingTerminalDelivery: null, + nextSourceSeq: mode === "progressing" ? 2345 : 2205, ackedSourceSeq: 2204, lastControllerCommandSeq: 61 })).toString("base64") + : script.includes("cat --") ? `nonce\n1806\n2026-11-13T08:30:36.187Z\n${identity.runnerInstanceId}\nlinux:abcd-1234:100\n` : "" }; + }); + state.createBackend.mockClear(); state.createTransport.mockClear(); + const logs: string[] = []; + await createRunnerdBackend({ db: leaseDb(current), execution: current, runnerInstanceId: identity.runnerInstanceId, + runnerIngressAuthorized: true, onLog: async (_stream, chunk) => { logs.push(chunk); }, + controllerPersistence: { bootId: "successor", runnerArtifact: { version: "admitted-version", digest: `sha256:${"a".repeat(64)}` } }, + runnerExecutionTarget: { kind: "remote", transport: "sandbox", remoteCwd: "/workspace", environmentId: "environment", + leaseId: identity.environmentLeaseId, providerKey: "daytona", runner: { execute }, + effectiveCapabilities: { runnerWebSocketIngress: true }, + sandboxLeaseAcquisition: { outcome: "resumed", providerLeaseId: "sandbox" } } as never, + restartRecovery: { kind: "reattach_remote_runner", runId: current.binding.runId, leaseOwner: "controller", + controllerGeneration: 6, providerAttempt: 2, restartKind: "hard", recoveryRequestId: null, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" }, replaceUnansweredRunner: true }, + }); + state.createBackend.mock.calls.at(-1)![1].codexTransportFactory!(); + const options = state.createTransport.mock.calls.at(-1)![0] as RunnerTransportOptions & { prepareExternalRunnerState: () => Promise }; + if (mode === "progressing") expect(options.adoptExistingRunner).toMatchObject({ pid: 1806 }); + else expect(options.adoptExistingRunner).toBeUndefined(); + await expect(options.prepareExternalRunnerState()).resolves.toBeUndefined(); + expect(execute.mock.calls.some(([request]) => /kill -/.test(request.args.join(" ")))).toBe(false); + expect(execute.mock.calls.some(([request]) => request.args.includes("paperclip-runner-launch"))).toBe(false); + expect(logs.some(line => line.includes("made no progress"))).toBe(mode === "wedged"); + }, 30_000); + it.each(["failed", "timeout", "succeeded"] as const)("retains the local controller checkpoint until remote continuity archiving succeeds: %s", async mode => { const current = scopedNativeExecution("failed-remote-continuity-archive"); state.createBackend.mockClear(); state.createTransport.mockClear(); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index dc3d0d78106..279c8c81ed0 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7124,6 +7124,18 @@ type RunnerdDurableIdentity = Record & { environmentLeaseId: string; }; +function readRunnerdControllerAckedSourceSeq(root: string): number | null { + const statePath = resolve(root, "control-plane", "control-plane-state.json"); + if (!isSafeNativeStateDirectory(root) || !existsSync(statePath)) return null; + try { + const acked = record(JSON.parse(readBoundedNativeFile(statePath, NATIVE_CONTROL_PLANE_STATE_MAX_BYTES, + "runner_durable_identity_too_large").toString("utf8"))).ackedSourceSeq; + return typeof acked === "number" && Number.isSafeInteger(acked) && acked >= 0 ? acked : null; + } catch { + return null; + } +} + function readRunnerdDurableIdentity( root: string, allowExternal = false, @@ -15388,10 +15400,68 @@ async function readRemoteRunnerExitEvidence(runner: CommandManagedRuntimeRunner, } } +export interface UnansweredRunnerProgressProbe { + controllerAckedSourceSeq: number | null; + onReplace: () => void; + windowMs?: number; + pollMs?: number; +} + +const UNANSWERED_RUNNER_PROGRESS_WINDOW_MS = 10_000; +const UNANSWERED_RUNNER_PROGRESS_POLL_MS = 2_000; + +async function observeUnansweredRunnerProgress(input: { + runner: CommandManagedRuntimeRunner; + stateDirectory: string; + identity: Record; + probe: UnansweredRunnerProgressProbe; + isAlive: () => Promise; + recoveryDeadline: number; + stall: RemoteRunnerPluginStall; +}): Promise<"progressing" | "wedged" | "exited"> { + const read = async () => { + const state = await readRemoteRunnerState({ runner: input.runner, stateDirectory: input.stateDirectory, + recoveryDeadline: input.recoveryDeadline, stall: input.stall }).catch(() => null); + if (state && REMOTE_RUNNER_IDENTITY_FIELDS.some(field => state[field] !== input.identity[field])) { + throw new Error("runner_remote_recovery_identity_mismatch"); + } + return state; + }; + const progressMarker = (state: Record | null) => state ? JSON.stringify([state.generation ?? null, + state.reconnectCount ?? null, state.nextSourceSeq ?? null, state.ackedSourceSeq ?? null, + state.lastControllerCommandSeq ?? null, state.lifecycle ?? null]) : null; + const settle = async () => await input.isAlive() ? "progressing" as const : "exited" as const; + const emittedPastAck = (state: Record) => { + const emittedSourceSeq = Number(state.nextSourceSeq) - 1; + return input.probe.controllerAckedSourceSeq !== null && Number.isSafeInteger(emittedSourceSeq) + && emittedSourceSeq > input.probe.controllerAckedSourceSeq; + }; + let baseline: string | null = null; + const progressed = (state: Record | null) => { + if (!state) return false; + if (emittedPastAck(state)) return true; + const marker = progressMarker(state); + if (baseline === null) { + baseline = marker; + return false; + } + return marker !== baseline; + }; + if (progressed(await read())) return settle(); + const end = Math.min(Date.now() + (input.probe.windowMs ?? UNANSWERED_RUNNER_PROGRESS_WINDOW_MS), input.recoveryDeadline); + while (Date.now() < end) { + await new Promise(done => setTimeout(done, Math.min(input.probe.pollMs ?? UNANSWERED_RUNNER_PROGRESS_POLL_MS, end - Date.now()))); + if (progressed(await read())) return settle(); + } + if (baseline === null) return settle(); + return await input.isAlive() ? "wedged" : "exited"; +} + export async function verifyRemoteRunnerOrSettledExit( input: Parameters[0] & { providerAttempt: number; - claim: { exitedBeforeSessionOpen?: true; replaceUnansweredRunner?: true } }, -) { + claim: { exitedBeforeSessionOpen?: true; replaceUnansweredRunner?: true }; + unansweredRunnerProgress?: UnansweredRunnerProgressProbe; admittedAttemptLimit?: number }, +): Promise> | undefined> { const recoveryDeadline = Math.min(input.recoveryDeadline ?? Number.POSITIVE_INFINITY, Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS); const stall = input.stall ?? remoteRunnerPluginStall(input.recoveryDeadline); @@ -15406,8 +15476,8 @@ export async function verifyRemoteRunnerOrSettledExit( const diagnostic = readRemoteRunnerRecoveryError(error); const exitedBeforeSessionOpen = input.claim.exitedBeforeSessionOpen === true; const freshBootstrap = diagnostic !== null && exitedRunnerRequestedFreshBootstrap(diagnostic); - const attemptLimit = freshBootstrap && input.claim.replaceUnansweredRunner === true - ? NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + 1 : NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT; + const attemptLimit = Math.max(input.admittedAttemptLimit ?? 0, freshBootstrap && input.claim.replaceUnansweredRunner === true + ? NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + 1 : NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT); if (input.providerAttempt > attemptLimit || diagnostic?.stage !== "process_verification" || diagnostic.reason !== "process_not_running" || (!exitedBeforeSessionOpen && !freshBootstrap && @@ -15420,7 +15490,20 @@ export async function verifyRemoteRunnerOrSettledExit( : state.lifecycle !== "suspended" || !describeExitedRemoteRunner(state).runnerSettled) throw error; return undefined; } - return replaceUnanswered ? undefined : verified; + if (!replaceUnanswered) return verified; + if (!input.unansweredRunnerProgress) return undefined; + const progress = await observeUnansweredRunnerProgress({ runner: target.runner!, + stateDirectory: remoteRunnerStateDirectory(target.remoteCwd, input.normalizedSessionId), identity: input.identity, + probe: input.unansweredRunnerProgress, isAlive: () => verified.isAlive(), stall, + recoveryDeadline: Math.min(input.recoveryDeadline ?? Number.POSITIVE_INFINITY, Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS) }); + if (progress === "progressing") return verified; + if (progress === "exited") { + const { replaceUnansweredRunner: _replace, ...claim } = input.claim; + return verifyRemoteRunnerOrSettledExit({ ...input, claim, unansweredRunnerProgress: undefined, + admittedAttemptLimit: NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + 1 }); + } + input.unansweredRunnerProgress.onReplace(); + return undefined; } function runnerStateAllowsFreshStart(state: Record): boolean { @@ -16778,6 +16861,7 @@ async function createRunnerdBackendWithinSessionClaim( // exited. The transport then relaunches it from durable state, which needs // the admitted provider pack's launch inputs that adoption does not. let relaunchesSettledRemoteRunner = false; + let replacesUnansweredLiveRunner = false; const explicitRemoteCodex = input.runnerRemoteCodexPath?.trim() || null; const remoteCodexNpmSpec = input.runnerRemoteCodexNpmSpec?.trim() || null; if (explicitRemoteCodex && remoteCodexNpmSpec) { @@ -17515,6 +17599,9 @@ async function createRunnerdBackendWithinSessionClaim( throw new RemoteRunnerRecoveryError({ stage: "process_verification", reason: "process_not_running" }); } }); + } else if (sandboxLeaseAcquisition?.outcome === "resumed" && replacesUnansweredLiveRunner) { + await input.onLog?.("stdout", "[paperclip-runner] The runner left its recovery read unanswered and made no progress; " + + "the relaunch stops it and its process lineage, then restores its saved journal.\n"); } else if (sandboxLeaseAcquisition?.outcome === "resumed" && !adoptedProcess && !stagedRecovery && input.restartRecovery?.kind === "reattach_remote_runner" && await measureNativeRunnerSpan(input.trace, "sandbox.lease.resume", retainsFreshBootstrapRunnerState)) { @@ -18170,6 +18257,10 @@ async function createRunnerdBackendWithinSessionClaim( runId: input.execution.binding.runId, normalizedSessionId: nativeSessionKey(input.execution), providerAttempt: input.restartRecovery.providerAttempt, + unansweredRunnerProgress: { + controllerAckedSourceSeq: readRunnerdControllerAckedSourceSeq(root), + onReplace: () => { replacesUnansweredLiveRunner = true; }, + }, }) : target.kind === "local" && localRecoveryProcess ? {