diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 4f1daf6b540..29aefe9693b 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1782,6 +1782,35 @@ in-flight turn stay for the next same-run reattachment. An abort (a Stop or the recovery deadline), any identity or authority refusal, any other recovery failure, and a protocol integrity fault still stop it. +A kept Runner can answer its WebSocket pings but no recovery command. On +staging, a reattached Codex Runner authenticated and restored its session, then +left `session.snapshot` unanswered (`pending ahead 0; last runner event seq 338 +76s ago`), and each later attempt adopted the same Runner and failed the same +way until the attempt budget ended the run. When a same-run reattachment fails +because its Runner left a recovery command unanswered (`PRP command timed +out`, or `runnerd exited while waiting for `), and the failure is not an +identity, authority or integrity refusal, the run records +`adoptedRunnerUnanswered` on its failure and the next claim carries +`replaceUnansweredRunner`. That attempt does not adopt the Runner again. After +the usual identity checks it launches a fresh Runner from the Runner's saved +journal and resumes the provider session from the saved checkpoint. This applies +only while the old Runner is alive: the launch fence stops that +`paperclip-runnerd` on the state folder and its whole process lineage, and fails +the launch closed when it cannot prove them gone. A Runner that already exited +mid-turn can leave provider or tool processes that the fence cannot find, so it +keeps the fail-closed path below, and the task continues in a fresh session from +the saved work. The fresh Runner +restores the provider thread; its in-flight turn is reconciled from history and +never submitted again. The provider attempt budget still bounds these attempts. + +A paginated Codex history read (`thread/turns/list`, `thread/items/list`) that +fails without an answer is reported as `codex_history_read_failed: read +failed`, with the underlying error. Only a provider answer that refuses the method +on its first page means the build lacks paginated history. Then the driver reads +the complete thread once (`thread/read` with `includeTurns: true`) and uses its +turns and items, failing closed when that read names another thread or omits +turns, items or turn identities. A refusal on a later page still fails closed. + A failed close can leave its `runner.suspend` (or `runner.shutdown`) pending in the saved journal of a Runner it kept alive. The controller sends one command at a time, so neither that command nor any command queued after it reached the @@ -5896,7 +5925,8 @@ lifecycle `suspended`, an empty outbox and no pending terminal delivery, provide cleanup or warm transition, the retry launches a Runner from that state instead of adopting one. This matches a settled staged source. Any other exit stays fail-closed for reconciliation, and the provider attempt budget still bounds the -retries. +retries. A live Runner that left the previous attempt's recovery read unanswered +is replaced, not adopted (see above). The final admitted provider attempt may use this path; attempts above the limit cannot. Process verification and the final state read share one recovery deadline. A timeout, command failure, transport failure or invalid saved JSON on diff --git a/packages/paperclip-runner/src/drivers/codex/codex-history.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-history.test.ts index 4979549fa59..4a51c29a4ec 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-history.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-history.test.ts @@ -1,10 +1,13 @@ import { describe, it, expect, vi } from "vitest"; +import { CODEX_METHOD_NOT_FOUND, CodexRpcError } from "./app-server-transport.js"; import { readCodexThreadState, readCodexTurnMetadata, readCodexTurnItems, } from "./codex-history.js"; +const unsupported = (method: string) => new CodexRpcError(`${method} is not supported`, CODEX_METHOD_NOT_FOUND); + describe("Codex paginated history", () => { it("reads state without hydrating history", async () => { const request = vi.fn().mockResolvedValue({ thread: { id: "thread" } }); @@ -79,14 +82,67 @@ describe("Codex paginated history", () => { "codex_history_incomplete", ); }); - it("does not turn an interrupted page or unsupported method into idle history", async () => { + it("does not turn an unsupported later page into idle history", async () => { const request = vi .fn() .mockResolvedValueOnce({ data: [], nextCursor: "next" }) - .mockRejectedValueOnce(new Error("method not found")); + .mockRejectedValueOnce(unsupported("thread/turns/list")); await expect(readCodexTurnMetadata({ request }, "thread")).rejects.toThrow( "supported paginated history", ); + expect(request.mock.calls.map(call => call[0])).toEqual(["thread/turns/list", "thread/turns/list"]); + }); + it("names a failed history read as a failed read, not as unsupported history", async () => { + const timeout = new Error("provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0)"); + const request = vi.fn().mockRejectedValueOnce(timeout); + await expect(readCodexTurnMetadata({ request }, "thread")).rejects.toThrow( + `codex_history_read_failed: thread/turns/list read failed; Error: ${timeout.message}`, + ); + expect(request).toHaveBeenCalledTimes(1); + }); + it("reads complete turn history from the thread when paginated history is unsupported", async () => { + const request = vi.fn(async (method: string) => { + if (method === "thread/turns/list") throw unsupported(method); + return { thread: { id: "thread", turns: [ + { id: "one", status: "completed", items: [{ id: "answer", type: "agentMessage", text: "done" }] }, + { id: "two", status: "inProgress", items: [] }, + ] } }; + }); + expect(await readCodexTurnMetadata({ request }, "thread")).toEqual([ + { id: "one", status: "completed", items: [], itemsView: "notLoaded" }, + { id: "two", status: "inProgress", items: [], itemsView: "notLoaded" }, + ]); + expect(request).toHaveBeenLastCalledWith("thread/read", { threadId: "thread", includeTurns: true }); + }); + it("reads a turn's items from the thread when paginated items are unsupported", async () => { + const answer = { id: "answer", type: "agentMessage", text: "done" }; + const request = vi.fn(async (method: string) => { + if (method === "thread/items/list") throw unsupported(method); + return { thread: { id: "thread", turns: [ + { id: "one", status: "completed", items: [answer] }, + { id: "two", status: "inProgress", items: [{ id: "other" }] }, + ] } }; + }); + expect(await readCodexTurnItems({ request }, "thread", "one")).toEqual([answer]); + }); + it.each([ + ["another thread", { thread: { id: "other", turns: [] } }, "different driver session"], + ["no turns", { thread: { id: "thread" } }, "supported paginated history"], + ["a turn without a status", { thread: { id: "thread", turns: [{ id: "one" }] } }, "codex_history_incomplete"], + ["a turn without an id", { thread: { id: "thread", turns: [{ status: "completed" }] } }, "codex_history_incomplete"], + ])("fails closed when the unpaginated thread read returns %s", async (_case, thread, message) => { + const request = vi.fn(async (method: string) => { + if (method === "thread/turns/list") throw unsupported(method); + return thread; + }); + await expect(readCodexTurnMetadata({ request }, "thread")).rejects.toThrow(message); + }); + it("fails closed when the unpaginated thread read omits the requested turn's items", async () => { + const request = vi.fn(async (method: string) => { + if (method === "thread/items/list") throw unsupported(method); + return { thread: { id: "thread", turns: [{ id: "one", status: "completed" }] } }; + }); + await expect(readCodexTurnItems({ request }, "thread", "one")).rejects.toThrow("codex_history_incomplete"); }); it("rejects another thread or turn", async () => { await expect( diff --git a/packages/paperclip-runner/src/drivers/codex/codex-history.ts b/packages/paperclip-runner/src/drivers/codex/codex-history.ts index 438f067484e..091db4941b2 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-history.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-history.ts @@ -1,9 +1,13 @@ -import type { CodexAppServerTransport } from "./app-server-transport.js"; +import { isCodexMethodUnavailable, type CodexAppServerTransport } from "./app-server-transport.js"; import { HarnessReconciliationError } from "../../contracts/harness-driver.js"; import { record, text } from "./codex-driver-values.js"; type Requester = Pick; +const TURN_STATUSES = ["inProgress", "completed", "failed", "interrupted", "cancelled"]; + +class CodexPaginatedHistoryUnsupported extends HarnessReconciliationError {} + /** A partial or unsupported provider read is never evidence that work is idle. */ async function pages( transport: Requester, @@ -23,9 +27,14 @@ async function pages( ...(cursor ? { cursor } : {}), }); } catch (error) { - throw new HarnessReconciliationError( - `codex_history_read_failed: ${method} requires supported paginated history; ${String(error)}`, - ); + if (!isCodexMethodUnavailable(error)) + throw new HarnessReconciliationError( + `codex_history_read_failed: ${method} read failed; ${String(error)}`, + ); + const unsupported = `codex_history_read_failed: ${method} requires supported paginated history; ${String(error)}`; + throw page === 0 + ? new CodexPaginatedHistoryUnsupported(unsupported) + : new HarnessReconciliationError(unsupported); } if (!Array.isArray(response.data)) throw new HarnessReconciliationError( @@ -71,31 +80,57 @@ export async function readCodexThreadState( return snapshot; } -export function readCodexTurnMetadata( +function turnIdentity(value: Record): string { + if (!TURN_STATUSES.includes(text(value.status))) { + throw new HarnessReconciliationError( + "codex_history_incomplete: invalid turn status", + ); + } + return text(value.id); +} + +async function readCompleteThreadTurns( + transport: Requester, + threadId: string, + method: string, + cause: unknown, +): Promise[]> { + if (!(cause instanceof CodexPaginatedHistoryUnsupported)) throw cause; + const snapshot = await transport.request("thread/read", { threadId, includeTurns: true }); + const thread = record(snapshot.thread); + if (text(thread.id) !== threadId) + throw new HarnessReconciliationError( + "thread/read returned a different driver session", + ); + if (!Array.isArray(thread.turns)) + throw new HarnessReconciliationError( + `codex_history_read_failed: ${method} requires supported paginated history; thread/read omitted turns`, + ); + const turns = thread.turns.map(record); + for (const turn of turns) { + if (!turnIdentity(turn)) + throw new HarnessReconciliationError( + "codex_history_incomplete: thread/read omitted an identity", + ); + } + return turns; +} + +export async function readCodexTurnMetadata( transport: Requester, threadId: string, ): Promise[]> { - return pages( - transport, - "thread/turns/list", - { threadId, sortDirection: "asc", itemsView: "notLoaded" }, - (value) => { - if ( - ![ - "inProgress", - "completed", - "failed", - "interrupted", - "cancelled", - ].includes(text(value.status)) - ) { - throw new HarnessReconciliationError( - "codex_history_incomplete: invalid turn status", - ); - } - return text(value.id); - }, - ); + try { + return await pages( + transport, + "thread/turns/list", + { threadId, sortDirection: "asc", itemsView: "notLoaded" }, + turnIdentity, + ); + } catch (error) { + const turns = await readCompleteThreadTurns(transport, threadId, "thread/turns/list", error); + return turns.map((turn) => ({ ...turn, items: [], itemsView: "notLoaded" })); + } } export async function readCodexTurnItems( @@ -103,17 +138,28 @@ export async function readCodexTurnItems( threadId: string, turnId: string, ): Promise[]> { - const entries = await pages( - transport, - "thread/items/list", - { threadId, turnId, sortDirection: "asc" }, - (value) => { - if (text(value.turnId) !== turnId) - throw new HarnessReconciliationError( - "thread/items/list returned a different turn", - ); - return text(record(value.item).id); - }, - ); + let entries: Record[]; + try { + entries = await pages( + transport, + "thread/items/list", + { threadId, turnId, sortDirection: "asc" }, + (value) => { + if (text(value.turnId) !== turnId) + throw new HarnessReconciliationError( + "thread/items/list returned a different turn", + ); + return text(record(value.item).id); + }, + ); + } catch (error) { + const turns = await readCompleteThreadTurns(transport, threadId, "thread/items/list", error); + const items = turns.find((turn) => text(turn.id) === turnId)?.items; + if (!Array.isArray(items)) + throw new HarnessReconciliationError( + "codex_history_incomplete: thread/read omitted the turn's items", + ); + return items.map(record); + } return entries.map((entry) => record(entry.item)); } diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts index dd23825af1c..10dc17e21a3 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts @@ -189,3 +189,111 @@ it("keeps a retained remote Runner for the reattaching controller, which continu await rm(root, { recursive: true, force: true }); } }, 30_000); + +it("continues the same provider thread from a fresh Runner over the journal of a retained Runner that stopped answering, without replaying its turn", async () => { + const root = await mkdtemp(join(tmpdir(), "runnerd-remote-retention-replace-")); + const callLog = join(root, "provider-calls.log"); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + server.on("upgrade", (request, socket, head) => { + if (!authority) { socket.destroy(); return; } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing replacement test listener"); + const registration = async (next: DurablePrpControlPlane) => { + authority = next; + return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: () => { if (authority === next) authority = null; } }; + }; + const children: ReturnType[] = []; + const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore", detached: true }); + children.push(child); + return { + child, + completion: new Promise((resolveExit, rejectExit) => { + child.once("error", rejectExit); + child.once("exit", (code, exitSignal) => resolveExit({ code, signal: exitSignal, stdout: "", stderr: "" })); + }), + isExitConfirmed: () => child.exitCode !== null || child.signalCode !== null, + }; + }); + const shared = { + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: resolve(import.meta.dirname, "../../runner/target/debug/fake-codex-app-server"), + codexArgs: ["--state-file", join(root, "provider-state.json"), "--hold-first-durable-turn", "--call-log", callLog], + stateDirectory: root, + prpIdentity: { + runnerInstanceId: "runner-replaced", environmentLeaseId: "lease-replaced", runId: "run-replaced", + normalizedSessionId: "session-replaced", turnId: "turn-replaced", itemId: "item-replaced", + }, + runnerReconnectGraceMs: 5_000, + runnerProcessReconnectGraceMs: null, + controlPlaneRegistration: registration, + runnerProcessLauncher: launch, + }; + const first = createCapabilityRunnerdCodexTransport({ ...shared, closeGraceMs: 600 }); + let adopted: ReturnType | undefined; + let replacement: ReturnType | undefined; + const providerPids: number[] = []; + try { + const opened = await first.transport.request("thread/start", { cwd: root, dynamicTools: codexSemanticToolSpecs() }); + await first.transport.request("turn/start", { input: [{ type: "text", text: "Keep this task active." }] }); + for await (const notification of first.transport.notifications()) { + if (notification.method === "turn/started") break; + } + const wedged = children[0]!.pid!; + providerPids.push(first.evidence().codexPid!); + signal(wedged, "SIGSTOP"); + await expect(first.transport.close("native session execution complete", { retainUnsettledRemoteRunner: true })) + .rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); + expect(processState(wedged)).toBe("alive"); + signal(wedged, "SIGCONT"); + const thread = opened.thread as Record; + const resume = { + resumeDynamicTools: codexSemanticToolSpecs(), + resumeProviderSession: { driverSessionId: String(thread.id), providerSessionId: String(thread.sessionId) }, + }; + adopted = createCapabilityRunnerdCodexTransport({ + ...shared, ...resume, + runnerProcessLauncher: vi.fn(() => { throw new Error("Must adopt the existing runner"); }), + adoptExistingRunner: { + pid: wedged, processGroupId: wedged, startedAt: new Date().toISOString(), + isAlive: () => processState(wedged) === "alive", + signal: name => { try { process.kill(-wedged, name); return true; } catch { return false; } }, + }, + }); + adopted.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + await expect(adopted.transport.request("thread/read", {})).resolves.toEqual( + expect.objectContaining({ thread: expect.objectContaining({ id: thread.id }) })); + signal(wedged, "SIGSTOP"); + await adopted.transport.detachControllerForRestart!(); + expect(processState(wedged)).toBe("alive"); + const journal = JSON.parse(await readFile(join(root, "runner", "runner-state.json"), "utf8")) as Record; + expect(journal.lifecycle).not.toBe("suspended"); + signal(-wedged, "SIGKILL"); + for (const pid of providerPids) signal(pid, "SIGKILL"); + await vi.waitFor(() => expect(processState(wedged)).toBe("gone"), { timeout: 5_000 }); + replacement = createCapabilityRunnerdCodexTransport({ + ...shared, ...resume, + }); + replacement.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + await expect(replacement.transport.request("thread/read", {})).resolves.toEqual( + expect.objectContaining({ thread: expect.objectContaining({ id: thread.id }) })); + if (replacement.evidence().codexPid) providerPids.push(replacement.evidence().codexPid!); + expect(launch).toHaveBeenCalledTimes(2); + expect(replacement.evidence().runnerPid).toBe(children[1]!.pid); + const calls = (await readFile(callLog, "utf8")).trim().split("\n"); + expect(calls.filter(call => call === "thread/start")).toHaveLength(1); + expect(calls.filter(call => call === "turn/start")).toHaveLength(1); + } finally { + await replacement?.transport.close().catch(() => undefined); + await adopted?.transport.close().catch(() => undefined); + for (const child of children) { signal(child.pid, "SIGCONT"); signal(child.pid ? -child.pid : null, "SIGKILL"); } + for (const pid of providerPids) signal(pid, "SIGKILL"); + server.closeAllConnections(); + if (server.listening) await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(root, { recursive: true, force: true }); + } +}, 60_000); diff --git a/server/src/services/native-runtime/native-restart-recovery.ts b/server/src/services/native-runtime/native-restart-recovery.ts index ff976cc2d45..76ea996d941 100644 --- a/server/src/services/native-runtime/native-restart-recovery.ts +++ b/server/src/services/native-runtime/native-restart-recovery.ts @@ -103,6 +103,7 @@ export type NativeRestartRecoveryClaim = /** Discovery evidence only; executor revalidates the encrypted header. */ stagedAttachment?: Pick; exitedBeforeSessionOpen?: true; + replaceUnansweredRunner?: true; } | { kind: "resume_dead_runner"; @@ -2121,6 +2122,8 @@ export async function claimNativeRestartRecoveries(input: { }, ...(!pluginTask && row.coordinator.failureDetail?.runnerExitedBeforeSessionOpen === true ? { exitedBeforeSessionOpen: true as const } : {}), + ...(!pluginTask && row.coordinator.failureDetail?.adoptedRunnerUnanswered === true + ? { replaceUnansweredRunner: true as const } : {}), } satisfies NativeRestartRecoveryClaim; } if (claimKind === "reattach_existing_runner") { diff --git a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts index 9a01087d3e5..336971ff09f 100644 --- a/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts +++ b/server/src/services/native-runtime/native-runner-restart-recovery.integration.test.ts @@ -1573,6 +1573,39 @@ process.on("SIGTERM", () => child.kill("SIGTERM")); expect(run?.processPid).toBe(remotePid); }); + it.each([true, false])("asks the same-run reattachment to replace a Runner that left its recovery read unanswered: %s", async (unanswered) => { + const fixture = await seedRun(`remote-unanswered-${unanswered}`); + await fixture.db.update(heartbeatRuns).set({ + processPid: null, + processStartedAt: new Date("2020-01-01T00:00:00Z"), + runnerProfileJson: { + nativeWorkspaceSync: { + schema: "paperclip.native-workspace-sync/v1", state: "prepared", + descriptorSha256: "a".repeat(64), baselineSha256: "b".repeat(64), + workspaceId: "workspace", leaseId: "lease", providerLeaseId: "sandbox", + remoteCwd: "/workspace", + }, + sessionCheckpoint: { + identity: { companyId, agentId, issueId: fixture.issueId, runId: fixture.runId, + sessionId: fixture.native.normalizedSessionId }, + providerSessionId: "provider-session", + process: { providerPid: process.pid }, + }, + }, + }).where(eq(heartbeatRuns.id, fixture.runId)); + await fixture.db.update(nativeRunFinalizations).set({ attempt: 2, failureDetail: { + message: "native_session_recovery_failed: HarnessReconciliationError: codex_history_read_failed: thread/turns/list requires supported paginated history; Error: provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0; last runner event seq 337 111s ago)", + recoveryMode: "exact_checkpoint_resume", checkpointExists: true, + ...(unanswered ? { adoptedRunnerUnanswered: true } : {}), + } }).where(eq(nativeRunFinalizations.runId, fixture.runId)); + const [claim] = await claimNativeRestartRecoveries({ db: fixture.db, controller: successor, + restartKind: "graceful", runIds: [fixture.runId] }); + expect(claim).toMatchObject({ kind: "reattach_remote_runner", runId: fixture.runId, providerAttempt: 2, + remote: { providerLeaseId: "sandbox", remoteCwd: "/workspace" } }); + expect(claim?.kind === "reattach_remote_runner" ? claim.replaceUnansweredRunner : "not remote") + .toBe(unanswered ? true : undefined); + }); + it("keeps a restart recovery's attempt credit while the agent directory is busy, then resumes at attempt 3", async () => { const fixture = await seedRun("remote-busy-directory"); await fixture.db.update(heartbeatRuns).set({ diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 06c2ec0115a..8f841d8be5a 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -476,6 +476,7 @@ import { verifyRemoteRunnerReattachment, stopStaleRemoteRunnerBeforeBootstrap, verifyRemoteRunnerOrSettledExit, + nativeRecoveryCommandUnanswered, readRemoteProviderPackManifest, verifyRemoteProviderPackManifest, prepareRemoteProviderPackOnTarget, @@ -1141,6 +1142,54 @@ describe("remote controller restart adoption", () => { target: exitedRunner(state) as never, identity, runId: "run", normalizedSessionId: "session", providerAttempt }).catch(caught => caught); expect(readRemoteRunnerRecoveryError(error)).toMatchObject({ stage: "process_verification", reason: "process_not_running" }); }); + describe("a same-run reattachment whose adopted Runner never answered its recovery read", () => { + const unanswered = { ...claim, replaceUnansweredRunner: true as const }; + const midTurn = { lifecycle: "turn_active", outbox: [{ id: 338 }], pendingTerminalDelivery: null, pendingProviderCleanup: null }; + it.each([2, 3])("launches a fresh Runner instead of adopting the live one again on attempt %i", async providerAttempt => { + const { target, execute } = fixture(midTurn); + await expect(verifyRemoteRunnerOrSettledExit({ claim: unanswered, target: target as never, identity, + runId: "run", normalizedSessionId: "session", providerAttempt })).resolves.toBeUndefined(); + expect(execute.mock.calls.some(([request]) => /kill -/.test(request.args.join(" ")))).toBe(false); + }); + it("adopts the live Runner when the earlier attempt did not report an unanswered read", async () => { + const { target } = fixture(midTurn); + const adopted = await verifyRemoteRunnerOrSettledExit({ claim, target: target as never, identity, + runId: "run", normalizedSessionId: "session", providerAttempt: 2 }); + expect(adopted?.pid).toBe(123); + }); + it.each([ + ["a mid-turn exit whose provider and tools may survive it", midTurn, 3], + ["a pending provider cleanup", { ...midTurn, pendingProviderCleanup: { pid: 7 } }, 3], + ["an exhausted attempt budget", midTurn, 4], + ] as const)("never resumes beside an exited unanswered Runner's lineage: %s", async (_case, state, providerAttempt) => { + const target = exitedRunner(state); + const error = await verifyRemoteRunnerOrSettledExit({ claim: unanswered, target: target as never, + identity, runId: "run", normalizedSessionId: "session", providerAttempt }).catch(caught => caught); + expect(readRemoteRunnerRecoveryError(error)).toMatchObject({ stage: "process_verification", reason: "process_not_running" }); + }); + it("keeps the live Runner's identity checks before replacing it", async () => { + const { target } = fixture({ ...midTurn, runnerInstanceId: "different" }); + await expect(verifyRemoteRunnerOrSettledExit({ claim: unanswered, target: target as never, identity, + runId: "run", normalizedSessionId: "session", providerAttempt: 2 })).rejects.toThrow("runner_remote_recovery_identity_mismatch"); + }); + it.each([ + "native_session_recovery_failed: HarnessReconciliationError: codex_history_read_failed: thread/turns/list requires supported paginated history; Error: provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0; last runner event seq 338 76s ago)", + "native_session_recovery_failed: HarnessCapabilityUnavailableError: read is unavailable: Error: runnerd exited while waiting for session.snapshot", + "native_session_recovery_failed: Error: runner_ingress_unavailable: PRP command runner.drain timed out (runner connected; pending ahead 0; last runner event seq 93 391s ago)", + ])("recognizes the staging recovery read that got no answer: %s", message => { + expect(nativeRecoveryCommandUnanswered(new Error(message))).toBe(true); + expect(nativeRecoveryCommandUnanswered(new Error("wrapped", { cause: new Error(message) }))).toBe(true); + }); + it.each([ + "runnerd did not report the provider turn identity", + "provider_transport_failed: PRP command session.snapshot timed out", + "native_session_recovery_failed: provider resumed a different session", + "native_session_recovery_failed: provider_transport_failed: PRP command session.snapshot timed out; native_adopted_provider_identity_mismatch", + "native_session_recovery_failed: restart interruption history is incomplete", + ])("does not replace a Runner for another failure: %s", message => { + expect(nativeRecoveryCommandUnanswered(new Error(message))).toBe(false); + }); + }); it.each([ ["an unsettled suspension", { ...settledSuspension, outbox: [{ id: 1 }] }, 2], ["a pending provider cleanup", { ...settledSuspension, pendingProviderCleanup: { pid: 7 } }, 2], diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 853df3f50b6..f8fc648a289 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8860,6 +8860,16 @@ export function nativeRunnerExitedBeforeSessionOpen(error: unknown): boolean { return nativeRunnerExitConfirmed(error) && runnerExitBeforeSessionOpenMessage(error); } +export function nativeRecoveryCommandUnanswered(error: unknown, depth = 0): boolean { + if (!error || typeof error !== "object" || depth > 5) return false; + const value = error as Record; + const message = typeof value.message === "string" ? value.message : ""; + if (message.startsWith("native_session_recovery_failed:") + && /PRP command [\w.]+ timed out|runnerd exited while waiting for [\w.]+/.test(message) + && !/mismatch|different|identity|authority|integrity|incompatible/i.test(message)) return true; + return nativeRecoveryCommandUnanswered(value.cause, depth + 1); +} + /** Same provider, category and message (numbers ignored): a retry would meet the same fault. */ export function nativeStartupFailureSignature(failure: unknown): string | null { const value = record(failure); @@ -13054,6 +13064,9 @@ async function executePaperclipNativeSessionAttempt( ? { runnerExitConfirmed: true } : {}), ...(input.runnerExecutionTarget?.kind === "remote" && nativeRunnerExitedBeforeSessionOpen(error) ? { runnerExitedBeforeSessionOpen: true } : {}), + ...(input.runnerExecutionTarget?.kind === "remote" && input.restartRecovery?.kind === "reattach_remote_runner" + && input.restartRecovery.remote.kind !== "plugin_task" && nativeRecoveryCommandUnanswered(error) + ? { adoptedRunnerUnanswered: true } : {}), ...(providerCooldownAt ? { providerCooldownAt: providerCooldownAt.toISOString() } : {}), ...(providerLimit?.plan.resetAt ? { providerLimitResetAt: providerLimit.plan.resetAt.toISOString() } : {}), recoveryOwner: recoveryProjection.recoveryOwner, @@ -14987,7 +15000,8 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT = * A launch therefore stops a `paperclip-runnerd` that is still serving this * exact `--state-dir`. A launch only happens after the server chose not to * adopt a Runner: a same-run reattachment proves the marker's Runner gone - * first, and a transient reattachment failure never reaches a launch. + * first, or replaces a Runner that left the previous attempt's recovery read + * unanswered. Any other transient reattachment failure never reaches a launch. * * The stop runs in node ($1 the runnerd binary, $2 the state directory, then * candidate PIDs): @@ -15270,16 +15284,20 @@ async function readRemoteRunnerExitEvidence(runner: CommandManagedRuntimeRunner, export async function verifyRemoteRunnerOrSettledExit( input: Parameters[0] & { providerAttempt: number; - claim: { exitedBeforeSessionOpen?: true } }, + claim: { exitedBeforeSessionOpen?: true; replaceUnansweredRunner?: true } }, ) { const recoveryDeadline = Math.min(input.recoveryDeadline ?? Number.POSITIVE_INFINITY, Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS); const stall = input.stall ?? remoteRunnerPluginStall(input.recoveryDeadline); + const target = input.target; + const replaceUnanswered = input.claim.replaceUnansweredRunner === true + && input.providerAttempt <= NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT + && target?.kind === "remote" && target.transport === "sandbox" && Boolean(target.runner); + let verified: Awaited>; try { - return await verifyRemoteRunnerReattachment({ ...input, recoveryDeadline, stall }); + verified = await verifyRemoteRunnerReattachment({ ...input, recoveryDeadline, stall }); } catch (error) { const diagnostic = readRemoteRunnerRecoveryError(error); - const target = input.target; const exitedBeforeSessionOpen = input.claim.exitedBeforeSessionOpen === true; if (input.providerAttempt > NATIVE_PROVIDER_RECOVERY_ATTEMPT_LIMIT || diagnostic?.stage !== "process_verification" || diagnostic.reason !== "process_not_running" || @@ -15292,6 +15310,7 @@ export async function verifyRemoteRunnerOrSettledExit( : state.lifecycle !== "suspended" || !describeExitedRemoteRunner(state).runnerSettled) throw error; return undefined; } + return replaceUnanswered ? undefined : verified; } function runnerStateAllowsFreshStart(state: Record): boolean {