diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 351ea9b579b..5a179465d7c 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1804,9 +1804,13 @@ needs an operator, still stops the Runner. A reattachment after a restart whose times out or loses its transport (for example `PRP command session.snapshot timed out`) detaches from the Runner the same way a server restart does: it sends no turn stop and no suspend and does not signal the process. The Runner and its -in-flight turn stay for the next same-run reattachment. An abort (a Stop or the -recovery deadline), any identity or authority refusal, any other recovery -failure, and a protocol integrity fault still stop it. +in-flight turn stay for the next same-run reattachment. The provider recovery +deadline (`native session provider recovery timed out after 900000ms`) detaches +the same way. On staging, every Codex Runner that a reattachment later found +gone (`process_not_running`) had been stopped by this deadline's cleanup, not +by the deploy, and the task then lost its session to a fresh one. A Stop, any +identity or authority refusal, any other recovery failure, and a protocol +integrity fault still stop it. A kept Runner can answer its WebSocket pings but no recovery command. On staging, a reattached Codex Runner authenticated and restored its session, then @@ -1814,7 +1818,8 @@ left `session.snapshot` unanswered (`pending ahead 0; last runner event seq 338 76s ago`), and each later attempt adopted the same Runner and failed the same way until the attempt budget ended the run. When a same-run reattachment fails because its Runner left a recovery command unanswered (`PRP command timed -out`, or `runnerd exited while waiting for `), and the failure is not an +out`, `runnerd exited while waiting for `, or the provider recovery +deadline), and the failure is not an identity, authority or integrity refusal, the run records `adoptedRunnerUnanswered` on its failure and the next claim carries `replaceUnansweredRunner`. That attempt does not adopt the Runner again. After diff --git a/packages/paperclip-runner/src/contracts/native-session-backend.ts b/packages/paperclip-runner/src/contracts/native-session-backend.ts index 647186f2691..fcd43490e2f 100644 --- a/packages/paperclip-runner/src/contracts/native-session-backend.ts +++ b/packages/paperclip-runner/src/contracts/native-session-backend.ts @@ -184,6 +184,20 @@ export class NativeSessionCloseUnrecoverableError extends Error { } } +/** The provider recovery deadline ended a recovery that was still waiting on its Runner. */ +export class NativeProviderRecoveryDeadlineError extends Error { + readonly code = "native_provider_recovery_deadline"; + + constructor(readonly timeoutMs: number) { + super(`native session provider recovery timed out after ${timeoutMs}ms`); + this.name = "NativeProviderRecoveryDeadlineError"; + } +} + +export function isNativeProviderRecoveryDeadlineError(error: unknown): error is NativeProviderRecoveryDeadlineError { + return error instanceof Error && "code" in error && error.code === "native_provider_recovery_deadline"; +} + /** An authenticated, exactly bound runner event failed permanent integrity checks. */ export class NativeSessionProtocolIntegrityError extends Error { readonly code = "native_event_replay_conflict"; diff --git a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts index 85072adf8f5..7b9802046ba 100644 --- a/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts +++ b/packages/paperclip-runner/src/drivers/codex/app-server-transport.ts @@ -83,6 +83,8 @@ export interface CodexAppServerTransport { release?(): Promise; /** Relinquish controller authority while leaving durable runner work alive. */ detachControllerForRestart?(): Promise; + /** True when this transport recovers a session only to stop it. */ + recoversForStop?(): boolean; processInfo?(): CodexTransportProcessInfo; beginRunAttachment?(input: { runId: string }): boolean; attachRun?(input: { diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts index 66b3d7a96ba..f5fa26a6a28 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts @@ -19,7 +19,7 @@ import type { PersistedHarnessSession, PersistedHarnessTurnTerminal, } from "../../contracts/harness-driver.js"; -import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; +import { NativeSessionProtocolIntegrityError, isNativeProviderRecoveryDeadlineError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; import { HarnessReconciliationError, isRunnerResumeStateUnavailable, PROVIDER_SIDE_EFFECT_PENDING, PROVIDER_STATE_INCOMPATIBLE, runnerRecoveryFailureClass } from "../../contracts/harness-driver.js"; import { CODEX_CODEX_PROTOCOL_VERSION, @@ -117,7 +117,7 @@ function bootstrapCancellation( // until its pending RPCs have been rejected and TERM/KILL cleanup has // completed. The cancellation reason remains authoritative if cleanup // itself reports a secondary failure. - void close().then( + void close(isNativeProviderRecoveryDeadlineError(signal?.reason) && transport.recoversForStop?.() !== true).then( () => rejectAborted(signal?.reason ?? new Error("Codex bootstrap aborted")), () => @@ -472,9 +472,9 @@ export class CodexAppServerDriver implements HarnessDriver { // A recovery whose Runner command timed out or lost its transport detaches // from the Runner, as a restart does, for the same run's next // reattachment: the Runner may still own an in-flight turn, and stopping - // or killing it loses the turn. An abort (a Stop or the recovery - // deadline), identity refusals, other recovery failures and protocol - // integrity faults still close and stop it. + // or killing it loses the turn. The provider recovery deadline detaches + // the same way. A Stop, identity refusals, other recovery failures and + // protocol integrity faults still close and stop it. const cancellation = bootstrapCancellation(transport, options.signal); try { await cancellation.wait(this.#persistProcessOwnership(transport)); diff --git a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts index 133d8005941..9e99cc0fd06 100644 --- a/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts +++ b/packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts @@ -42,7 +42,7 @@ import { type PrpEvent, type PrpStructuredRunResult, } from "./codex-app-server-driver.test-support.js"; -import { NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; +import { NativeProviderRecoveryDeadlineError, NativeSessionProtocolIntegrityError, nativeRestartInterruptedTurnId } from "../../contracts/native-session-backend.js"; import { canonicalJson } from "./codex-driver-values.js"; function cursorProviderIdentity(mode: unknown) { @@ -290,14 +290,36 @@ describe("Codex app-server Codex driver", () => { expect(recoveryTransport.calls.some(call => ["turn/start", "thread/start", "turn/interrupt"].includes(call.method))).toBe(false); }); - it("still stops the Runner when an abort (a Stop or the recovery deadline) ends a hung read", async () => { + async function abortHungRead(reason: Error, recoversForStop = false) { const controller = new AbortController(); - const { recovery, close, detach } = await recoverWith(method => { + const { recovery, close, detach, recoveryTransport } = await recoverWith(method => { if (method !== "thread/read") return null; - setTimeout(() => controller.abort(new Error("Stop requested")), 0); + setTimeout(() => controller.abort(reason), 0); return new Promise(() => {}); }, controller.signal); - await expect(recovery).rejects.toThrow("Stop requested"); + Object.assign(recoveryTransport, { recoversForStop: () => recoversForStop }); + return { outcome: await recovery.then(() => null, (error: unknown) => error), close, detach }; + } + + it("still stops the Runner when a Stop ends a hung read", async () => { + const { outcome, close, detach } = await abortHungRead(new Error("Stop requested")); + expect(outcome).toEqual(new Error("Stop requested")); + expect(detach).not.toHaveBeenCalled(); + expect(close).toHaveBeenCalledTimes(1); + expect(close.mock.calls[0]).toEqual([]); + }); + + it("detaches and keeps the Runner when the provider recovery deadline ends a hung read", async () => { + const { outcome, close, detach } = await abortHungRead(new NativeProviderRecoveryDeadlineError(900_000)); + expect(outcome).toBeInstanceOf(NativeProviderRecoveryDeadlineError); + expect(String(outcome)).toContain("native session provider recovery timed out after 900000ms"); + expect(detach).toHaveBeenCalledTimes(1); + expect(close).not.toHaveBeenCalled(); + }); + + it("still stops the Runner when the provider recovery deadline ends a Stop's cleanup recovery", async () => { + const { outcome, close, detach } = await abortHungRead(new NativeProviderRecoveryDeadlineError(900_000), true); + expect(outcome).toBeInstanceOf(NativeProviderRecoveryDeadlineError); expect(detach).not.toHaveBeenCalled(); expect(close).toHaveBeenCalledTimes(1); expect(close.mock.calls[0]).toEqual([]); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts index 10dc17e21a3..275a0dbe1ca 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport-remote-retention.test.ts @@ -4,11 +4,12 @@ import { createServer } from "node:http"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { expect, it, vi } from "vitest"; -import { NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; +import { NativeProviderRecoveryDeadlineError, NativeSessionCloseUnrecoverableError } from "../contracts/native-session-backend.js"; import type * as durableControlPlane from "../control-plane/durable-prp-control-plane.js"; import type { DurablePrpControlPlane } from "../control-plane/durable-prp-control-plane.js"; -import { codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; +import { CodexAppServerDriver, codexSemanticToolSpecs } from "../drivers/codex/codex-app-server-driver.js"; import { createCapabilityRunnerdCodexTransport } from "./runnerd-codex-transport.js"; +import { createCodexTaskEnvelope } from "../contracts/codex.js"; import { defaultCapabilityRunnerdBinary, expectTurnStarted, fakeCodex, fakeCodexArgs } from "../../test/support/runnerd-transport.js"; function processState(pid: number): "alive" | "gone" { @@ -297,3 +298,130 @@ it("continues the same provider thread from a fresh Runner over the journal of a await rm(root, { recursive: true, force: true }); } }, 60_000); + +async function recoveryEndedWhileAdoptedRunnerHangs(abortReason: Error, recoveryPurpose?: "stop_cleanup") { + const root = await mkdtemp(join(tmpdir(), "runnerd-remote-retention-deadline-")); + const server = createServer(); + let authority: DurablePrpControlPlane | null = null; + server.on("upgrade", (request, socket, head) => { + if (!authority) { socket.destroy(); return; } + authority.handleUpgrade(request, socket, "/runner", head); + }); + await new Promise(resolveListen => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing deadline test listener"); + const registration = async (next: DurablePrpControlPlane) => { + authority = next; + return { connectUrl: `ws://127.0.0.1:${address.port}/runner`, release: () => { if (authority === next) authority = null; } }; + }; + const children: ReturnType[] = []; + const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore", detached: true }); + children.push(child); + return { + child, + completion: new Promise((resolveExit, rejectExit) => { + child.once("error", rejectExit); + child.once("exit", (code, exitSignal) => resolveExit({ code, signal: exitSignal, stdout: "", stderr: "" })); + }), + isExitConfirmed: () => child.exitCode !== null || child.signalCode !== null, + }; + }); + const identity = { + runnerInstanceId: "runner-deadline", environmentLeaseId: "lease-deadline", runId: "run-deadline", + normalizedSessionId: "session-deadline", turnId: "turn-deadline", itemId: "item-deadline", + }; + const shared = { + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(root, "--hold-first-durable-turn"), stateDirectory: root, prpIdentity: identity, + runnerReconnectGraceMs: 5_000, runnerProcessReconnectGraceMs: null, controlPlaneRegistration: registration, + }; + const first = createCapabilityRunnerdCodexTransport({ ...shared, runnerProcessLauncher: launch, closeGraceMs: 600 }); + let adopted: ReturnType | undefined; + let providerPid: number | null = null; + try { + const opened = await first.transport.request("thread/start", { cwd: root, dynamicTools: codexSemanticToolSpecs() }); + await first.transport.request("turn/start", { input: [{ type: "text", text: "Keep this task active." }] }); + for await (const notification of first.transport.notifications()) { + if (notification.method === "turn/started") break; + } + const runnerPid = children[0]!.pid!; + providerPid = first.evidence().codexPid; + signal(runnerPid, "SIGSTOP"); + await expect(first.transport.close("native session execution complete", { retainUnsettledRemoteRunner: true })) + .rejects.toBeInstanceOf(NativeSessionCloseUnrecoverableError); + signal(runnerPid, "SIGCONT"); + const thread = opened.thread as Record; + adopted = createCapabilityRunnerdCodexTransport({ + ...shared, + ...(recoveryPurpose ? { recoveryPurpose } : {}), + resumeDynamicTools: codexSemanticToolSpecs(), + resumeProviderSession: { driverSessionId: String(thread.id), providerSessionId: String(thread.sessionId) }, + runnerProcessLauncher: vi.fn(() => { throw new Error("Must adopt the existing runner"); }), + adoptExistingRunner: { + pid: runnerPid, processGroupId: runnerPid, startedAt: new Date().toISOString(), + isAlive: () => processState(runnerPid) === "alive", + signal: name => { try { process.kill(-runnerPid, name); return true; } catch { return false; } }, + }, + }); + adopted.transport.setServerRequestHandler(async () => ({ success: true, contentItems: [] })); + const driver = new CodexAppServerDriver({ + taskEnvelope: createCodexTaskEnvelope({ objective: "Keep this task active.", contractRevision: "1", criteria: [{ id: "objective", requirement: "Deliver the result." }], constraints: [] }), + transportFactory: () => adopted!.transport, + }); + const controller = new AbortController(); + const recovery = driver.recoverSession({ + driverKind: "codex", driverSessionId: String(thread.id), providerSessionId: String(thread.sessionId), + runId: identity.runId, normalizedSessionId: identity.normalizedSessionId, + }, { signal: controller.signal }).then(() => null, (error: unknown) => error); + const diagnostics = () => adopted!.evidence().diagnostics.join("\n"); + await vi.waitFor(() => expect(diagnostics()).toMatch(/adopted runner \d+ authenticated/), { timeout: 20_000, interval: 5 }); + signal(runnerPid, "SIGSTOP"); + await new Promise(resolveWait => setTimeout(resolveWait, 2_000)); + const abortedAt = Date.now(); + controller.abort(abortReason); + const outcome = await recovery; + const settledMs = Date.now() - abortedAt; + const watchUntil = Date.now() + 15_000; + while (processState(runnerPid) === "alive" && Date.now() < watchUntil) { + await new Promise(resolveWait => setTimeout(resolveWait, 250)); + } + const runnerState = processState(runnerPid); + signal(runnerPid, "SIGCONT"); + const pendingTerminalCommands = authority + ? (authority as DurablePrpControlPlane).store.state.commands + .filter(command => command.status === "pending" && ["turn.stop", "runner.suspend", "runner.shutdown"].includes(command.type)) + .map(command => command.type) + : []; + const providerState = providerPid === null ? "gone" : processState(providerPid); + return { outcome, runnerState, providerState, settledMs, pendingTerminalCommands }; + } finally { + await adopted?.transport.detachControllerForRestart?.().catch(() => undefined); + for (const child of children) { signal(child.pid, "SIGCONT"); signal(child.pid ? -child.pid : null, "SIGKILL"); } + signal(providerPid, "SIGKILL"); + server.closeAllConnections(); + if (server.listening) await new Promise(resolveClose => server.close(() => resolveClose())); + await rm(root, { recursive: true, force: true }); + } +} + +it("keeps an adopted Runner and its turn alive when the provider recovery deadline ends a recovery it never answered", async () => { + const result = await recoveryEndedWhileAdoptedRunnerHangs(new NativeProviderRecoveryDeadlineError(900_000)); + expect(result.outcome).toBeInstanceOf(NativeProviderRecoveryDeadlineError); + expect(result.runnerState).toBe("alive"); + expect(result.pendingTerminalCommands).toEqual([]); + expect(result.settledMs).toBeLessThan(5_000); +}, 60_000); + +it("still stops an adopted Runner when a Stop ends a recovery it never answered", async () => { + const result = await recoveryEndedWhileAdoptedRunnerHangs(new Error("Stop requested")); + expect(String(result.outcome)).toContain("Stop requested"); + expect(result.runnerState).toBe("gone"); +}, 60_000); + +it("still stops the adopted Runner and its provider when the provider recovery deadline ends a Stop's cleanup recovery", async () => { + const result = await recoveryEndedWhileAdoptedRunnerHangs(new NativeProviderRecoveryDeadlineError(900_000), "stop_cleanup"); + expect(result.outcome).toBeInstanceOf(NativeProviderRecoveryDeadlineError); + expect(result.runnerState).toBe("gone"); + expect(result.providerState).toBe("gone"); +}, 60_000); diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index d24560d5cbe..53dd42c4053 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -4374,6 +4374,10 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { * neither is required. A remote same-run bootstrap retry may then launch a * new runner: the old one is gone, and its provider never opened a thread. */ + recoversForStop(): boolean { + return this.options.recoveryPurpose === "stop_cleanup"; + } + runnerExitConfirmed(): boolean { return this.#closeFinished && !this.#startupComplete && (this.#providerIdentity === null || this.#openingCommandUnanswered()) diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index 3496497b61d..2f930321671 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -26,6 +26,7 @@ import { type HarnessThreadGoal, } from "./contracts/harness-driver.js"; import { + NativeProviderRecoveryDeadlineError, NativeProviderSideEffectPendingError, NativeProviderStateIncompatibleError, NativeProviderTerminalFailure, @@ -2548,6 +2549,7 @@ export async function executeNativeSession( ? await runAbortableOperationWithin({ timeoutMs: recoveryTimeoutMs, timeoutMessage: `native session provider recovery timed out after ${recoveryTimeoutMs}ms`, + timeoutError: () => new NativeProviderRecoveryDeadlineError(recoveryTimeoutMs), operation: (signal) => options.backend.recoverSession!(providerRecoveryCheckpoint, { signal, diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index b8a49c3cb1e..56fd255e96d 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1209,6 +1209,7 @@ describe("remote controller restart adoption", () => { "native_session_recovery_failed: HarnessReconciliationError: codex_history_read_failed: thread/turns/list requires supported paginated history; Error: provider_transport_failed: PRP command session.snapshot timed out (runner connected; pending ahead 0; last runner event seq 338 76s ago)", "native_session_recovery_failed: HarnessCapabilityUnavailableError: read is unavailable: Error: runnerd exited while waiting for session.snapshot", "native_session_recovery_failed: Error: runner_ingress_unavailable: PRP command runner.drain timed out (runner connected; pending ahead 0; last runner event seq 93 391s ago)", + "native session provider recovery timed out after 900000ms", ])("recognizes the staging recovery read that got no answer: %s", message => { expect(nativeRecoveryCommandUnanswered(new Error(message))).toBe(true); expect(nativeRecoveryCommandUnanswered(new Error("wrapped", { cause: new Error(message) }))).toBe(true); @@ -1219,6 +1220,8 @@ describe("remote controller restart adoption", () => { "native_session_recovery_failed: provider resumed a different session", "native_session_recovery_failed: provider_transport_failed: PRP command session.snapshot timed out; native_adopted_provider_identity_mismatch", "native_session_recovery_failed: restart interruption history is incomplete", + "native session recovery replay timed out after 900000ms", + "native session provider recovery timed out after 900000ms; native_adopted_provider_identity_mismatch", ])("does not replace a Runner for another failure: %s", message => { expect(nativeRecoveryCommandUnanswered(new Error(message))).toBe(false); }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index a00299d9cd6..428077c8025 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -8895,6 +8895,8 @@ export function nativeRecoveryCommandUnanswered(error: unknown, depth = 0): bool if (message.startsWith("native_session_recovery_failed:") && /PRP command [\w.]+ timed out|runnerd exited while waiting for [\w.]+/.test(message) && !/mismatch|different|identity|authority|integrity|incompatible/i.test(message)) return true; + if (value.code === "native_provider_recovery_deadline" + || /^native session provider recovery timed out after \d+ms$/.test(message)) return true; return nativeRecoveryCommandUnanswered(value.cause, depth + 1); }