diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 936eabbf687..7f530b8a1f4 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -538,6 +538,67 @@ it("retries a replacement whose staging failed before any process launched", asy } }, 15_000); +it("fails instead of relaunching a runner that keeps exiting right after it is restored", async () => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-restore-crash-loop-")); + const crash = "paperclip-runnerd: redacted display activity exceeds its schema bounds"; + const children: ReturnType[] = []; + const exits: Promise[] = []; + const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => { + const child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore" }); + children.push(child); + exits.push(new Promise(resolveExit => child.once("exit", () => resolveExit()))); + return { + child, + completion: new Promise((resolveExit, rejectExit) => { + child.once("error", rejectExit); + child.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: crash })); + }), + isExitConfirmed: () => child.exitCode !== null || child.signalCode !== null, + }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(directory), stateDirectory: directory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 5_000, runnerProcessLauncher: launch, + }); + const restored = () => bundle.evidence().diagnostics + .filter(line => line === "runner process restored its durable PRP session").length; + try { + await bundle.transport.request("thread/start", { cwd: directory }); + children[0]!.kill("SIGKILL"); + for (let restore = 1; restore <= 3; restore += 1) { + await vi.waitFor(() => expect(restored()).toBe(restore), { timeout: 4_000 }); + children[restore]!.kill("SIGKILL"); + } + await vi.waitFor( + () => expect(bundle.evidence().diagnostics).toContain( + `native_runner_process_exited: runnerd exited 3 times within 60000ms of being restored: ${crash}`, + ), + { timeout: 4_000 }, + ); + await new Promise(resolveWait => setTimeout(resolveWait, 500)); + expect(launch).toHaveBeenCalledTimes(4); + expect(restored()).toBe(3); + } finally { + const providerPid = bundle.evidence().codexPid; + if (providerPid) { try { process.kill(providerPid, "SIGKILL"); } catch {} } + await bundle.transport.detachControllerForRestart!(); + for (const child of children) child.kill("SIGKILL"); + await Promise.all(exits); + await rm(directory, { recursive: true, force: true }); + } +}, 20_000); + +it("counts only runner exits that follow a restore within the stable window", () => { + const count = runnerdRecoveryInternals.unstableRunnerRestoreCount; + expect(count(0, null, 1_000)).toBe(0); + expect(count(0, 1_000, 1_000)).toBe(1); + expect(count(2, 1_000, 60_999)).toBe(3); + expect(count(2, 1_000, 61_000)).toBe(0); + expect(count(5, null, 1_000)).toBe(0); +}); + it.each(["alive", "pending_liveness", "pending_registration"] as const)( "bounds adopted runner authentication at its exact deadline with %s evidence", async (mode) => { diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index 7534e459948..fdf1760d57f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -117,6 +117,8 @@ const MAX_NOTIFICATION_COUNT = 2_048; const MAX_NOTIFICATION_BYTES = 4 * 1024 * 1024; const RUNNER_CLIENT_VERSION = "0.3.0"; const RUNNER_BOOTSTRAP_TICKET_TTL_MS = 60_000; +const RUNNER_RESTORE_STABLE_MS = 60_000; +const MAX_UNSTABLE_RUNNER_RESTORES = 3; const RUNNERD_MAX_OUTBOX_BYTES = 16 * 1024 * 1024; const RUNNERD_P0_RESERVE_BYTES = 1024 * 1024; @@ -4050,6 +4052,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { readonly #failureSignal: Promise; #rejectFailureSignal!: (error: Error) => void; #runnerRecoveryInProgress = false; + #lastRunnerRestoreAt: number | null = null; + #unstableRunnerRestores = 0; #startupComplete = false; readonly #authorityRunId: () => string | undefined; #startupFailureCode = "native_runner_process_exited"; @@ -8489,7 +8493,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.options.runnerReconnectGraceMs !== undefined && handle.restart ) { - void this.#recoverRunnerProcess(handle, detail); + this.#recoverOrFailRunnerProcess(handle, detail); return; } this.#failTransport( @@ -8507,7 +8511,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.options.runnerReconnectGraceMs !== undefined && handle.restart ) { - void this.#recoverRunnerProcess( + this.#recoverOrFailRunnerProcess( handle, error instanceof Error ? error.message : String(error), ); @@ -8528,6 +8532,23 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); } + #recoverOrFailRunnerProcess(handle: RunnerProcessHandle, detail: string): void { + this.#unstableRunnerRestores = unstableRunnerRestoreCount( + this.#unstableRunnerRestores, + this.#lastRunnerRestoreAt, + Date.now(), + ); + if (this.#unstableRunnerRestores >= MAX_UNSTABLE_RUNNER_RESTORES) { + this.#failTransport( + new Error( + `native_runner_process_exited: runnerd exited ${this.#unstableRunnerRestores} times within ${RUNNER_RESTORE_STABLE_MS}ms of being restored${detail ? `: ${detail.slice(-1_000)}` : ""}`, + ), + ); + return; + } + void this.#recoverRunnerProcess(handle, detail); + } + async #recoverRunnerProcess( failedHandle: RunnerProcessHandle, initialDetail: string, @@ -8622,6 +8643,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { })(); if (await Promise.race([completion, authenticated])) { this.#diagnostic("runner process restored its durable PRP session"); + this.#lastRunnerRestoreAt = Date.now(); this.#watchRunner(recoveredHandle); return; } @@ -9057,7 +9079,12 @@ export const runnerdLaunchProfileInternals = Object.freeze({ p0ReserveBytes: RUNNERD_P0_RESERVE_BYTES, }); +function unstableRunnerRestoreCount(previous: number, restoredAt: number | null, now: number): number { + return restoredAt !== null && now - restoredAt < RUNNER_RESTORE_STABLE_MS ? previous + 1 : 0; +} + export const runnerdRecoveryInternals = Object.freeze({ + unstableRunnerRestoreCount, projectCopilotRunGrant, completedMaintenanceTerminalReceipt, completedMaintenanceTerminalReplayMatches,