diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 1df952eea59..c59bab4b3f6 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1532,8 +1532,11 @@ Every continuation carries the triggering request, ordered user direction, inter ### Remote runner process evidence A failed remote health request does not prove that runnerd stopped. Read-only -process checks retry transient transport failures up to three times within one -30-second budget. Missing or changed identity, an unknown process generation, +process checks retry transient transport failures within one 30-second budget, +with backoff from 250 milliseconds to two seconds between failed reads. Fast +failures do not exhaust that budget after three attempts. Each read uses at +most ten seconds of the remaining budget. Signals still have one attempt when +their acknowledgement is lost. Missing or changed identity, an unknown process generation, and exhausted health reads retain uncertainty. They do not authorize a signal, replacement launch, or provider replay. Confirmed predecessor exit permits the transport to attempt recovery within its existing reconnect deadline. Repeated diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index e18533217d4..0624f02976e 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -17,7 +17,8 @@ point, and still requires the caller's exact stopped-source and workspace fences Provider transfer tests do not establish task admission or agent authority. Remote process probes remain server-owned. Confirmed exit requires `ESRCH` or a -zombie with the saved generation. Unknown adopted-process ownership rejects the +zombie with the saved generation. Read-only transport retries use the existing +shared deadline with backoff; signal requests remain single-attempt. Unknown adopted-process ownership rejects the probe and grants no permission to replace a runner, signal a different process, or replay work. Launch requires a working Node probe before detachment. diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 00983d8aeb8..071404c8b83 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -287,8 +287,10 @@ historical provider outage or successful hosted task recovery. Active remote process supervision distinguishes a failed health read from a confirmed exit. The [executor tests](../../server/src/services/native-runtime/native-session-executor.test.ts) -exercise timeout recovery with a real listening process, unchanged ownership, -and rejected identity evidence. The Runner restart tests require positive exit +exercise timeout and repeated gateway-failure recovery with a real listening +process, unchanged ownership, and rejected identity evidence. Read-only probes +use their shared deadline with bounded backoff; fast failures do not exhaust a +three-attempt limit. Signal requests remain single-attempt. The Runner restart tests require positive exit evidence before replacement. The executor regressions also retire a real child during its metadata read, keep unknown adopted-process ownership held, and reject launch before detachment when Node is unavailable. The diagnostic tests cover diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index a6f6ff86327..b7775c11fda 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -692,6 +692,8 @@ describe("remote controller restart adoption", () => { remoteRunnerRecovery: { stage: "identity_marker", reason } }); }); it.each(["timeout", "transport"] as const)("records a marker %s without retaining transport text", async failure => { + vi.useFakeTimers(); + onTestFinished(() => { vi.useRealTimers(); }); const { target, execute } = fixture(); const transportError = new Error("private bridge response"); const original = execute.getMockImplementation()!; @@ -700,7 +702,9 @@ describe("remote controller restart adoption", () => { if (failure === "transport") throw transportError; return { stdout: "", stderr: "private bridge response", exitCode: -1, timedOut: true }; }); - const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error); + const outcome = verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error); + await vi.advanceTimersByTimeAsync(30_000); + const error = await outcome; const diagnostic = readRemoteRunnerRecoveryError(error); expect(diagnostic).toEqual({ stage: "identity_marker", reason: failure === "timeout" ? "probe_timeout" : "probe_transport_failed" }); expect(JSON.stringify(diagnostic)).not.toContain("private"); @@ -752,6 +756,8 @@ describe("remote controller restart adoption", () => { } finally { await rm(root, { recursive: true, force: true }); } }); it.each(["liveness", "timeout", "transport"] as const)("retains the reason for a failed process %s probe", async failure => { + vi.useFakeTimers(); + onTestFinished(() => { vi.useRealTimers(); }); const { target, execute } = fixture(); const original = execute.getMockImplementation()!; const transportError = new Error("provider_transport_failed: private bridge response"); @@ -760,7 +766,9 @@ describe("remote controller restart adoption", () => { if (failure === "transport") throw transportError; return { stdout: "", stderr: "private probe output", exitCode: failure === "liveness" ? 45 : -1, timedOut: failure === "timeout" }; }); - const error = await verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error); + const outcome = verifyRemoteRunnerReattachment({ claim, target: target as never, identity, runId: "run", normalizedSessionId: "session" }).catch(error => error); + await vi.advanceTimersByTimeAsync(30_000); + const error = await outcome; const diagnostic = readRemoteRunnerRecoveryError(error); expect(diagnostic).toEqual({ stage: "process_verification", reason: failure === "liveness" ? "process_not_running" : failure === "timeout" ? "probe_timeout" : "probe_transport_failed" }); expect(JSON.stringify(diagnostic)).not.toContain("private"); @@ -1046,7 +1054,7 @@ fs.readFileSync = function(path, ...args) { } }, 15_000); - it("preserves a real listening process and its marker when a health RPC times out", async () => { + it.each(["timeout", "gateway"] as const)("preserves a real listening process and its marker through a %s health outage", async failure => { const root = await mkdtemp(join(tmpdir(), "runner-health-listener-")); const marker = join(root, "identity"); const fixture = join(root, "listener.cjs"); @@ -1054,6 +1062,7 @@ fs.readFileSync = function(path, ...args) { let child: ReturnType | undefined; let probes = 0; let healthyProbes = 0; + const failedProbes = failure === "gateway" ? 3 : 1; let releaseExitProbe!: () => void; const exitProbeReady = new Promise(resolve => { releaseExitProbe = resolve; }); let settled = false; @@ -1070,8 +1079,11 @@ fs.readFileSync = function(path, ...args) { await writeFile(marker, `${input.args![4]}\n${child.pid}\n2026-10-19T00:00:00.000Z\nrunner-health\n`); return ok; } - if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true }; - if (label === "paperclip-runner-monitor" && probes > 2) await exitProbeReady; + if (label === "paperclip-runner-monitor" && ++probes <= failedProbes) { + if (failure === "gateway") throw new Error("Request failed with status code 502"); + return { ...ok, exitCode: null, timedOut: true }; + } + if (label === "paperclip-runner-monitor" && probes > failedProbes + 1) await exitProbeReady; const observed = await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => { execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({ exitCode: error ? typeof error.code === "number" ? error.code : null : 0, @@ -1189,7 +1201,7 @@ fs.readFileSync = function(path, ...args) { expect(await failure).toEqual({ stage: "process_verification", reason: "probe_timeout" }); expect(handle.isExitConfirmed?.()).toBe(false); const probes = execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-monitor"); - expect(probes.map(([x]) => x.timeoutMs)).toEqual([10_000, 10_000, 9_500]); + expect(probes.map(([x]) => x.timeoutMs)).toEqual([10_000, 10_000, 9_250]); expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); } finally { vi.useRealTimers(); } }); diff --git a/server/src/services/native-runtime/remote-runner-recovery-probe.test.ts b/server/src/services/native-runtime/remote-runner-recovery-probe.test.ts index fe25cf3296d..d8c31383bc8 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-probe.test.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-probe.test.ts @@ -7,6 +7,52 @@ afterEach(() => vi.useRealTimers()); const timeout = { exitCode: null, signal: null, timedOut: true, stdout: "", stderr: "", pid: null, startedAt: "2026-10-15T00:00:00Z" }; +it.each([NaN, Infinity, -Infinity])("rejects a non-finite deadline (%s) before a probe", async deadline => { + const execute = vi.fn().mockResolvedValue({ ...timeout, timedOut: false, exitCode: 0 }); + const result = await readRemoteRunnerRecoveryProbe({ + runner: { execute }, request: { command: "sh" }, stage: "process_verification", + readOnly: true, deadline, + }).catch(readRemoteRunnerRecoveryError); + expect(result).toEqual({ stage: "process_verification", reason: "probe_timeout" }); + expect(execute).not.toHaveBeenCalled(); +}); + +it("uses the read deadline when a transport outage outlasts three fast probes", async () => { + vi.useFakeTimers(); + const unavailable = Object.assign(new Error("Request failed with status code 502"), { response: { status: 502 } }); + const healthy = { ...timeout, timedOut: false, exitCode: 0 }; + const execute = vi.fn() + .mockRejectedValueOnce(unavailable) + .mockRejectedValueOnce(unavailable) + .mockRejectedValueOnce(unavailable) + .mockResolvedValue(healthy); + const result = readRemoteRunnerRecoveryProbe({ + runner: { execute }, request: { command: "sh" }, stage: "process_verification", + readOnly: true, deadline: Date.now() + 30_000, + }).catch(error => error); + await vi.runAllTimersAsync(); + expect(await result).toBe(healthy); + expect(execute).toHaveBeenCalledTimes(4); +}); + +it("bounds fast transport failures by the existing deadline and retains the cause", async () => { + vi.useFakeTimers(); + const unavailable = new Error("sandbox unavailable"); + const execute = vi.fn().mockRejectedValue(unavailable); + const startedAt = Date.now(); + const result = readRemoteRunnerRecoveryProbe({ + runner: { execute }, request: { command: "sh" }, stage: "process_verification", + readOnly: true, deadline: startedAt + 2_000, + }).catch(error => error); + await vi.runAllTimersAsync(); + expect(await result).toBe(unavailable); + expect(readRemoteRunnerRecoveryError(unavailable)).toEqual({ stage: "process_verification", reason: "probe_transport_failed" }); + expect(execute.mock.calls.length).toBeGreaterThan(3); + expect(execute.mock.calls.length).toBeLessThan(9); + expect(Date.now() - startedAt).toBeLessThanOrEqual(2_000); + expect(execute.mock.calls.at(-1)?.[0].timeoutMs).toBeLessThanOrEqual(250); +}); + it("shares the remaining deadline across timed-out reads", async () => { vi.useFakeTimers(); const execute = vi.fn(async (request) => { diff --git a/server/src/services/native-runtime/remote-runner-recovery-probe.ts b/server/src/services/native-runtime/remote-runner-recovery-probe.ts index 7f48277f7e3..77486aa4496 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-probe.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-probe.ts @@ -17,9 +17,12 @@ export async function readRemoteRunnerRecoveryProbe(input: { let lastError: unknown = new RemoteRunnerRecoveryError({ stage: input.stage, reason: "probe_timeout", }); - for (let attempt = 0; attempt < (input.readOnly ? 3 : 1); attempt++) { + // A few fast gateway failures can exhaust an attempt count in under a + // second. Read-only observations share the caller's deadline instead; + // signals still have one attempt because their outcome may be unknown. + for (let attempt = 0; ; attempt++) { const remaining = input.deadline - Date.now(); - if (remaining <= 0) break; + if (!Number.isFinite(remaining) || remaining <= 0) break; try { const result = await input.runner.execute({ ...input.request, @@ -32,8 +35,9 @@ export async function readRemoteRunnerRecoveryProbe(input: { stage: input.stage, reason: "probe_transport_failed", }); } - if (!input.readOnly || attempt === 2 || input.deadline - Date.now() <= 250) break; - await new Promise(resolve => setTimeout(resolve, 250)); + const delayMs = Math.min(250 * 2 ** Math.min(attempt, 3), 2_000); + if (!input.readOnly || input.deadline - Date.now() <= delayMs) break; + await new Promise(resolve => setTimeout(resolve, delayMs)); } throw lastError; }