diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 5593fd17f38..1a631958412 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -1551,6 +1551,15 @@ its active readers before closing the manifest and publishes no success receipt. ### Warm sandbox continuity +An active remote runner's health probe is an observation, not an exit receipt. +Timed-out or rejected read-only probes retry within the existing 30-second +budget. A successful probe keeps the same runner and turn; no provider work is +replayed. Monitoring checks the saved process generation when available. +Only a matching ownership marker and confirmed process exit permit automatic +replacement. Missing or conflicting identity evidence, failed reads, and an +exhausted probe budget retain the existing ownership and recovery safeguards. +They must not launch a competing listener or overwrite the original marker. + Healthy native warm turns retain their runner and provider processes. After a controller crash, continuation can recover a completed remote session from its saved checkpoint and exact sandbox process identity. The prior controller must diff --git a/maps/agent-access-map/recovery.md b/maps/agent-access-map/recovery.md index a9feb5b4063..9eb68f7ef32 100644 --- a/maps/agent-access-map/recovery.md +++ b/maps/agent-access-map/recovery.md @@ -36,6 +36,11 @@ Older unmarked preparations do not qualify for this automatic path during an upgrade. The fencing marker comes from fresh server-owned identity creation; caller profile data cannot upgrade an existing preparation. +Remote process-health reads retry within a bounded budget. An unavailable probe +cannot authorize replacement or erase the original ownership marker. This is +internal supervision; it grants no new agent recovery action. Existing task, +company, process-identity and workspace-recovery checks remain required. + Evidence: `source-only` — [implementation](../../server/src/routes/issues.ts#L9478) at the reviewed commit above. diff --git a/maps/feature-map/recovery.md b/maps/feature-map/recovery.md index 7712f1eed95..677f40fe39f 100644 --- a/maps/feature-map/recovery.md +++ b/maps/feature-map/recovery.md @@ -110,6 +110,13 @@ provider restart. For that journey, interrupt a disposable deployment after workspace preparation and before its first executor claim, then confirm one provider turn starts on the original run through the normal admission gates. +Active remote process supervision distinguishes a failed health read from a +confirmed exit. The [executor tests](../../server/src/services/native-runtime/native-session-executor.test.ts) +exercise timeout recovery with a real listening process, unchanged ownership, +and rejected identity evidence. The Runner restart tests require positive exit +evidence before replacement. This coverage does not prove recovery of a +historical retained sandbox or behavior during a live provider outage. + The same database suite pauses an original preparation while recovery claims the remote run. It then verifies that the old preparation yields without opening a provider or changing the recovery owner, for both live and expired leases. diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts index 4d44db5c05b..b88959e1873 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts @@ -31,6 +31,7 @@ import { SemanticToolNotDispatchedError, inspectWarmRunTransition, spawnRunner, + waitForProcess, type RunnerProcessLaunchSpec, } from "./durable-prp-control-plane.js"; import type { DurableRecoveryIdentity } from "./prp-transport-types.js"; @@ -393,6 +394,29 @@ it.skipIf(process.platform === "win32")( }, ); +it("does not release or replace a remote runner until its exit is confirmed", async () => { + let exited = false; + const launch = vi.fn(() => ({ + child: { pid: 42, exitCode: null, kill: () => true }, + completion: Promise.resolve({ code: null, signal: null, stdout: "", stderr: "health probe unavailable" }), + isExitConfirmed: () => exited, + })); + const handle = spawnRunner({ + connection: { mode: "connect", connectUrl: "ws://127.0.0.1:43127" }, + stateDirectory: "/tmp/paperclip-runner-exit-test", identity, ticket: "bootstrap-ticket", + maxOutboxBytes: 256 * 1024, p0ReserveBytes: 64 * 1024, + runnerVersion: expectedRunnerVersion, runnerDigest: expectedRunnerDigest, + processLauncher: launch, + }); + expect(() => handle.restart!("replacement-ticket")).toThrow("runner_process_termination_unconfirmed"); + await expect(waitForProcess(handle)).rejects.toThrow("runner_process_termination_unconfirmed"); + expect(launch).toHaveBeenCalledTimes(1); + exited = true; + await expect(waitForProcess(handle)).resolves.toMatchObject({ code: null }); + handle.restart!("replacement-ticket"); + expect(launch).toHaveBeenCalledTimes(2); +}); + it("pins the ACPX launch profile in runner startup arguments and restarts", () => { const launches: RunnerProcessLaunchSpec[] = []; const handle = spawnRunner({ diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index c671e5b778e..d86aafdca7e 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -289,6 +289,7 @@ export interface RunnerProcessHandle { completion: Promise; processGroupId?: number | null; startedAt?: string; + isExitConfirmed?(): boolean; /** Relaunches the same immutable process specification with a fresh ticket. */ restart?(ticket: string): RunnerProcessHandle; } @@ -3654,7 +3655,12 @@ export function spawnRunner(options: { const environment = runnerEnvironment(options.ticket, options.identity.normalizedSessionId, options.environment); const withRestart = (handle: RunnerProcessHandle): RunnerProcessHandle => ({ ...handle, - restart: (ticket) => spawnRunner({ ...options, ticket }), + restart: (ticket) => { + if (handle.isExitConfirmed?.() === false) { + throw new Error("runner_process_termination_unconfirmed"); + } + return spawnRunner({ ...options, ticket }); + }, }); if (options.processLauncher !== undefined) { return withRestart( @@ -3739,7 +3745,7 @@ export async function waitForProcess( ): Promise { let timer: NodeJS.Timeout | undefined; try { - return await Promise.race([ + const result = await Promise.race([ handle.completion, new Promise((_resolve, reject) => { timer = setTimeout(() => { @@ -3760,6 +3766,10 @@ export async function waitForProcess( }, timeoutMs); }), ]); + if (handle.isExitConfirmed?.() === false) { + throw new Error("runner_process_termination_unconfirmed"); + } + return result; } finally { if (timer !== undefined) clearTimeout(timer); } diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index 445f6eebacf..2395aa7ac4f 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -288,6 +288,125 @@ it("replaces an owned v1 runner with fresh v2 authorization before warm attachme } }, 30_000); +it.each(["resolved", "rejected"] as const)( + "retains a live authenticated runner when remote supervision is %s without exit proof", + async outcome => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-uncertain-health-")); + const children: ReturnType[] = []; + const exits: Promise[] = []; + const probeError = new Error("remote health probe unavailable"); + let failObservation!: () => void; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(directory), + stateDirectory: directory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 1_000, + runnerProcessLauncher: spec => { + const child = spawn(spec.command, [...spec.args], { + cwd: spec.cwd, env: spec.environment, stdio: "ignore", + }); + children.push(child); + exits.push(new Promise(resolveExit => child.once("exit", () => resolveExit()))); + const completion = new Promise((resolveObservation, rejectObservation) => { + child.once("error", rejectObservation); + failObservation = () => { + if (outcome === "rejected") rejectObservation(probeError); + else resolveObservation({ code: null, signal: null, stdout: "", stderr: "remote health probe unavailable" }); + }; + }); + return { child, completion, isExitConfirmed: () => false }; + }, + }); + try { + await bundle.transport.request("thread/start", { cwd: directory }); + const notifications = (async () => { + for await (const _event of bundle.transport.notifications()) {} + })(); + const failure = notifications.catch(error => error as Error); + failObservation(); + const error = await failure; + expect(error).toMatchObject({ message: expect.stringContaining("runner_process_termination_unconfirmed") }); + if (outcome === "rejected") expect(error).toHaveProperty("cause", probeError); + expect(children).toHaveLength(1); + expect(bundle.transport.processInfo()).toMatchObject({ exited: false }); + expect(() => process.kill(children[0]!.pid!, 0)).not.toThrow(); + } finally { + const providerPid = bundle.evidence().codexPid; + if (providerPid) { + try { process.kill(providerPid, "SIGKILL"); } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + await bundle.transport.detachControllerForRestart!(); + for (const child of children) child.kill("SIGKILL"); + await Promise.all(exits); + await rm(directory, { recursive: true, force: true }); + } + }, + 15_000, +); + +it("preserves the cause when a replacement runner's health cannot be verified", async () => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-replacement-health-")); + const probeError = Object.assign(new Error("remote recovery process_verification: probe_timeout"), { + code: "runner_remote_process_identity_unavailable", + remoteRunnerRecovery: { stage: "process_verification", reason: "probe_timeout" }, + }); + let child: ReturnType | undefined; + let exited: Promise | undefined; + const launch = vi.fn((spec: durableControlPlane.RunnerProcessLaunchSpec) => { + if (child) return { + child: { pid: 42, exitCode: null, kill: () => false }, + completion: Promise.reject(probeError), + isExitConfirmed: () => false, + }; + child = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore" }); + const ownedChild = child; + exited = new Promise(resolveExit => ownedChild.once("exit", () => resolveExit())); + return { + child: ownedChild, + completion: new Promise((resolveExit, rejectExit) => { + ownedChild.once("error", rejectExit); + ownedChild.once("exit", (code, signal) => resolveExit({ code, signal, stdout: "", stderr: "" })); + }), + isExitConfirmed: () => ownedChild.exitCode !== null || ownedChild.signalCode !== null, + }; + }); + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(directory), stateDirectory: directory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 1_000, runnerProcessLauncher: launch, + }); + try { + await bundle.transport.request("thread/start", { cwd: directory }); + const failed = (async () => { + for await (const _event of bundle.transport.notifications()) {} + })().catch(error => error as Error); + child!.kill("SIGKILL"); + const error = await failed; + expect(error).toMatchObject({ + message: expect.stringContaining("runner_process_termination_unconfirmed"), cause: probeError, + }); + expect(error).toHaveProperty("message", expect.stringContaining("probe_timeout")); + expect(launch).toHaveBeenCalledTimes(2); + expect(bundle.transport.processInfo()).toMatchObject({ exited: false }); + } finally { + const providerPid = bundle.evidence().codexPid; + if (providerPid) { + try { process.kill(providerPid, "SIGKILL"); } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + await bundle.transport.detachControllerForRestart!(); + child?.kill("SIGKILL"); + await exited; + await rm(directory, { recursive: true, force: true }); + } +}, 15_000); + // The retained-settlement maintenance family ("settles only retained control // authority without starting another provider turn") lives in // runnerd-codex-transport-settlement.test.ts so vitest can run its ~160s of diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index e65db3b7ad4..4d76126b578 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -6864,7 +6864,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } async #runnerHasExited(): Promise { - if (this.#handle) return this.#handle.child.exitCode !== null; + if (this.#handle) return this.#handle.isExitConfirmed?.() ?? this.#handle.child.exitCode !== null; const adoptedRunner = this.options.adoptExistingRunner; if (!adoptedRunner) return true; try { @@ -6948,7 +6948,8 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { #watchRunner(handle: RunnerProcessHandle): void { void handle.completion.then( (result) => { - this.#evidence.runnerExited = true; + const exitConfirmed = handle.isExitConfirmed?.() ?? true; + this.#evidence.runnerExited = exitConfirmed; this.#evidence.runnerExitCode = result.code; this.#evidence.runnerSignal = result.signal as NodeJS.Signals | null; const detail = result.stderr.trim() || result.stdout.trim(); @@ -6961,6 +6962,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { // process completion; clean terminal exit is the expected lifecycle. this.#pumpEventsSafely(); const expectedPerTurnExit = + exitConfirmed && result.code === 0 && (this.options.lifecyclePolicy?.mode ?? "per_turn") === "per_turn" && (this.#core?.store.state.committedEvents.some( @@ -6968,19 +6970,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ) ?? false); if (expectedPerTurnExit) return; - const code = /provider_frame_too_large|stdout frame exceeded/i.test( - detail, - ) - ? "provider_frame_too_large" - : /runner_ingress_bind_conflict/i.test(detail) - ? "runner_ingress_bind_conflict" - : /transport_reconnect_grace_exceeded/i.test(detail) - ? "transport_reconnect_grace_exceeded" - : this.#startupComplete - ? "native_runner_process_exited" - : this.#startupFailureCode; + const code = !exitConfirmed + ? "runner_process_termination_unconfirmed" + : /provider_frame_too_large|stdout frame exceeded/i.test(detail) + ? "provider_frame_too_large" + : /runner_ingress_bind_conflict/i.test(detail) + ? "runner_ingress_bind_conflict" + : /transport_reconnect_grace_exceeded/i.test(detail) + ? "transport_reconnect_grace_exceeded" + : this.#startupComplete + ? "native_runner_process_exited" + : this.#startupFailureCode; if ( code === "native_runner_process_exited" && + exitConfirmed && this.options.runnerReconnectGraceMs !== undefined && handle.restart ) { @@ -6989,7 +6992,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { } this.#failTransport( new Error( - `${code}: runnerd exited unexpectedly${result.code === null ? "" : ` with code ${result.code}`}${detail ? `: ${detail.slice(-1_000)}` : ""}`, + `${code}: runnerd ${exitConfirmed ? "exited unexpectedly" : "process supervision ended"}${result.code === null ? "" : ` with code ${result.code}`}${detail ? `: ${detail.slice(-1_000)}` : ""}`, ), ); }, @@ -6998,6 +7001,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { return; if ( this.#startupComplete && + handle.isExitConfirmed?.() !== false && this.options.runnerReconnectGraceMs !== undefined && handle.restart ) { @@ -7007,9 +7011,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ); return; } + const code = handle.isExitConfirmed?.() === false + ? "runner_process_termination_unconfirmed" + : this.#startupComplete + ? "native_runner_process_exited" + : this.#startupFailureCode; this.#failTransport( new Error( - `${this.#startupComplete ? "native_runner_process_exited" : this.#startupFailureCode}: runnerd process failed: ${error instanceof Error ? error.message : String(error)}`, + `${code}: runnerd ${handle.isExitConfirmed?.() === false ? "process supervision" : "process"} failed: ${error instanceof Error ? error.message : String(error)}`, + { cause: error }, ), ); }, @@ -7073,15 +7083,20 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#publish(); let processSettled = false; + let observationError: unknown; + let observationDetail = ""; const completion = recoveredHandle.completion.then( (result) => { processSettled = true; - lastDetail = result.stderr.trim() || result.stdout.trim(); + observationDetail = result.stderr.trim() || result.stdout.trim(); + lastDetail = observationDetail; return false; }, (error) => { processSettled = true; - lastDetail = error instanceof Error ? error.message : String(error); + observationError = error; + observationDetail = error instanceof Error ? error.message : String(error); + lastDetail = observationDetail; return false; }, ); @@ -7107,6 +7122,15 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { this.#watchRunner(recoveredHandle); return; } + if (recoveredHandle.isExitConfirmed?.() === false) { + this.#failTransport( + new Error( + `runner_process_termination_unconfirmed: replacement runner health could not be verified${observationDetail ? `: ${observationDetail.slice(-1_000)}` : ""}`, + { cause: observationError }, + ), + ); + return; + } attempt += 1; } if (!this.#closed) { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index db4fba7f905..0c9a546214c 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -22,7 +22,7 @@ import { truncate, writeFile, } from "node:fs/promises"; -import { execFileSync } from "node:child_process"; +import { execFile, execFileSync, spawn } from "node:child_process"; import { tmpdir } from "node:os"; import { inspect } from "node:util"; import { join } from "node:path"; @@ -779,6 +779,143 @@ describe("remote runner launch fingerprint compatibility", () => { }); describe("remote runner process supervision", () => { + it("preserves a real listening process and its marker when a health RPC times out", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-health-listener-")); + const marker = join(root, "identity"); + const fixture = join(root, "listener.cjs"); + await writeFile(fixture, 'require("node:net").createServer().listen(0,"127.0.0.1",()=>process.stdout.write("ready\\n"));'); + let child: ReturnType | undefined; + let probes = 0; + let settled = false; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { command: string; args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { + child = spawn(process.execPath, [fixture, "--runner-id", "runner-health"], { stdio: ["ignore", "pipe", "pipe"] }); + await new Promise((resolve, reject) => { + child!.stdout!.once("data", () => resolve()); + child!.once("error", reject); + child!.once("exit", code => reject(new Error(`listener fixture exited before readiness: ${code}`))); + }); + await writeFile(marker, `${input.args![4]}\n${child.pid}\n2026-10-19T00:00:00.000Z\nrunner-health\n`); + return ok; + } + if (label === "paperclip-runner-monitor" && ++probes === 1) return { ...ok, exitCode: null, timedOut: true }; + return await new Promise<{ exitCode: number | null; signal: null; timedOut: boolean; stdout: string; stderr: string }>(resolve => { + execFile(input.command, input.args ?? [], { timeout: 2_000 }, (error, stdout, stderr) => resolve({ + exitCode: error ? typeof error.code === "number" ? error.code : null : 0, + signal: null, timedOut: !!error?.killed, stdout, stderr, + })); + }); + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: root }, + runner: { execute } as never, remoteBinary: fixture, processIdentityPath: marker, + stateDirectory: root, diagnosticsDirectory: join(root, "diagnostics"), runnerInstanceId: "runner-health", + })({ command: fixture, args: [], cwd: root, environment: {} }); + void handle.completion.then(() => { settled = true; }, () => { settled = true; }); + try { + await vi.waitFor(() => expect(probes).toBeGreaterThanOrEqual(2), { timeout: 5_000 }); + expect(settled).toBe(false); + expect(handle.isExitConfirmed?.()).toBe(false); + expect(() => process.kill(child!.pid!, 0)).not.toThrow(); + expect((await readFile(marker, "utf8")).split("\n")[1]).toBe(String(child!.pid)); + expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); + child!.kill("SIGTERM"); + await handle.completion; + expect(handle.isExitConfirmed?.()).toBe(true); + } finally { + child?.kill("SIGKILL"); + await handle.completion.catch(() => undefined); + await rm(root, { recursive: true, force: true }); + } + }); + + it.each([ + [41, "marker_missing"], [44, "process_identity_changed"], [47, "process_generation_changed"], [1, "probe_failed"], + ] as const)("does not authorize replacement from health probe exit %s", async (exitCode, reason) => { + let nonce = ""; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; } + if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + return { ...ok, exitCode }; + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + expect(await handle.completion.catch(readRemoteRunnerRecoveryError)).toEqual({ stage: "process_verification", reason }); + expect(handle.isExitConfirmed?.()).toBe(false); + }); + + it.each(["timeout", "transport"])("keeps the same runner through a transient %s health probe", async failure => { + vi.useFakeTimers(); + let nonce = ""; + let probes = 0; + let settled = false; + const result = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return result; } + if (label === "paperclip-runner-process-identity") return { ...result, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + if (label === "paperclip-runner-monitor") { + probes++; + if (probes === 1) { + if (failure === "transport") throw new Error("sandbox request timed out"); + return { ...result, timedOut: true, exitCode: null }; + } + return { ...result, exitCode: probes === 2 ? 0 : 45 }; + } + return { ...result, exitCode: 1 }; + }); + const launcher = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + }); + const handle = launcher({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + void handle.completion.then(() => { settled = true; }, () => { settled = true; }); + try { + await vi.advanceTimersByTimeAsync(300); + expect(settled).toBe(false); + expect(probes).toBe(2); + expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); + expect(execute.mock.calls.some(([x]) => x.args?.[2] === "paperclip-runner-diagnostics")).toBe(false); + await vi.advanceTimersByTimeAsync(1_000); + await expect(handle.completion).resolves.toMatchObject({ code: null }); + } finally { vi.useRealTimers(); } + }); + + it("bounds failed health reads without confirming exit or replacing the marker", async () => { + vi.useFakeTimers(); + let nonce = ""; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[]; timeoutMs?: number }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; } + if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + await new Promise(resolve => setTimeout(resolve, input.timeoutMs)); + return { ...ok, exitCode: null, timedOut: true }; + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {} }); + const failure = handle.completion.catch(readRemoteRunnerRecoveryError); + try { + await vi.advanceTimersByTimeAsync(30_000); + expect(await failure).toEqual({ stage: "process_verification", reason: "probe_timeout" }); + expect(handle.isExitConfirmed?.()).toBe(false); + const probes = execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-monitor"); + expect(probes.map(([x]) => x.timeoutMs)).toEqual([10_000, 10_000, 9_500]); + expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); + } finally { vi.useRealTimers(); } + }); + it.each(["delivered", "sandbox_missing", "logging_failed"] as const)( "detaches runnerd and contains asynchronous signal failures (%s)", async (signalOutcome) => { let launchNonce = ""; @@ -812,7 +949,7 @@ describe("remote runner process supervision", () => { } if (label === "paperclip-runner-monitor") { return { - exitCode: 3, + exitCode: 45, signal: null, timedOut: false, stdout: "", diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 4f332c53aef..0684682854e 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11013,6 +11013,7 @@ export function createRemoteRunnerProcessLauncher(input: { }): (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle { const runner = input.runner; return (spec) => { + let exitConfirmed = false; let launchedIdentity: { nonce: string; pid: number; @@ -11130,7 +11131,7 @@ export function createRemoteRunnerProcessLauncher(input: { : "runner_remote_process_launch_failed", ); } - let identity: { pid: number; startedAt: string }; + let identity: Awaited>; try { identity = await waitForRemoteRunnerProcessIdentity({ runner, @@ -11177,19 +11178,27 @@ export function createRemoteRunnerProcessLauncher(input: { }); while (true) { - const observed = await runner.execute({ - command: "sh", - args: [ - "-c", - REMOTE_RUNNER_IDENTITY_CHECK_SCRIPT, - "paperclip-runner-monitor", - input.processIdentityPath, - identityNonce, - input.runnerInstanceId, - String(identity.pid), - ], - bypassSession: true, - timeoutMs: 10_000, + const observed = await readRemoteRunnerRecoveryProbe({ + runner, + stage: "process_verification", + deadline: Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS, + readOnly: true, + request: { + command: "sh", + args: [ + "-c", + REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT + (identity.processStartFingerprint + ? `; if ${REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT}; then :; else exit 46; fi; test "$process_fingerprint" = "$5" || exit 47` + : ""), + "paperclip-runner-monitor", + input.processIdentityPath, + identityNonce, + input.runnerInstanceId, + String(identity.pid), + ...(identity.processStartFingerprint ? [identity.processStartFingerprint] : []), + ], + bypassSession: true, + }, }); if (observed.exitCode === 0 && !observed.timedOut) { await new Promise((resolve) => @@ -11197,8 +11206,14 @@ export function createRemoteRunnerProcessLauncher(input: { ); continue; } + if (observed.exitCode !== 45) { + throw new RemoteRunnerRecoveryError({ + stage: "process_verification", + reason: remoteRecoveryProbeReason("process_verification", observed.exitCode, observed.timedOut), + }); + } + exitConfirmed = true; child.exitCode = null; - const identityMismatch = observed.exitCode === 4; const diagnostic = await runner .execute({ command: "sh", @@ -11241,13 +11256,11 @@ export function createRemoteRunnerProcessLauncher(input: { code: null, signal: null, stdout: "", - stderr: identityMismatch - ? "runner_remote_process_identity_mismatch" - : diagnosticTail || stateSummary, + stderr: diagnosticTail || stateSummary, }; } })(); - return { child, completion }; + return { child, completion, isExitConfirmed: () => exitConfirmed }; }; }