diff --git a/doc/execution-semantics.md b/doc/execution-semantics.md index 6eac6f41144..65ebcda93af 100644 --- a/doc/execution-semantics.md +++ b/doc/execution-semantics.md @@ -4814,7 +4814,13 @@ configuration from the same company, task, and bot binding. An active remote runner's health probe is an observation, not an exit receipt. Timed-out or rejected read-only probes retry within the existing 30-second budget. A successful probe keeps the same runner and turn; no provider work is -replayed. Monitoring checks the saved process generation when available. +replayed. A probe that times out or fails in transport is unknown, not an exit. +For example, a stalled environment plugin RPC times out after 40 seconds. While +the Runner holds its own authenticated connection to the controller, the monitor +keeps the same runner and turn and probes again. The bound is the remote runner +reconnect grace (7 minutes). When the Runner has no connection and the 30-second +budget is spent, or when the bound is spent, the monitor reports +`runner_process_termination_unconfirmed` and the run rests with Retry. Monitoring checks the saved process generation when available. Only a matching ownership marker and confirmed process exit permit automatic replacement. Missing or conflicting identity evidence, failed reads, and an exhausted probe budget retain the existing ownership and recovery safeguards. diff --git a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts index 3fc4438b0b2..3a737b70f1b 100644 --- a/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts +++ b/packages/paperclip-runner/src/control-plane/durable-prp-control-plane.ts @@ -646,6 +646,8 @@ export interface RunnerProcessLaunchSpec { args: readonly string[]; cwd: string; environment: NodeJS.ProcessEnv; + /** True while the launched Runner holds an authenticated connection to its control plane. */ + isRunnerConnected?: () => boolean; } function domainDigest(domain: string, parts: readonly Buffer[]): Buffer { @@ -4732,6 +4734,7 @@ export function spawnRunner(options: { }; environment?: NodeJS.ProcessEnv; processLauncher?: (spec: RunnerProcessLaunchSpec) => RunnerProcessHandle; + isRunnerConnected?: () => boolean; diagnosticsDirectory?: string; }): RunnerProcessHandle { for (const value of [options.maxRuntimeMs, options.maxLifetimeMs]) { @@ -4881,6 +4884,7 @@ export function spawnRunner(options: { if (options.processLauncher !== undefined) { return withRestart( options.processLauncher({ command, args, cwd: packageRoot, environment, + ...(options.isRunnerConnected ? { isRunnerConnected: options.isRunnerConnected } : {}), nativeRunner: { identity: options.identity, ticket: options.ticket, connection, stateDirectory: options.stateDirectory, runnerVersion: options.runnerVersion, runnerDigest: options.runnerDigest, diff --git a/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts b/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts index 3daecd15d60..bf361d3211d 100644 --- a/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts +++ b/packages/paperclip-runner/src/drivers/external/runnerd-external-driver.ts @@ -232,7 +232,7 @@ class RunnerdExternalSession implements HarnessSession { const ticket = core.issueBootstrapTicket(60_000); await core.flushCheckpoint(); this.#process = spawnRunner({ - processLauncher: o.runnerProcessLauncher, connectUrl: registration?.connectUrl ?? (registration?.connection ? undefined : core.connectUrl), + processLauncher: o.runnerProcessLauncher, isRunnerConnected: () => core.activeRunnerConnectionCount() > 0, connectUrl: registration?.connectUrl ?? (registration?.connection ? undefined : core.connectUrl), connection: registration?.connection, stateDirectory: runnerState, identity: o.identity, ticket, runnerBinaryPath: binary, runnerVersion: artifact.version, runnerDigest: artifact.digest, diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts index a9ce3e931f0..ee70d048216 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts @@ -372,6 +372,51 @@ it.each(["resolved", "rejected"] as const)( 15_000, ); +it("gives the process owner live evidence of the runner's own authenticated connection", async () => { + const directory = await mkdtemp(join(tmpdir(), "runnerd-connection-evidence-")); + const specs: durableControlPlane.RunnerProcessLaunchSpec[] = []; + let child: ReturnType | undefined; + let exited: Promise | undefined; + const bundle = createCapabilityRunnerdCodexTransport({ + runnerBinary: defaultCapabilityRunnerdBinary(), + codexCommand: fakeCodex, + codexArgs: fakeCodexArgs(directory), + stateDirectory: directory, + lifecyclePolicy: { mode: "warm", idleTimeoutMs: 60_000 }, + runnerReconnectGraceMs: 60_000, + runnerProcessLauncher: spec => { + specs.push(spec); + const ownedChild = spawn(spec.command, [...spec.args], { cwd: spec.cwd, env: spec.environment, stdio: "ignore" }); + child = ownedChild; + exited = new Promise(resolveExit => ownedChild.once("exit", () => resolveExit())); + return { + child: ownedChild, + completion: new Promise(() => {}), + isExitConfirmed: () => false, + }; + }, + }); + try { + await bundle.transport.request("thread/start", { cwd: directory }); + expect(specs).toHaveLength(1); + expect(specs[0]!.isRunnerConnected?.()).toBe(true); + child!.kill("SIGKILL"); + await exited; + await vi.waitFor(() => expect(specs[0]!.isRunnerConnected?.()).toBe(false), { timeout: 5_000 }); + } finally { + const providerPid = bundle.evidence().codexPid; + if (providerPid) { + try { process.kill(providerPid, "SIGKILL"); } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + } + await bundle.transport.detachControllerForRestart!(); + child?.kill("SIGKILL"); + await exited; + await rm(directory, { recursive: true, force: true }); + } +}, 15_000); + it.each([ "transport_reconnect_grace_exceeded", "transport reconnect grace exceeded; durable state is preserved", diff --git a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts index b55c9229682..2d33e6a8182 100644 --- a/packages/paperclip-runner/src/live/runnerd-codex-transport.ts +++ b/packages/paperclip-runner/src/live/runnerd-codex-transport.ts @@ -6258,6 +6258,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { environment: prepared.environment, diagnosticsDirectory: resolve(this.#root, "diagnostics"), processLauncher: this.options.runnerProcessLauncher, + isRunnerConnected: () => core.activeRunnerConnectionCount() > 0, }); this.#handle = handle; this.#watchRunner(handle); @@ -7132,6 +7133,7 @@ class DurablePrpCodexTransport implements CodexAppServerTransport { ), diagnosticsDirectory: resolve(this.#root, "diagnostics"), processLauncher: this.options.runnerProcessLauncher, + isRunnerConnected: () => core.activeRunnerConnectionCount() > 0, }); if (handle) { this.#handle = handle; diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 21d662aa422..26a0cbd008f 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -83,6 +83,7 @@ import { DatabaseSync } from "node:sqlite"; import { nativeSha256 } from "./canonical.js"; import { resolveRemoteProviderPackAuthority } from "./remote-provider-pack-authority.js"; import { REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, RemoteRunnerRecoveryError, readRemoteRunnerRecoveryError } from "./remote-runner-recovery-diagnostics.js"; +import { REMOTE_RUNNER_RECONNECT_GRACE_MS } from "./native-runner-recovery-contract.js"; import { agentDirectoryWorkingCopyService } from "../agent-directory-working-copies.js"; import { probeAgentDirectory } from "../agent-directory-probe.js"; import { AGENT_FILES_CONTRACT } from "../agent-file-store.js"; @@ -1608,6 +1609,97 @@ fs.readFileSync = function(path, ...args) { } finally { vi.useRealTimers(); } }); + it("keeps supervising a connected runner through plugin RPC timeouts until the probe answers", async () => { + vi.useFakeTimers(); + let nonce = ""; + let stalledProbes = 0; + let connected = true; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; } + if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + if (label === "paperclip-runner-monitor" && stalledProbes < 3) { + stalledProbes++; + await new Promise(resolve => setTimeout(resolve, 40_000)); + throw new Error('RPC call "environmentExecute" timed out after 40000ms'); + } + if (label === "paperclip-runner-monitor") return { ...ok, exitCode: connected ? 0 : 45 }; + return { ...ok, exitCode: 1 }; + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {}, isRunnerConnected: () => connected }); + let settled = false; + const outcome = handle.completion.then(() => { settled = true; }, error => { settled = true; return error as Error; }); + try { + await vi.advanceTimersByTimeAsync(130_000); + expect(settled ? readRemoteRunnerRecoveryError(await outcome) : "supervising").toBe("supervising"); + expect(stalledProbes).toBe(3); + expect(handle.isExitConfirmed?.()).toBe(false); + connected = false; + await vi.advanceTimersByTimeAsync(5_000); + expect(await outcome).toBeUndefined(); + expect(handle.isExitConfirmed?.()).toBe(true); + expect(execute.mock.calls.filter(([x]) => x.args?.[2] === "paperclip-runner-launch")).toHaveLength(1); + } finally { vi.useRealTimers(); } + }); + + it("fails closed without exit proof when the runner is also disconnected through plugin RPC timeouts", async () => { + vi.useFakeTimers(); + let nonce = ""; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; } + if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + await new Promise(resolve => setTimeout(resolve, 40_000)); + throw new Error('RPC call "environmentExecute" timed out after 40000ms'); + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {}, isRunnerConnected: () => false }); + const failure = handle.completion.catch(readRemoteRunnerRecoveryError); + try { + await vi.advanceTimersByTimeAsync(45_000); + expect(await failure).toEqual({ stage: "process_verification", reason: "probe_transport_failed" }); + expect(handle.isExitConfirmed?.()).toBe(false); + expect(execute.mock.calls.some(([x]) => x.args?.[2] === "paperclip-runner-diagnostics")).toBe(false); + } finally { vi.useRealTimers(); } + }); + + it("bounds supervision without exit proof when a connected runner's probes never answer", async () => { + vi.useFakeTimers(); + let nonce = ""; + const ok = { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "" }; + const execute = vi.fn(async (input: { args?: string[] }) => { + const label = input.args?.[2]; + if (label === "paperclip-runner-launch") { nonce = input.args![4]!; return ok; } + if (label === "paperclip-runner-process-identity") return { ...ok, stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\nrunner-remote\n` }; + await new Promise(resolve => setTimeout(resolve, 40_000)); + throw new Error('RPC call "environmentExecute" timed out after 40000ms'); + }); + const handle = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "sandbox", environmentId: "environment", leaseId: "lease", remoteCwd: "/workspace" }, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/identity", + stateDirectory: "/runtime", diagnosticsDirectory: "/runtime/diagnostics", runnerInstanceId: "runner-remote", + })({ command: "/controller/runnerd", args: [], cwd: "/controller", environment: {}, isRunnerConnected: () => true }); + let settled = false; + const failure = handle.completion.catch(error => { settled = true; return readRemoteRunnerRecoveryError(error); }); + try { + await vi.advanceTimersByTimeAsync(REMOTE_RUNNER_RECONNECT_GRACE_MS - 60_000); + expect(settled).toBe(false); + await vi.advanceTimersByTimeAsync(180_000); + expect(settled).toBe(true); + expect(await failure).toEqual({ stage: "process_verification", reason: "probe_transport_failed" }); + expect(handle.isExitConfirmed?.()).toBe(false); + } finally { vi.useRealTimers(); } + }); + it.each(["delivered", "sandbox_missing", "logging_failed"] as const)( "detaches runnerd and contains asynchronous signal failures (%s)", async (signalOutcome) => { let launchNonce = ""; diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 18fc09d6954..c4a1b48c36f 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -15168,27 +15168,42 @@ export function createRemoteRunnerProcessLauncher(input: { attributes: { identitySource: "remote_marker", detached: true }, }); + let supervisionUnknownSince: number | null = null; while (true) { - const observed = await readRemoteRunnerRecoveryProbe({ - runner, - stage: "process_verification", - deadline: Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS, - readOnly: true, - request: { - command: "sh", - args: [ - "-c", - REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, - "paperclip-runner-monitor", - input.processIdentityPath, - identityNonce, - input.runnerInstanceId, - String(identity.pid), - ...(identity.processStartFingerprint ? [identity.processStartFingerprint] : []), - ], - bypassSession: true, - }, - }); + let observed: Awaited>; + try { + observed = await readRemoteRunnerRecoveryProbe({ + runner, + stage: "process_verification", + deadline: Date.now() + REMOTE_RUNNER_RECOVERY_PROBE_BUDGET_MS, + readOnly: true, + request: { + command: "sh", + args: [ + "-c", + REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT, + "paperclip-runner-monitor", + input.processIdentityPath, + identityNonce, + input.runnerInstanceId, + String(identity.pid), + ...(identity.processStartFingerprint ? [identity.processStartFingerprint] : []), + ], + bypassSession: true, + }, + }); + } catch (error) { + supervisionUnknownSince ??= Date.now(); + if ( + spec.isRunnerConnected?.() !== true || + Date.now() - supervisionUnknownSince >= REMOTE_RUNNER_RECONNECT_GRACE_MS + ) throw error; + await new Promise((resolve) => + setTimeout(resolve, REMOTE_RUNNER_PROCESS_POLL_MS), + ); + continue; + } + supervisionUnknownSince = null; if (observed.exitCode === 0 && !observed.timedOut) { await new Promise((resolve) => setTimeout(resolve, REMOTE_RUNNER_PROCESS_POLL_MS),