diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 43727f505ee..8e1ce40548f 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -711,6 +711,71 @@ describe("remote runner launch fingerprint compatibility", () => { }); describe("remote runner process supervision", () => { + it.each(["timeout", "ssh_failure", "rejected_rpc"] as const)( + "keeps the admitted process after a transient %s monitor failure", async failure => { + vi.useFakeTimers(); + try { + let nonce = ""; + let polls = 0; + const launch = vi.fn(async (request: { args: string[] }) => { nonce = request.args[4]!; }); + const execute = vi.fn(async (request: { args: string[] }) => { + const label = request.args[2]; + const result = { exitCode: 0, timedOut: false, signal: null, stdout: "", stderr: "" }; + if (label === "paperclip-runner-process-identity") return { ...result, + stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\ncomputer-runner\nlinux:abcd-1234:100\n` }; + if (label === "paperclip-runner-monitor") { + polls += 1; + if (polls === 1) { + if (failure === "rejected_rpc") throw new Error("SSH connection reset"); + return { ...result, exitCode: failure === "timeout" ? 3 : 255, timedOut: failure === "timeout" }; + } + return { ...result, exitCode: polls === 3 ? 4 : 0 }; + } + if (label === "paperclip-runner-diagnostics") return result; + return { ...result, stdout: Buffer.from("{}").toString("base64") }; + }); + const launcher = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "computer", remoteCwd: "/home/user/project", launch } as never, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/state/identity", + stateDirectory: "/runtime/state", diagnosticsDirectory: "/runtime/logs", runnerInstanceId: "computer-runner", + }); + const handle = launcher({ command: "/controller/runnerd", args: ["--runner-id", "computer-runner"], cwd: "/controller", environment: {} }); + let settled = false; + void handle.completion.then(() => { settled = true; }); + await vi.advanceTimersByTimeAsync(1_100); + expect(polls).toBe(2); + expect(settled).toBe(false); + expect(launch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1_000); + await expect(handle.completion).resolves.toMatchObject({ stderr: "runner_remote_process_identity_mismatch" }); + expect(polls).toBe(3); + } finally { vi.useRealTimers(); } + }, + ); + + it("bounds inconclusive process monitoring without claiming the process exited", async () => { + vi.useFakeTimers(); + try { + let nonce = ""; + const launch = vi.fn(async (request: { args: string[] }) => { nonce = request.args[4]!; }); + const execute = vi.fn(async (request: { args: string[] }) => ({ + exitCode: request.args[2] === "paperclip-runner-monitor" ? 255 : 0, + timedOut: false, signal: null, stderr: "", + stdout: `${nonce}\n4321\n2026-09-06T00:00:00.000Z\ncomputer-runner\nlinux:abcd-1234:100\n`, + })); + const launcher = createRemoteRunnerProcessLauncher({ + target: { kind: "remote", transport: "computer", remoteCwd: "/home/user/project", launch } as never, + runner: { execute } as never, remoteBinary: "/runtime/runnerd", processIdentityPath: "/runtime/state/identity", + stateDirectory: "/runtime/state", diagnosticsDirectory: "/runtime/logs", runnerInstanceId: "computer-runner", + }); + const handle = launcher({ command: "/controller/runnerd", args: ["--runner-id", "computer-runner"], cwd: "/controller", environment: {} }); + const completion = expect(handle.completion).rejects.toThrow("runner_remote_process_monitor_unavailable exitCode=255 timedOut=false"); + await vi.advanceTimersByTimeAsync(2_100); + await completion; + expect(launch).toHaveBeenCalledOnce(); + } finally { vi.useRealTimers(); } + }); + it("launches a computer runner through owned process admission, never detached shell allocation", async () => { let nonce = ""; const launch = vi.fn(async (request: { args: string[] }) => { nonce = request.args[4]!; return {}; }); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index e5bef3d137d..cfb6791ea1d 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -11191,6 +11191,7 @@ export function createRemoteRunnerProcessLauncher(input: { attributes: { identitySource: "remote_marker", detached: true }, }); + let consecutiveUnknownObservations = 0; while (true) { const observed = await runner.execute({ command: "sh", @@ -11205,8 +11206,18 @@ export function createRemoteRunnerProcessLauncher(input: { ], bypassSession: true, timeoutMs: 10_000, - }); - if (observed.exitCode === 0 && !observed.timedOut) { + }).catch(() => null); + const identityConfirmed = observed?.exitCode === 0 && !observed.timedOut; + const identityLost = observed !== null && !observed.timedOut + && (observed.exitCode === 3 || observed.exitCode === 4); + if (!identityLost) { + // SSH/provider failures do not prove that runnerd exited. In particular, + // concurrent artifact uploads can delay a monitor RPC while PRP stays + // healthy. Retry unknown observations without allocating another runner. + consecutiveUnknownObservations = identityConfirmed ? 0 : consecutiveUnknownObservations + 1; + if (consecutiveUnknownObservations >= 3) { + throw new Error(`runner_remote_process_monitor_unavailable exitCode=${observed?.exitCode ?? "unknown"} timedOut=${observed?.timedOut ?? "unknown"}`); + } await new Promise((resolve) => setTimeout(resolve, REMOTE_RUNNER_PROCESS_POLL_MS), );