diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts index d872a071ff2..f7bb79bb987 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.test.ts @@ -11,7 +11,7 @@ import { function shellExit(script: string, args: string[]) { try { - execFileSync("sh", ["-c", script, "recovery-test", ...args], { stdio: "pipe", timeout: 2_000 }); + execFileSync("sh", ["-c", script, "recovery-test", ...args], { stdio: "pipe", timeout: 5_000 }); return 0; } catch (error) { return (error as { status: number }).status; @@ -77,6 +77,34 @@ describe("remote recovery diagnostic shell receipts", () => { } finally { await rm(root, { recursive: true, force: true }); } }); + it.each([ + { exit: 'fs.writeFileSync(stat, real.call(fs, stat, "utf8").replace(" R ", " Z "))', name: "becomes a zombie" }, + { exit: "fs.rmSync(stat)", name: "is reaped" }, + ])("reports a Runner caught mid-exit, with its command line already released, as not running once it $name", async ({ exit }) => { + const root = await mkdtemp(join(tmpdir(), "remote-probe-exiting-")); + const marker = join(root, "marker"); + const stat = join(root, "stat"); + const hook = join(root, "exit-on-second-stat-read.cjs"); + try { + await writeFile(marker, `nonce\n${process.pid}\n2026-11-17T08:51:21.931Z\nrunner\n`); + await writeFile(join(root, "cmdline"), ""); + await writeFile(stat, `${process.pid} (MainThread) ${["R", ...Array(18).fill("0"), 100].join(" ")}\n`); + await writeFile(join(root, "boot_id"), "abcd-1234\n"); + await writeFile(hook, [ + 'const fs = require("node:fs");', + `const stat = ${JSON.stringify(stat)};`, + "const real = fs.readFileSync;", + "let reads = 0;", + `fs.readFileSync = function (path, ...rest) { if (path === stat && ++reads === 2) ${exit}; return real.call(this, path, ...rest); };`, + ].join("\n")); + const script = `NODE_OPTIONS="--require=${hook}"; export NODE_OPTIONS; ` + REMOTE_RECOVERY_PROCESS_CHECK_SCRIPT + .replaceAll('"/proc/" + pid + "/cmdline"', JSON.stringify(join(root, "cmdline"))) + .replaceAll('"/proc/" + pid + "/stat"', JSON.stringify(stat)) + .replaceAll('"/proc/sys/kernel/random/boot_id"', JSON.stringify(join(root, "boot_id"))); + expect(shellExit(script, [marker, "nonce", "runner", String(process.pid), "linux:abcd-1234:100"])).toBe(45); + } finally { await rm(root, { recursive: true, force: true }); } + }); + it("distinguishes missing, unsafe and valid marker reads without exposing their contents", async () => { const root = await mkdtemp(join(tmpdir(), "remote-recovery-read-")); const marker = join(root, "marker"); diff --git a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts index 58d95862346..ffdee69d62d 100644 --- a/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts +++ b/server/src/services/native-runtime/remote-runner-recovery-diagnostics.ts @@ -206,6 +206,27 @@ function verifyGeneration() { process.exit(46); } } +function awaitExitingProcess() { + const pause = new Int32Array(new SharedArrayBuffer(4)); + const until = Date.now() + 1000; + while (Date.now() < until) { + let state; + try { + const stat = fs.readFileSync("/proc/" + pid + "/stat", "utf8"); + state = stat.slice(stat.lastIndexOf(") ") + 2).trim().split(/\s+/)[0]; + } catch (error) { + if (error.code === "ENOENT") process.exit(45); + return; + } + if (state === "Z" || state === "X" || state === "x") process.exit(45); + try { process.kill(Number(pid), 0); } + catch (error) { + if (error.code === "ESRCH") process.exit(45); + return; + } + Atomics.wait(pause, 0, 0, 50); + } +} requireLiveProcess(); const commandPath = "/proc/" + pid + "/cmdline"; if (process.platform !== "linux" && !expectedGeneration && !fs.existsSync(commandPath)) process.exit(0); @@ -219,6 +240,7 @@ catch (error) { if (expectedGeneration) verifyGeneration(); if (!args.includes("--runner-id") || !args.includes(runnerId)) { requireLiveProcess(); + if (expectedGeneration && args.join("") === "") awaitExitingProcess(); process.exit(48); } ' "$pid" "$expected_runner_id"` + ' "${5:-}"',