From 8df6a9099ec26c9cee60b7ae9d9f27a97d01dab8 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 05:00:41 -0600 Subject: [PATCH] fix(runner): stop the stale Runner's whole lineage and fail closed if it survives The fence now stops the leftover runnerd together with its provider and tool processes (their own process groups), via a Node step that only runs when a stale holder is found. If any of them survives TERM and KILL, the launch exits before it writes a marker and logs runner_stale_state_holder_alive in the process stderr log. Co-Authored-By: Claude Opus 5.5 --- .../native-session-executor.test.ts | 15 +++++++++--- .../native-runtime/native-session-executor.ts | 24 ++++++++++++++----- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index f9d6548cf9d..86c52595621 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1369,25 +1369,34 @@ describe("remote runner launch fingerprint compatibility", () => { it.skipIf(!existsSync("/proc/self/cmdline"))("stops a live runner that still holds this state directory before launching", async () => { const root = await mkdtemp(join(tmpdir(), "remote-launch-fence-")); const stateDirectory = join(root, "state"); - const runnerAt = (directory: string) => spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", "--", "--state-dir", directory], + const toolPidFile = join(root, "tool.pid"); + // The stale Runner's provider tool runs in its own process group. + const staleScript = `const c=require("child_process").spawn(process.execPath,["-e","setInterval(()=>{},1000)"],{detached:true,stdio:"ignore"});` + + `require("fs").writeFileSync(${JSON.stringify(toolPidFile)},String(c.pid));setInterval(()=>{},1000)`; + const runnerAt = (directory: string, script = "setInterval(() => {}, 1000)") => spawn(process.execPath, ["-e", script, "--", "--state-dir", directory], { argv0: join(root, "bin", "paperclip-runnerd"), stdio: "ignore", detached: true }); - const stale = runnerAt(stateDirectory); + const stale = runnerAt(stateDirectory, staleScript); const unrelated = runnerAt(join(root, "other-state")); const staleExit = new Promise(resolveExit => stale.once("exit", (_code, signal) => resolveExit(String(signal)))); + const gone = (pid: number) => { try { return readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]?.startsWith("Z") ?? true; } catch { return true; } }; + let toolPid = 0; try { await vi.waitFor(async () => expect((await readFile(`/proc/${stale.pid}/cmdline`, "utf8")).split("\0")).toContain(stateDirectory)); await vi.waitFor(async () => expect((await readFile(`/proc/${unrelated.pid}/cmdline`, "utf8")).split("\0")).toContain("--state-dir")); + await vi.waitFor(async () => { toolPid = Number(await readFile(toolPidFile, "utf8")); expect(toolPid).toBeGreaterThan(0); }); const result = execFileSync("/bin/sh", ["-c", REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "launch", join(root, "identity"), "nonce", "runner", join(root, "diagnostics"), "/bin/sh", "-c", "printf launched", "paperclip-runnerd", "--state-dir", stateDirectory], { encoding: "utf8" }); expect(result).toBe("launched"); expect(await Promise.race([staleExit, new Promise(resolveAlive => setTimeout(() => resolveAlive("alive"), 8_000))])).toBe("SIGTERM"); + expect(gone(toolPid)).toBe(true); expect(() => process.kill(unrelated.pid!, 0)).not.toThrow(); } finally { stale.kill("SIGKILL"); unrelated.kill("SIGKILL"); + if (toolPid) try { process.kill(toolPid, "SIGKILL"); } catch { /* already gone */ } await rm(root, { recursive: true, force: true }); } - }, 20_000); + }, 30_000); }); describe("remote runner process supervision", () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7912ef7083e..f81f4b5bbe8 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -14818,8 +14818,21 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT = * just before an upgrade) can still hold that state and the ingress port, and * each new launch overwrites the identity marker that pointed at it. Stop any * live runnerd whose arguments name this exact state directory before the new - * one starts. A normal launch finds none and pays one grep over /proc. + * one starts, together with its whole process lineage (providers and tools run + * in their own process groups). A normal launch finds none and pays one grep + * over /proc. When a holder cannot be stopped, the launch fails before it + * writes a marker, with a named reason in the process stderr log. */ +const REMOTE_RUNNER_STALE_LINEAGE_STOP_SCRIPT = + `const fs=require('fs'),root=Number(process.argv[1]);` + + `const stat=p=>{try{const s=fs.readFileSync('/proc/'+p+'/stat','utf8'),r=s.slice(s.lastIndexOf(')')+2).split(' ');return{state:r[0],ppid:Number(r[1])}}catch{return null}};` + + `const kids=new Map();for(const d of fs.readdirSync('/proc')){if(String(Number(d))!==d)continue;const s=stat(d);if(s){const k=kids.get(s.ppid)||[];k.push(Number(d));kids.set(s.ppid,k)}}` + + `const all=[root];for(let i=0;i{const s=stat(p);return s!==null&&s.state!=='Z'};` + + `const send=sig=>{for(const p of all)for(const t of[-p,p]){try{process.kill(t,sig)}catch{}}};` + + `const pause=new Int32Array(new SharedArrayBuffer(4));const wait=ms=>{const end=Date.now()+ms;while(Date.now()/dev/null || kill -TERM "$stale_pid" 2>/dev/null || continue; ' + - 'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; ' + - 'if kill -0 "$stale_pid" 2>/dev/null; then kill -KILL -- "-$stale_pid" 2>/dev/null || kill -KILL "$stale_pid" 2>/dev/null || true; ' + - 'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; fi; ' + + `node -e "${REMOTE_RUNNER_STALE_LINEAGE_STOP_SCRIPT}" "$stale_pid" || { ` + + `printf "%s\\n" "runner_stale_state_holder_alive: runner $stale_pid still holds this state folder" >> "$diagnostics_directory/${REMOTE_RUNNER_PROCESS_STDERR_FILE}"; exit 75; }; ` + 'done; fi; '; export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT = 'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; ' + + 'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; ' + REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT + - 'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + + 'started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT + '; then :; else process_fingerprint=""; fi; ' + // Bounded stderr evidence: rotate an oversized file once, then trim it in place while runnerd runs.