fix(runner): stop the stale Runner's whole lineage and fail closed if it survives

The fence now stops the leftover runnerd together with its provider and
tool processes (their own process groups), via a Node step that only runs
when a stale holder is found. If any of them survives TERM and KILL, the
launch exits before it writes a marker and logs
runner_stale_state_holder_alive in the process stderr log.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 05:00:41 -06:00
1 parent 407bf42ca1
commit 8df6a9099e
2 files changed
+30 -9

No files matched your search

@@ -1369,25 +1369,34 @@ describe("remote runner launch fingerprint compatibility", () => {
it.skipIf(!existsSync("/proc/self/cmdline"))("stops a live runner that still holds this state directory before launching", async () => {
const root = await mkdtemp(join(tmpdir(), "remote-launch-fence-"));
const stateDirectory = join(root, "state");
const runnerAt = (directory: string) => spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", "--", "--state-dir", directory],
const toolPidFile = join(root, "tool.pid");
// The stale Runner's provider tool runs in its own process group.
const staleScript = `const c=require("child_process").spawn(process.execPath,["-e","setInterval(()=>{},1000)"],{detached:true,stdio:"ignore"});`
+ `require("fs").writeFileSync(${JSON.stringify(toolPidFile)},String(c.pid));setInterval(()=>{},1000)`;
const runnerAt = (directory: string, script = "setInterval(() => {}, 1000)") => spawn(process.execPath, ["-e", script, "--", "--state-dir", directory],
{ argv0: join(root, "bin", "paperclip-runnerd"), stdio: "ignore", detached: true });
const stale = runnerAt(stateDirectory);
const stale = runnerAt(stateDirectory, staleScript);
const unrelated = runnerAt(join(root, "other-state"));
const staleExit = new Promise<string>(resolveExit => stale.once("exit", (_code, signal) => resolveExit(String(signal))));
const gone = (pid: number) => { try { return readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]?.startsWith("Z") ?? true; } catch { return true; } };
let toolPid = 0;
try {
await vi.waitFor(async () => expect((await readFile(`/proc/${stale.pid}/cmdline`, "utf8")).split("\0")).toContain(stateDirectory));
await vi.waitFor(async () => expect((await readFile(`/proc/${unrelated.pid}/cmdline`, "utf8")).split("\0")).toContain("--state-dir"));
await vi.waitFor(async () => { toolPid = Number(await readFile(toolPidFile, "utf8")); expect(toolPid).toBeGreaterThan(0); });
const result = execFileSync("/bin/sh", ["-c", REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "launch", join(root, "identity"), "nonce", "runner",
join(root, "diagnostics"), "/bin/sh", "-c", "printf launched", "paperclip-runnerd", "--state-dir", stateDirectory], { encoding: "utf8" });
expect(result).toBe("launched");
expect(await Promise.race([staleExit, new Promise(resolveAlive => setTimeout(() => resolveAlive("alive"), 8_000))])).toBe("SIGTERM");
expect(gone(toolPid)).toBe(true);
expect(() => process.kill(unrelated.pid!, 0)).not.toThrow();
} finally {
stale.kill("SIGKILL");
unrelated.kill("SIGKILL");
if (toolPid) try { process.kill(toolPid, "SIGKILL"); } catch { /* already gone */ }
await rm(root, { recursive: true, force: true });
}
}, 20_000);
}, 30_000);
});
describe("remote runner process supervision", () => {
@@ -14818,8 +14818,21 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT =
* just before an upgrade) can still hold that state and the ingress port, and
* each new launch overwrites the identity marker that pointed at it. Stop any
* live runnerd whose arguments name this exact state directory before the new
* one starts. A normal launch finds none and pays one grep over /proc.
* one starts, together with its whole process lineage (providers and tools run
* in their own process groups). A normal launch finds none and pays one grep
* over /proc. When a holder cannot be stopped, the launch fails before it
* writes a marker, with a named reason in the process stderr log.
*/
const REMOTE_RUNNER_STALE_LINEAGE_STOP_SCRIPT =
`const fs=require('fs'),root=Number(process.argv[1]);` +
`const stat=p=>{try{const s=fs.readFileSync('/proc/'+p+'/stat','utf8'),r=s.slice(s.lastIndexOf(')')+2).split(' ');return{state:r[0],ppid:Number(r[1])}}catch{return null}};` +
`const kids=new Map();for(const d of fs.readdirSync('/proc')){if(String(Number(d))!==d)continue;const s=stat(d);if(s){const k=kids.get(s.ppid)||[];k.push(Number(d));kids.set(s.ppid,k)}}` +
`const all=[root];for(let i=0;i<all.length;i+=1)for(const c of kids.get(all[i])||[])if(!all.includes(c))all.push(c);` +
`const alive=p=>{const s=stat(p);return s!==null&&s.state!=='Z'};` +
`const send=sig=>{for(const p of all)for(const t of[-p,p]){try{process.kill(t,sig)}catch{}}};` +
`const pause=new Int32Array(new SharedArrayBuffer(4));const wait=ms=>{const end=Date.now()+ms;while(Date.now()<end&&all.some(alive))Atomics.wait(pause,0,0,100)};` +
`send('SIGTERM');wait(5000);if(all.some(alive)){send('SIGKILL');wait(5000)}process.exit(all.some(alive)?1:0)`;
const REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT =
'state_dir=""; previous_arg=""; for arg in "$@"; do if test "$previous_arg" = "--state-dir"; then state_dir=$arg; fi; previous_arg=$arg; done; ' +
'if test -n "$state_dir" && test -r /proc/self/cmdline; then ' +
@@ -14829,16 +14842,15 @@ const REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT =
'stale_binary=$(head -z -n 1 "$stale_cmdline" 2>/dev/null | tr -d "\\000") || continue; ' +
'test "${stale_binary##*/}" = "paperclip-runnerd" || continue; ' +
'grep -q -s -z -F -x -- "--state-dir" "$stale_cmdline" || continue; ' +
'kill -TERM -- "-$stale_pid" 2>/dev/null || kill -TERM "$stale_pid" 2>/dev/null || continue; ' +
'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; ' +
'if kill -0 "$stale_pid" 2>/dev/null; then kill -KILL -- "-$stale_pid" 2>/dev/null || kill -KILL "$stale_pid" 2>/dev/null || true; ' +
'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; fi; ' +
`node -e "${REMOTE_RUNNER_STALE_LINEAGE_STOP_SCRIPT}" "$stale_pid" || { ` +
`printf "%s\\n" "runner_stale_state_holder_alive: runner $stale_pid still holds this state folder" >> "$diagnostics_directory/${REMOTE_RUNNER_PROCESS_STDERR_FILE}"; exit 75; }; ` +
'done; fi; ';
export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT =
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; ' +
'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; ' +
REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT +
'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
'started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT +
'; then :; else process_fingerprint=""; fi; ' +
// Bounded stderr evidence: rotate an oversized file once, then trim it in place while runnerd runs.