diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index f9d6548cf9d..86c52595621 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1369,25 +1369,34 @@ describe("remote runner launch fingerprint compatibility", () => { it.skipIf(!existsSync("/proc/self/cmdline"))("stops a live runner that still holds this state directory before launching", async () => { const root = await mkdtemp(join(tmpdir(), "remote-launch-fence-")); const stateDirectory = join(root, "state"); - const runnerAt = (directory: string) => spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", "--", "--state-dir", directory], + const toolPidFile = join(root, "tool.pid"); + // The stale Runner's provider tool runs in its own process group. + const staleScript = `const c=require("child_process").spawn(process.execPath,["-e","setInterval(()=>{},1000)"],{detached:true,stdio:"ignore"});` + + `require("fs").writeFileSync(${JSON.stringify(toolPidFile)},String(c.pid));setInterval(()=>{},1000)`; + const runnerAt = (directory: string, script = "setInterval(() => {}, 1000)") => spawn(process.execPath, ["-e", script, "--", "--state-dir", directory], { argv0: join(root, "bin", "paperclip-runnerd"), stdio: "ignore", detached: true }); - const stale = runnerAt(stateDirectory); + const stale = runnerAt(stateDirectory, staleScript); const unrelated = runnerAt(join(root, "other-state")); const staleExit = new Promise(resolveExit => stale.once("exit", (_code, signal) => resolveExit(String(signal)))); + const gone = (pid: number) => { try { return readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]?.startsWith("Z") ?? true; } catch { return true; } }; + let toolPid = 0; try { await vi.waitFor(async () => expect((await readFile(`/proc/${stale.pid}/cmdline`, "utf8")).split("\0")).toContain(stateDirectory)); await vi.waitFor(async () => expect((await readFile(`/proc/${unrelated.pid}/cmdline`, "utf8")).split("\0")).toContain("--state-dir")); + await vi.waitFor(async () => { toolPid = Number(await readFile(toolPidFile, "utf8")); expect(toolPid).toBeGreaterThan(0); }); const result = execFileSync("/bin/sh", ["-c", REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "launch", join(root, "identity"), "nonce", "runner", join(root, "diagnostics"), "/bin/sh", "-c", "printf launched", "paperclip-runnerd", "--state-dir", stateDirectory], { encoding: "utf8" }); expect(result).toBe("launched"); expect(await Promise.race([staleExit, new Promise(resolveAlive => setTimeout(() => resolveAlive("alive"), 8_000))])).toBe("SIGTERM"); + expect(gone(toolPid)).toBe(true); expect(() => process.kill(unrelated.pid!, 0)).not.toThrow(); } finally { stale.kill("SIGKILL"); unrelated.kill("SIGKILL"); + if (toolPid) try { process.kill(toolPid, "SIGKILL"); } catch { /* already gone */ } await rm(root, { recursive: true, force: true }); } - }, 20_000); + }, 30_000); }); describe("remote runner process supervision", () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 7912ef7083e..f81f4b5bbe8 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -14818,8 +14818,21 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT = * just before an upgrade) can still hold that state and the ingress port, and * each new launch overwrites the identity marker that pointed at it. Stop any * live runnerd whose arguments name this exact state directory before the new - * one starts. A normal launch finds none and pays one grep over /proc. + * one starts, together with its whole process lineage (providers and tools run + * in their own process groups). A normal launch finds none and pays one grep + * over /proc. When a holder cannot be stopped, the launch fails before it + * writes a marker, with a named reason in the process stderr log. */ +const REMOTE_RUNNER_STALE_LINEAGE_STOP_SCRIPT = + `const fs=require('fs'),root=Number(process.argv[1]);` + + `const stat=p=>{try{const s=fs.readFileSync('/proc/'+p+'/stat','utf8'),r=s.slice(s.lastIndexOf(')')+2).split(' ');return{state:r[0],ppid:Number(r[1])}}catch{return null}};` + + `const kids=new Map();for(const d of fs.readdirSync('/proc')){if(String(Number(d))!==d)continue;const s=stat(d);if(s){const k=kids.get(s.ppid)||[];k.push(Number(d));kids.set(s.ppid,k)}}` + + `const all=[root];for(let i=0;i{const s=stat(p);return s!==null&&s.state!=='Z'};` + + `const send=sig=>{for(const p of all)for(const t of[-p,p]){try{process.kill(t,sig)}catch{}}};` + + `const pause=new Int32Array(new SharedArrayBuffer(4));const wait=ms=>{const end=Date.now()+ms;while(Date.now()/dev/null || kill -TERM "$stale_pid" 2>/dev/null || continue; ' + - 'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; ' + - 'if kill -0 "$stale_pid" 2>/dev/null; then kill -KILL -- "-$stale_pid" 2>/dev/null || kill -KILL "$stale_pid" 2>/dev/null || true; ' + - 'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; fi; ' + + `node -e "${REMOTE_RUNNER_STALE_LINEAGE_STOP_SCRIPT}" "$stale_pid" || { ` + + `printf "%s\\n" "runner_stale_state_holder_alive: runner $stale_pid still holds this state folder" >> "$diagnostics_directory/${REMOTE_RUNNER_PROCESS_STDERR_FILE}"; exit 75; }; ` + 'done; fi; '; export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT = 'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; ' + + 'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; ' + REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT + - 'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + + 'started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT + '; then :; else process_fingerprint=""; fi; ' + // Bounded stderr evidence: rotate an oversized file once, then trim it in place while runnerd runs.