From 407bf42ca128812e7d585dc8e84e03c2f1a6e62e Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 12 Nov 2026 04:48:42 -0600 Subject: [PATCH] fix(runner): stop a leftover Runner on the same state folder before a relaunch A Runner launched by the previous server just before an upgrade kept the session's state folder and ingress port. Each relaunch failed with runner_ingress_bind_conflict and overwrote the process marker, so the stale Runner was never found and stopped. The launch script now stops any live paperclip-runnerd that names the same --state-dir before it starts. Co-Authored-By: Claude Opus 5.5 --- .../native-session-executor.test.ts | 26 ++++++++++++++++++ .../native-runtime/native-session-executor.ts | 27 ++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 81a80f99f1d..f9d6548cf9d 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -1362,6 +1362,32 @@ describe("remote runner launch fingerprint compatibility", () => { expect(lines[4]).toBe(hasBootIdentity ? "linux:abcd-1234:1234" : ""); } finally { await rm(root, { recursive: true, force: true }); } }); + + // Staging failure: the previous server launched a runner just before an + // upgrade. It kept the run's state directory and ingress port, and each new + // launch failed with runner_ingress_bind_conflict in a restart loop. + it.skipIf(!existsSync("/proc/self/cmdline"))("stops a live runner that still holds this state directory before launching", async () => { + const root = await mkdtemp(join(tmpdir(), "remote-launch-fence-")); + const stateDirectory = join(root, "state"); + const runnerAt = (directory: string) => spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", "--", "--state-dir", directory], + { argv0: join(root, "bin", "paperclip-runnerd"), stdio: "ignore", detached: true }); + const stale = runnerAt(stateDirectory); + const unrelated = runnerAt(join(root, "other-state")); + const staleExit = new Promise(resolveExit => stale.once("exit", (_code, signal) => resolveExit(String(signal)))); + try { + await vi.waitFor(async () => expect((await readFile(`/proc/${stale.pid}/cmdline`, "utf8")).split("\0")).toContain(stateDirectory)); + await vi.waitFor(async () => expect((await readFile(`/proc/${unrelated.pid}/cmdline`, "utf8")).split("\0")).toContain("--state-dir")); + const result = execFileSync("/bin/sh", ["-c", REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "launch", join(root, "identity"), "nonce", "runner", + join(root, "diagnostics"), "/bin/sh", "-c", "printf launched", "paperclip-runnerd", "--state-dir", stateDirectory], { encoding: "utf8" }); + expect(result).toBe("launched"); + expect(await Promise.race([staleExit, new Promise(resolveAlive => setTimeout(() => resolveAlive("alive"), 8_000))])).toBe("SIGTERM"); + expect(() => process.kill(unrelated.pid!, 0)).not.toThrow(); + } finally { + stale.kill("SIGKILL"); + unrelated.kill("SIGKILL"); + await rm(root, { recursive: true, force: true }); + } + }, 20_000); }); describe("remote runner process supervision", () => { diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 80e36428abb..7912ef7083e 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -14812,8 +14812,33 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT = `if test "\${size:-0}" -gt ${REMOTE_RUNNER_PROCESS_STDERR_TRIM_BYTES} && test -f "$log" && test ! -L "$log"; then ` + `tail -c ${REMOTE_RUNNER_PROCESS_STDERR_KEEP_BYTES} -- "$log" > "$log.trim" && cat -- "$log.trim" > "$log"; rm -f -- "$log.trim"; fi; done`; +/** + * Two runnerd processes must never share one durable state directory. A runner + * from an interrupted attempt (for example, launched by the previous server + * just before an upgrade) can still hold that state and the ingress port, and + * each new launch overwrites the identity marker that pointed at it. Stop any + * live runnerd whose arguments name this exact state directory before the new + * one starts. A normal launch finds none and pays one grep over /proc. + */ +const REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT = + 'state_dir=""; previous_arg=""; for arg in "$@"; do if test "$previous_arg" = "--state-dir"; then state_dir=$arg; fi; previous_arg=$arg; done; ' + + 'if test -n "$state_dir" && test -r /proc/self/cmdline; then ' + + 'for stale_cmdline in $(grep -l -s -z -F -x -- "$state_dir" /proc/[0-9]*/cmdline 2>/dev/null || true); do ' + + 'stale_pid=${stale_cmdline#/proc/}; stale_pid=${stale_pid%/cmdline}; ' + + 'case "$stale_pid" in ""|*[!0-9]*) continue ;; esac; test "$stale_pid" != "$$" || continue; ' + + 'stale_binary=$(head -z -n 1 "$stale_cmdline" 2>/dev/null | tr -d "\\000") || continue; ' + + 'test "${stale_binary##*/}" = "paperclip-runnerd" || continue; ' + + 'grep -q -s -z -F -x -- "--state-dir" "$stale_cmdline" || continue; ' + + 'kill -TERM -- "-$stale_pid" 2>/dev/null || kill -TERM "$stale_pid" 2>/dev/null || continue; ' + + 'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; ' + + 'if kill -0 "$stale_pid" 2>/dev/null; then kill -KILL -- "-$stale_pid" 2>/dev/null || kill -KILL "$stale_pid" 2>/dev/null || true; ' + + 'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; fi; ' + + 'done; fi; '; + export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT = - 'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + + 'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; ' + + REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT + + 'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' + REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT + '; then :; else process_fingerprint=""; fi; ' + // Bounded stderr evidence: rotate an oversized file once, then trim it in place while runnerd runs.