fix(runner): stop a leftover Runner on the same state folder before a relaunch

A Runner launched by the previous server just before an upgrade kept the
session's state folder and ingress port. Each relaunch failed with
runner_ingress_bind_conflict and overwrote the process marker, so the
stale Runner was never found and stopped. The launch script now stops any
live paperclip-runnerd that names the same --state-dir before it starts.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-12 04:48:42 -06:00
1 parent f30005440b
commit 407bf42ca1
2 files changed
+52 -1

No files matched your search

@@ -1362,6 +1362,32 @@ describe("remote runner launch fingerprint compatibility", () => {
expect(lines[4]).toBe(hasBootIdentity ? "linux:abcd-1234:1234" : "");
} finally { await rm(root, { recursive: true, force: true }); }
});
// Staging failure: the previous server launched a runner just before an
// upgrade. It kept the run's state directory and ingress port, and each new
// launch failed with runner_ingress_bind_conflict in a restart loop.
it.skipIf(!existsSync("/proc/self/cmdline"))("stops a live runner that still holds this state directory before launching", async () => {
const root = await mkdtemp(join(tmpdir(), "remote-launch-fence-"));
const stateDirectory = join(root, "state");
const runnerAt = (directory: string) => spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", "--", "--state-dir", directory],
{ argv0: join(root, "bin", "paperclip-runnerd"), stdio: "ignore", detached: true });
const stale = runnerAt(stateDirectory);
const unrelated = runnerAt(join(root, "other-state"));
const staleExit = new Promise<string>(resolveExit => stale.once("exit", (_code, signal) => resolveExit(String(signal))));
try {
await vi.waitFor(async () => expect((await readFile(`/proc/${stale.pid}/cmdline`, "utf8")).split("\0")).toContain(stateDirectory));
await vi.waitFor(async () => expect((await readFile(`/proc/${unrelated.pid}/cmdline`, "utf8")).split("\0")).toContain("--state-dir"));
const result = execFileSync("/bin/sh", ["-c", REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "launch", join(root, "identity"), "nonce", "runner",
join(root, "diagnostics"), "/bin/sh", "-c", "printf launched", "paperclip-runnerd", "--state-dir", stateDirectory], { encoding: "utf8" });
expect(result).toBe("launched");
expect(await Promise.race([staleExit, new Promise(resolveAlive => setTimeout(() => resolveAlive("alive"), 8_000))])).toBe("SIGTERM");
expect(() => process.kill(unrelated.pid!, 0)).not.toThrow();
} finally {
stale.kill("SIGKILL");
unrelated.kill("SIGKILL");
await rm(root, { recursive: true, force: true });
}
}, 20_000);
});
describe("remote runner process supervision", () => {
@@ -14812,8 +14812,33 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT =
`if test "\${size:-0}" -gt ${REMOTE_RUNNER_PROCESS_STDERR_TRIM_BYTES} && test -f "$log" && test ! -L "$log"; then ` +
`tail -c ${REMOTE_RUNNER_PROCESS_STDERR_KEEP_BYTES} -- "$log" > "$log.trim" && cat -- "$log.trim" > "$log"; rm -f -- "$log.trim"; fi; done`;
/**
* Two runnerd processes must never share one durable state directory. A runner
* from an interrupted attempt (for example, launched by the previous server
* just before an upgrade) can still hold that state and the ingress port, and
* each new launch overwrites the identity marker that pointed at it. Stop any
* live runnerd whose arguments name this exact state directory before the new
* one starts. A normal launch finds none and pays one grep over /proc.
*/
const REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT =
'state_dir=""; previous_arg=""; for arg in "$@"; do if test "$previous_arg" = "--state-dir"; then state_dir=$arg; fi; previous_arg=$arg; done; ' +
'if test -n "$state_dir" && test -r /proc/self/cmdline; then ' +
'for stale_cmdline in $(grep -l -s -z -F -x -- "$state_dir" /proc/[0-9]*/cmdline 2>/dev/null || true); do ' +
'stale_pid=${stale_cmdline#/proc/}; stale_pid=${stale_pid%/cmdline}; ' +
'case "$stale_pid" in ""|*[!0-9]*) continue ;; esac; test "$stale_pid" != "$$" || continue; ' +
'stale_binary=$(head -z -n 1 "$stale_cmdline" 2>/dev/null | tr -d "\\000") || continue; ' +
'test "${stale_binary##*/}" = "paperclip-runnerd" || continue; ' +
'grep -q -s -z -F -x -- "--state-dir" "$stale_cmdline" || continue; ' +
'kill -TERM -- "-$stale_pid" 2>/dev/null || kill -TERM "$stale_pid" 2>/dev/null || continue; ' +
'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; ' +
'if kill -0 "$stale_pid" 2>/dev/null; then kill -KILL -- "-$stale_pid" 2>/dev/null || kill -KILL "$stale_pid" 2>/dev/null || true; ' +
'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; fi; ' +
'done; fi; ';
export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT =
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; ' +
REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT +
'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT +
'; then :; else process_fingerprint=""; fi; ' +
// Bounded stderr evidence: rotate an oversized file once, then trim it in place while runnerd runs.