mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 04:21:24 +01:00
fix(runner): stop a leftover Runner on the same state folder before a relaunch
A Runner launched by the previous server just before an upgrade kept the session's state folder and ingress port. Each relaunch failed with runner_ingress_bind_conflict and overwrote the process marker, so the stale Runner was never found and stopped. The launch script now stops any live paperclip-runnerd that names the same --state-dir before it starts. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
f30005440b
commit
407bf42ca1
2 files changed
+52
-1
No files matched your search
@@ -1362,6 +1362,32 @@ describe("remote runner launch fingerprint compatibility", () => {
|
||||
expect(lines[4]).toBe(hasBootIdentity ? "linux:abcd-1234:1234" : "");
|
||||
} finally { await rm(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
// Staging failure: the previous server launched a runner just before an
|
||||
// upgrade. It kept the run's state directory and ingress port, and each new
|
||||
// launch failed with runner_ingress_bind_conflict in a restart loop.
|
||||
it.skipIf(!existsSync("/proc/self/cmdline"))("stops a live runner that still holds this state directory before launching", async () => {
|
||||
const root = await mkdtemp(join(tmpdir(), "remote-launch-fence-"));
|
||||
const stateDirectory = join(root, "state");
|
||||
const runnerAt = (directory: string) => spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)", "--", "--state-dir", directory],
|
||||
{ argv0: join(root, "bin", "paperclip-runnerd"), stdio: "ignore", detached: true });
|
||||
const stale = runnerAt(stateDirectory);
|
||||
const unrelated = runnerAt(join(root, "other-state"));
|
||||
const staleExit = new Promise<string>(resolveExit => stale.once("exit", (_code, signal) => resolveExit(String(signal))));
|
||||
try {
|
||||
await vi.waitFor(async () => expect((await readFile(`/proc/${stale.pid}/cmdline`, "utf8")).split("\0")).toContain(stateDirectory));
|
||||
await vi.waitFor(async () => expect((await readFile(`/proc/${unrelated.pid}/cmdline`, "utf8")).split("\0")).toContain("--state-dir"));
|
||||
const result = execFileSync("/bin/sh", ["-c", REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT, "launch", join(root, "identity"), "nonce", "runner",
|
||||
join(root, "diagnostics"), "/bin/sh", "-c", "printf launched", "paperclip-runnerd", "--state-dir", stateDirectory], { encoding: "utf8" });
|
||||
expect(result).toBe("launched");
|
||||
expect(await Promise.race([staleExit, new Promise(resolveAlive => setTimeout(() => resolveAlive("alive"), 8_000))])).toBe("SIGTERM");
|
||||
expect(() => process.kill(unrelated.pid!, 0)).not.toThrow();
|
||||
} finally {
|
||||
stale.kill("SIGKILL");
|
||||
unrelated.kill("SIGKILL");
|
||||
await rm(root, { recursive: true, force: true });
|
||||
}
|
||||
}, 20_000);
|
||||
});
|
||||
|
||||
describe("remote runner process supervision", () => {
|
||||
|
||||
@@ -14812,8 +14812,33 @@ const REMOTE_RUNNER_PROCESS_STDERR_TRIM_SCRIPT =
|
||||
`if test "\${size:-0}" -gt ${REMOTE_RUNNER_PROCESS_STDERR_TRIM_BYTES} && test -f "$log" && test ! -L "$log"; then ` +
|
||||
`tail -c ${REMOTE_RUNNER_PROCESS_STDERR_KEEP_BYTES} -- "$log" > "$log.trim" && cat -- "$log.trim" > "$log"; rm -f -- "$log.trim"; fi; done`;
|
||||
|
||||
/**
|
||||
* Two runnerd processes must never share one durable state directory. A runner
|
||||
* from an interrupted attempt (for example, launched by the previous server
|
||||
* just before an upgrade) can still hold that state and the ingress port, and
|
||||
* each new launch overwrites the identity marker that pointed at it. Stop any
|
||||
* live runnerd whose arguments name this exact state directory before the new
|
||||
* one starts. A normal launch finds none and pays one grep over /proc.
|
||||
*/
|
||||
const REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT =
|
||||
'state_dir=""; previous_arg=""; for arg in "$@"; do if test "$previous_arg" = "--state-dir"; then state_dir=$arg; fi; previous_arg=$arg; done; ' +
|
||||
'if test -n "$state_dir" && test -r /proc/self/cmdline; then ' +
|
||||
'for stale_cmdline in $(grep -l -s -z -F -x -- "$state_dir" /proc/[0-9]*/cmdline 2>/dev/null || true); do ' +
|
||||
'stale_pid=${stale_cmdline#/proc/}; stale_pid=${stale_pid%/cmdline}; ' +
|
||||
'case "$stale_pid" in ""|*[!0-9]*) continue ;; esac; test "$stale_pid" != "$$" || continue; ' +
|
||||
'stale_binary=$(head -z -n 1 "$stale_cmdline" 2>/dev/null | tr -d "\\000") || continue; ' +
|
||||
'test "${stale_binary##*/}" = "paperclip-runnerd" || continue; ' +
|
||||
'grep -q -s -z -F -x -- "--state-dir" "$stale_cmdline" || continue; ' +
|
||||
'kill -TERM -- "-$stale_pid" 2>/dev/null || kill -TERM "$stale_pid" 2>/dev/null || continue; ' +
|
||||
'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; ' +
|
||||
'if kill -0 "$stale_pid" 2>/dev/null; then kill -KILL -- "-$stale_pid" 2>/dev/null || kill -KILL "$stale_pid" 2>/dev/null || true; ' +
|
||||
'stale_wait=0; while kill -0 "$stale_pid" 2>/dev/null && test "$stale_wait" -lt 50; do stale_wait=$((stale_wait + 1)); sleep 0.1; done; fi; ' +
|
||||
'done; fi; ';
|
||||
|
||||
export const REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT =
|
||||
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
|
||||
'set -eu; identity_path=$1; identity_nonce=$2; runner_instance_id=$3; diagnostics_directory=$4; shift 4; umask 077; ' +
|
||||
REMOTE_RUNNER_STALE_STATE_FENCE_SCRIPT +
|
||||
'test ! -L "$diagnostics_directory"; if test -e "$diagnostics_directory"; then test -d "$diagnostics_directory"; else mkdir -p -- "$diagnostics_directory"; fi; chmod 0700 "$diagnostics_directory"; started_at=$(date -u +"%Y-%m-%dT%H:%M:%S.%3NZ"); pid=$$; process_fingerprint=""; if ' +
|
||||
REMOTE_RUNNER_PROCESS_FINGERPRINT_SCRIPT +
|
||||
'; then :; else process_fingerprint=""; fi; ' +
|
||||
// Bounded stderr evidence: rotate an oversized file once, then trim it in place while runnerd runs.
|
||||
|
||||
Reference in new issue
Block a user