From b5adf3511998689b9def47975d4c7df1b2b2338b Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 13 Nov 2026 14:46:24 -0600 Subject: [PATCH] test(runner): guard that a relaunch after a lost launch acknowledgement leaves one Runner (#21477) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Agent turns run in a remote Runner (`paperclip-runnerd`) that the server launches in a sandbox. > - A deploy can stop the server after the launch command reached the sandbox but before the server read back the Runner's process identity (a lost acknowledgement). > - The database then has no PID for the run, so the successor server launches the Runner again on the same state folder. > - Two Runners must never serve the same run's state. #20852 proposed a large launch-intent design for this window. Since then, #21079 added a fence that stops a leftover Runner on the same state folder before each launch. > - This pull request adds a Linux regression test that drives the real launcher through the lost-acknowledgement window. > - The benefit is that a future change to the launch path cannot bring back a second Runner without failing this test. ## Linked Issues or Issue Description Refs #20852 (superseded: current master already prevents the duplicate Runner through #21079). Refs #21079. **What happened?** #20852 claimed that a relaunch after a lost launch acknowledgement could start a second Runner for the same run. Nothing proved whether current master still has this problem. **Expected behavior** After a lost acknowledgement, the successor's relaunch stops the orphaned Runner. Exactly one Runner serves the run's state folder, and the process marker names it. **Steps to reproduce** 1. Launch a remote Runner, and lose the launch RPC's acknowledgement before the identity read (the old server stops). 2. Relaunch the same run on the same sandbox from a new launcher (the successor). 3. Count the live `paperclip-runnerd` processes that serve the run's `--state-dir`. ## What Changed - New `server/src/services/native-runtime/remote-runner-lost-launch-ack.test.ts`. It is a test only and changes no product code. - It runs only where `/proc` exists (Linux CI), like the existing #21079 fence tests. - It drives the exported `createRemoteRunnerProcessLauncher` with a local command runner. The old launcher's runner runs the launch command, then throws `remote_launch_ack_lost`. After that, every command fails, because the old server is gone. - A small bash stand-in for `paperclip-runnerd` keeps the Runner's argv[0] and `--state-dir`. Its `reap-lineage` reports a verified, empty lineage. - The test checks these results: the orphan was alive and had no PID on the handle; the successor's Runner has a new PID; the orphan stops; exactly one Runner serves the state folder; the marker names the successor's PID. ## Verification - Linux reproduction (Docker `node:24-bookworm-slim`). I used the exact outer launch command and `REMOTE_RUNNER_CHILD_LAUNCH_SCRIPT`, which I extracted from this head: - master scripts: `runners after successor relaunch [61], orphan alive false`. Result: one Runner (PASS). - the same scripts without the #21079 stale-state fence (counterfactual): `runners after successor relaunch [24, 56], orphan alive true`. Result: duplicate Runner (FAIL). - macOS: `npx vitest run src/services/native-runtime/remote-runner-lost-launch-ack.test.ts` reports the file as skipped (1 skipped), as intended, because there is no `/proc`. - The test runs for real only on Linux CI. CI on this head is the first real run of the vitest file. ## Risks - Low risk. The change is a test only. - The test starts real local processes. Its teardown kills only the PIDs it found on its own temporary state folder, and it never signals PID 0. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Claude Code agent with tool use (shell, code search, Docker, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../remote-runner-lost-launch-ack.test.ts | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 server/src/services/native-runtime/remote-runner-lost-launch-ack.test.ts diff --git a/server/src/services/native-runtime/remote-runner-lost-launch-ack.test.ts b/server/src/services/native-runtime/remote-runner-lost-launch-ack.test.ts new file mode 100644 index 00000000000..01439b6dd82 --- /dev/null +++ b/server/src/services/native-runtime/remote-runner-lost-launch-ack.test.ts @@ -0,0 +1,119 @@ +import { spawn } from "node:child_process"; +import { existsSync, readdirSync, readFileSync } from "node:fs"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { describe, expect, it, vi } from "vitest"; +import { createRemoteRunnerProcessLauncher } from "./native-session-executor.js"; + +type Execute = CommandManagedRuntimeRunner["execute"]; + +/** Runs each sandbox command on this machine, the way a sandbox command RPC would. */ +const localExecute: Execute = request => new Promise(resolve => { + const child = spawn(request.command, request.args ?? [], { cwd: request.cwd, + env: { ...process.env, ...request.env }, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = "", stderr = ""; + child.stdout.on("data", chunk => { stdout += String(chunk); }); + child.stderr.on("data", chunk => { stderr += String(chunk); }); + child.once("close", (code, signal) => resolve({ exitCode: code, signal, timedOut: false, pid: child.pid ?? null, + startedAt: null, stdout, stderr })); +}); + +const alive = (pid: number) => { + try { return !readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")[1]!.startsWith("Z"); } catch { return false; } +}; + +/** Every live `paperclip-runnerd` that serves this state folder, read from /proc as the launch fence does. */ +function runnersServing(stateDirectory: string): number[] { + const pids: number[] = []; + for (const entry of readdirSync("/proc")) { + if (String(Number(entry)) !== entry) continue; + let argv: string[]; + try { argv = readFileSync(`/proc/${entry}/cmdline`, "utf8").split("\0"); } catch { continue; } + if (basename(argv[0] ?? "") !== "paperclip-runnerd" || argv[1] === "reap-lineage") continue; + if (argv.some((value, index) => value === "--state-dir" && argv[index + 1] === stateDirectory) && alive(Number(entry))) { + pids.push(Number(entry)); + } + } + return pids; +} + +// The server can stop after the launch command reached the sandbox but before +// it read back the Runner's process identity (a lost acknowledgement). The +// database then holds no PID for the run, and the successor launches again. +// This guards that a relaunch never leaves two Runners on the run's state. +describe.skipIf(!existsSync("/proc/self/cmdline"))("remote Runner launch whose acknowledgement was lost", () => { + it("leaves exactly one Runner on the run's state folder after the successor relaunches", async () => { + const root = await mkdtemp(join(tmpdir(), "runner-lost-launch-ack-")); + const stateDirectory = join(root, "sandbox", "state"); + const binary = join(root, "bin", "paperclip-runnerd"); + await mkdir(join(root, "bin"), { recursive: true }); + await mkdir(stateDirectory, { recursive: true }); + // Stands in for runnerd: a long-lived process whose argv[0] is the Runner + // binary. `reap-lineage` reports a verified, empty lineage. + await writeFile(binary, [ + "#!/bin/bash", + 'if test "$1" = reap-lineage; then', + ` printf '%s\\n' '${JSON.stringify({ schema: "paperclip.runner.process-lineage-reap.v1", verified: true, reason: null, journals: 1, signalled: 0, remaining: [] })}'`, + " exit 0", + "fi", + `exec -a "$0" ${JSON.stringify(process.execPath)} -e 'setInterval(() => {}, 1000)' -- "$@"`, + "", + ].join("\n"), { mode: 0o755 }); + const launcherInput = { + target: { kind: "remote" as const, transport: "sandbox" as const, environmentId: "environment", leaseId: "lease", + remoteCwd: join(root, "sandbox") }, + remoteBinary: binary, + processIdentityPath: join(stateDirectory, "runner-process.identity"), + stateDirectory, + diagnosticsDirectory: join(root, "sandbox", "diagnostics"), + runnerInstanceId: "runner-lost-ack", + }; + const spec = { command: binary, args: ["--runner-id", "runner-lost-ack", "--state-dir", stateDirectory], + cwd: root, environment: {} }; + + // The old server: its launch command runs in the sandbox, then the + // connection is lost before the acknowledgement, and the server is gone. + let serverGone = false; + const oldExecute = vi.fn(async request => { + if (serverGone) throw new Error("old_controller_gone"); + if (request.args?.[2] !== "paperclip-runner-launch") return localExecute(request); + await localExecute(request); + serverGone = true; + throw new Error("remote_launch_ack_lost"); + }); + const oldHandle = createRemoteRunnerProcessLauncher({ ...launcherInput, runner: { execute: oldExecute } })(spec); + const successorSpawned = vi.fn<(meta: { pid: number }) => Promise>(async () => undefined); + let successorHandle: ReturnType> | undefined; + let orphanPid = 0; + try { + await expect(oldHandle.completion).rejects.toThrow("remote_launch_ack_lost"); + // The Runner the old server started is alive, and nothing recorded its PID. + await vi.waitFor(() => expect(runnersServing(stateDirectory)).toHaveLength(1), { timeout: 10_000 }); + orphanPid = runnersServing(stateDirectory)[0]!; + expect(oldHandle.child.pid).toBeUndefined(); + + // The successor relaunches the same run on the same sandbox. + successorHandle = createRemoteRunnerProcessLauncher({ ...launcherInput, runner: { execute: localExecute }, + onSpawn: successorSpawned })(spec); + void successorHandle.completion.catch(() => undefined); + await vi.waitFor(() => expect(successorSpawned).toHaveBeenCalledTimes(1), { timeout: 30_000 }); + const successorPid = successorSpawned.mock.calls[0]![0].pid; + + expect(successorPid).not.toBe(orphanPid); + await vi.waitFor(() => expect(alive(orphanPid)).toBe(false), { timeout: 10_000 }); + expect(runnersServing(stateDirectory)).toEqual([successorPid]); + expect((await readFile(launcherInput.processIdentityPath, "utf8")).split("\n")[1]).toBe(String(successorPid)); + } finally { + // Fresh /proc reads only: a saved PID may have been reused by now. + for (const pid of runnersServing(stateDirectory)) { + try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } + } + // The successor's supervision settles once it observes its Runner gone. + if (successorHandle) await Promise.race([successorHandle.completion.catch(() => undefined), + new Promise(resolve => setTimeout(resolve, 15_000))]); + await rm(root, { recursive: true, force: true }); + } + }, 60_000); +});