diff --git a/packages/adapter-utils/src/execution-target-stdin-race.test.ts b/packages/adapter-utils/src/execution-target-stdin-race.test.ts index 225adc00cc3..d86fa0d082e 100644 --- a/packages/adapter-utils/src/execution-target-stdin-race.test.ts +++ b/packages/adapter-utils/src/execution-target-stdin-race.test.ts @@ -766,6 +766,10 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", () // real "no usable creation time" filesystem is not reachable in this // sandbox, so the test simulates the exact Stats shape instead. fakeBirthtime?: { target: "sessionDir" | "stdinDir"; mode: "zero" | "followCtime" }; + // Extra environment variables for this wrapper process, merged in last. + // Used to drive the lease timing overrides directly, with no host bridge + // in front of the wrapper. + env?: Record; }) { const sessionDir = await mkdtemp(path.join(options?.parentDir ?? os.tmpdir(), "paperclip-wrapper-lifecycle-")); cleanupDirs.push(sessionDir); @@ -787,6 +791,7 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", () }; if (options?.maxRetries != null) env.PAPERCLIP_PROCESS_SESSION_STDIN_MAX_RETRIES = String(options.maxRetries); if (options?.terminateGraceMs != null) env.PAPERCLIP_PROCESS_SESSION_TERMINATE_GRACE_MS = String(options.terminateGraceMs); + if (options?.env) Object.assign(env, options.env); const execArgv: string[] = []; if (options?.fakeBirthtime) { @@ -1814,4 +1819,487 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", () } expect(wrapper.stderrText()).toMatch(/Latching on a lost process session identity/); }, 15_000); + + // ---- PAP-5361: a host-refreshed lease bounds the wrapper's own lifetime, + // so a host crash cannot orphan it. The host writes and refreshes + // ".paperclip-session-lease" under sessionDir; the wrapper reads only its + // filesystem metadata (device, inode, modification time) on a monotonic + // clock, and never its content. ----------------------------------------- + + const LEASE_FILE_NAME = ".paperclip-session-lease"; + + it("T21 no refresh: the wrapper terminates itself and its child after the lease duration elapses", async () => { + for (const outputToStdout of [false, true]) { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-no-refresh-")); + cleanupDirs.push(rootDir); + const pidFile = path.join(rootDir, "t21-child.pid"); + const childPath = path.join(rootDir, "t21-child.mjs"); + await writeFile(childPath, trackedChildSource(pidFile), "utf8"); + + const wrapper = await startWrapperProcess({ + outputToStdout, + command: process.execPath, + args: [childPath], + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400", + }, + }); + + const pid = await waitForTrackedChildPid(pidFile); + // No lease file is ever written under sessionDir. + await expectWrapperAndTrackedChildToDie(wrapper, pid); + expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/); + } + }, 20_000); + + it("T22 a valid rename-based refresh resets the deadline, and the wrapper stays alive past the lease duration", async () => { + for (const outputToStdout of [false, true]) { + const wrapper = await startWrapperProcess({ + outputToStdout, + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400", + }, + }); + const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME); + + let refreshing = true; + let refreshCount = 0; + const refresh = (async () => { + while (refreshing) { + // Mirrors the production shape: a fresh temp name, then a rename + // onto the fixed lease path. + const tempPath = `${leasePath}.${refreshCount}.tmp`; + await writeFile(tempPath, "", "utf8"); + await rename(tempPath, leasePath); + refreshCount += 1; + await delay(120); + } + })(); + + // Refresh faster than the 400ms lease duration, for more than twice + // the duration in total. A wrapper that ignored the refresh would + // already have expired well before this returns. + await delay(950); + refreshing = false; + await refresh; + + expect(refreshCount).toBeGreaterThan(2); + expect(wrapper.exitInfo().code).toBeNull(); + + // Let the wrapper expire on its own now that the refresh stopped, so + // the test leaves no lingering process. + await Promise.race([ + wrapper.exited, + delay(4_000).then(() => { + throw new Error("The wrapper did not expire once the refresh stopped."); + }), + ]); + } + }, 20_000); + + it("T23 a missing lease never resets the deadline, and ordinary stdin traffic does not substitute for it", async () => { + for (const outputToStdout of [false, true]) { + const wrapper = await startWrapperProcess({ + outputToStdout, + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400", + }, + }); + + // Ordinary stdin traffic while the lease stays absent. Writing and + // deleting a stdin file changes stdinDir's own metadata, not the + // lease's, so this must not reset the deadline. + const finalPath = path.join(wrapper.stdinDir, "000000000001.json"); + const tempPath = `${finalPath}.writing`; + await writeFile( + tempPath, + `${JSON.stringify({ type: "stdin", data: Buffer.from("hi", "utf8").toString("base64") })}\n`, + "utf8", + ); + await rename(tempPath, finalPath); + + await Promise.race([ + wrapper.exited, + delay(4_000).then(() => { + throw new Error("The wrapper did not expire with a missing lease."); + }), + ]); + expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/); + } + }, 20_000); + + it("T24 a symbolic link at the lease path is never followed, its target stays unchanged, and the wrapper expires", async () => { + const { symlink } = await import("node:fs/promises"); + for (const outputToStdout of [false, true]) { + const wrapper = await startWrapperProcess({ + outputToStdout, + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400", + }, + }); + const targetPath = path.join(wrapper.sessionDir, "lease-link-target"); + await writeFile(targetPath, "untouched", "utf8"); + const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME); + await symlink(targetPath, leasePath); + + await Promise.race([ + wrapper.exited, + delay(4_000).then(() => { + throw new Error("The wrapper did not expire with a symbolic link at the lease path."); + }), + ]); + expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/); + expect(await readFile(targetPath, "utf8")).toBe("untouched"); + } + }, 20_000); + + it("T25 a directory at the lease path is never treated as a valid lease, and the wrapper expires", async () => { + for (const outputToStdout of [false, true]) { + const wrapper = await startWrapperProcess({ + outputToStdout, + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400", + }, + }); + const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME); + await mkdir(leasePath, { recursive: true }); + + await Promise.race([ + wrapper.exited, + delay(4_000).then(() => { + throw new Error("The wrapper did not expire with a directory at the lease path."); + }), + ]); + expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/); + } + }, 20_000); + + // Pins the accepted residual (PAP-5354): a sandbox peer that + // touches the lease path defers this wrapper's expiry. This is not a + // capability the peer lacked -- it can already spawn its own long-lived + // work -- so a later change must not silently alter this shape. + it("T26 a peer that keeps touching the lease file defers this wrapper's expiry", async () => { + const { utimes } = await import("node:fs/promises"); + for (const outputToStdout of [false, true]) { + const wrapper = await startWrapperProcess({ + outputToStdout, + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400", + }, + }); + const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME); + await writeFile(leasePath, "", "utf8"); + + let churning = true; + const churn = (async () => { + while (churning) { + await utimes(leasePath, new Date(), new Date()).catch(() => undefined); + await delay(120); + } + })(); + + // Keep touching for more than twice the lease duration; a wrapper + // that ignored the touch would already have expired. + await delay(950); + churning = false; + await churn; + expect(wrapper.exitInfo().code).toBeNull(); + + // Stop touching and let the wrapper expire normally, so the test + // leaves no lingering process. + await Promise.race([ + wrapper.exited, + delay(4_000).then(() => { + throw new Error("The wrapper did not expire once the peer stopped touching the lease."); + }), + ]); + } + }, 20_000); + + it("T27 each wrapper source has exactly one lease deadline declaration and one lease check call site, and the lease code adds no kill caller or numeric process identifier", () => { + for (const outputToStdout of [true, false]) { + const src = getProcessSessionRemoteSource({ outputToStdout }); + const code = src + .split("\n") + .map((line) => line.replace(/\/\/.*$/, "")) + .join("\n"); + + const leaseDeadlineDeclarations = code.match(/\blet leaseDeadline = /g) ?? []; + expect(leaseDeadlineDeclarations.length).toBe(1); + + const leaseCheckCallSites = code.match(/\bawait checkProcessSessionLease\(\)/g) ?? []; + expect(leaseCheckCallSites.length).toBe(1); + expect(code.indexOf("await checkProcessSessionLease();")).toBeLessThan(code.indexOf("await readStdinDirNames()")); + + const killCallSites = [...code.matchAll(/[A-Za-z0-9_.$]*kill\(/g)].map((match) => match[0]); + expect(killCallSites.length).toBeGreaterThan(0); + for (const site of killCallSites) { + expect(site).toBe("child.kill("); + } + + const leaseSectionStart = code.indexOf("const leaseFileName ="); + const leaseSectionEnd = code.indexOf("async function terminate()"); + expect(leaseSectionStart).toBeGreaterThan(0); + expect(leaseSectionEnd).toBeGreaterThan(leaseSectionStart); + const leaseSection = code.slice(leaseSectionStart, leaseSectionEnd); + expect(leaseSection).not.toContain("kill("); + expect(leaseSection).not.toMatch(/\bpid\b/i); + } + }); + + it("T28 an injected slow lease refresh in flight during stop() cannot recreate sessionDir after removal", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-host-race-")); + cleanupDirs.push(rootDir); + const childPath = path.join(rootDir, "quiet-child.mjs"); + await writeFile(childPath, "process.stdin.resume();\n", "utf8"); + + const completionOrder: string[] = []; + let counter = 0; + let sessionDir = ""; + const runner = { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + }): Promise => { + const script = input.args?.[1] ?? ""; + if (script.startsWith("mkdir -p") && !sessionDir) { + const dirsMatch = /mkdir -p '([^']+)' '([^']+)'/.exec(script); + if (dirsMatch) sessionDir = path.posix.dirname(dirsMatch[1]); + } + if (script.includes("nohup node")) { + return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null }; + } + const isLeaseRefresh = script.includes("set -C") && script.includes(LEASE_FILE_NAME); + const isRemove = script.trim().startsWith("rm -rf"); + if (isLeaseRefresh) { + // Simulate a slow remote exec for the refresh command: it is + // already in flight when stop() runs below. + await delay(500); + } + counter += 1; + const command = + input.command === "bash" ? "/bin/bash" : input.command === "sh" ? "/bin/sh" : input.command; + const result = await runChildProcess(`lease-race-run-${counter}`, command, input.args ?? [], { + cwd: input.cwd ?? process.cwd(), + env: input.env ?? {}, + stdin: input.stdin, + timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)), + graceSec: 5, + onLog: input.onLog ?? (async () => {}), + }); + if (isLeaseRefresh) completionOrder.push("refresh"); + if (isRemove) completionOrder.push("remove"); + return result; + }, + }; + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "local-test", + remoteCwd: rootDir, + timeoutMs: 30_000, + runner, + }; + + const bridge = await startAdapterExecutionTargetProcessSessionBridge({ + runId: "run-lease-race", + target, + runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"), + adapterKey: "acpx", + command: process.execPath, + args: [childPath], + cwd: rootDir, + env: {}, + timeoutSec: 5, + onLog: async () => {}, + }); + expect(bridge).not.toBeNull(); + + // Give the initial lease write (fired right after launch) time to start + // before stop() runs, so this exercises a refresh already in flight, not + // one stop() outruns before it ever starts. + await waitFor(() => completionOrder.length === 0 && sessionDir !== "", 4_000); + await delay(50); + await bridge!.stop(); + + // stop() awaits the whole refresh chain before it removes sessionDir, so + // the delayed refresh's own `mv` always finishes first. + expect(completionOrder).toEqual(["refresh", "remove"]); + + expect(sessionDir).not.toBe(""); + const stillThere = await readdir(sessionDir).catch(() => null); + expect(stillThere).toBeNull(); + }, 15_000); + + it("T29 the lease refresh command carries bypassSession and contains no mkdir", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-command-shape-")); + cleanupDirs.push(rootDir); + const childPath = path.join(rootDir, "quiet-child.mjs"); + await writeFile(childPath, "process.stdin.resume();\n", "utf8"); + + const calls: Array<{ script: string; bypassSession?: boolean }> = []; + let counter = 0; + const runner = { + execute: async (input: { + command: string; + args?: string[]; + cwd?: string; + env?: Record; + stdin?: string; + timeoutMs?: number; + onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise; + bypassSession?: boolean; + }): Promise => { + const script = input.args?.[1] ?? ""; + calls.push({ script, bypassSession: input.bypassSession }); + if (script.includes("nohup node")) { + return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null }; + } + counter += 1; + const command = + input.command === "bash" ? "/bin/bash" : input.command === "sh" ? "/bin/sh" : input.command; + return runChildProcess(`lease-shape-run-${counter}`, command, input.args ?? [], { + cwd: input.cwd ?? process.cwd(), + env: input.env ?? {}, + stdin: input.stdin, + timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)), + graceSec: 5, + onLog: input.onLog ?? (async () => {}), + }); + }, + }; + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "local-test", + remoteCwd: rootDir, + timeoutMs: 30_000, + runner, + }; + + const bridge = await startAdapterExecutionTargetProcessSessionBridge({ + runId: "run-lease-shape", + target, + runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"), + adapterKey: "acpx", + command: process.execPath, + args: [childPath], + cwd: rootDir, + env: {}, + timeoutSec: 5, + onLog: async () => {}, + }); + expect(bridge).not.toBeNull(); + await waitFor(() => calls.some((call) => call.script.includes("set -C")), 4_000); + await bridge!.stop(); + + const refreshCalls = calls.filter((call) => call.script.includes("set -C")); + expect(refreshCalls.length).toBeGreaterThan(0); + for (const call of refreshCalls) { + expect(call.bypassSession).toBe(true); + expect(call.script).not.toContain("mkdir"); + expect(call.script).toContain("mv "); + } + }, 10_000); + + it("T30 validation: a zero, a negative, a non-numeric, and a D<=P lease override pair all fall back to the defaults on the host and in the wrapper", async () => { + const invalidPairs: Array<[string, string]> = [ + ["0", "900000"], + ["-1000", "900000"], + ["not-a-number", "900000"], + ["600000", "300000"], + ]; + + for (const [refreshRaw, ttlRaw] of invalidPairs) { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-validate-")); + cleanupDirs.push(rootDir); + const scripts: string[] = []; + const runner = createLocalSandboxRunner(async (script) => { + scripts.push(script); + }); + const target: AdapterSandboxExecutionTarget = { + kind: "remote", + transport: "sandbox", + providerKey: "local-test", + remoteCwd: rootDir, + timeoutMs: 30_000, + runner, + }; + + const prevRefresh = process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS; + const prevTtl = process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS; + process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS = refreshRaw; + process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS = ttlRaw; + + let bridge; + try { + bridge = await startAdapterExecutionTargetProcessSessionBridge({ + runId: `run-validate-${refreshRaw}-${ttlRaw}`, + target, + runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"), + adapterKey: "acpx", + command: process.execPath, + args: ["-e", "process.stdin.resume();"], + cwd: rootDir, + env: {}, + timeoutSec: 5, + onLog: async () => {}, + }); + } finally { + if (prevRefresh === undefined) delete process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS; + else process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS = prevRefresh; + if (prevTtl === undefined) delete process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS; + else process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS = prevTtl; + } + expect(bridge).not.toBeNull(); + + const nohupScript = scripts.find((script) => script.includes("nohup node")); + expect(nohupScript).toBeDefined(); + expect(nohupScript).toContain("PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS='60000'"); + expect(nohupScript).toContain("PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS='900000'"); + + await bridge!.stop(); + } + + // The wrapper side: an invalid override pair (P invalid, D alone would + // be a very short deadline if the wrapper only clamped D and ignored an + // invalid P) must fall the whole pair back to the 900s default, not a + // 50ms deadline. + const wrapper = await startWrapperProcess({ + outputToStdout: false, + terminateGraceMs: 200, + env: { + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "0", + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "50", + }, + }); + await delay(400); + expect(wrapper.exitInfo().code).toBeNull(); + // Force a clean exit through the existing session-identity latch, + // instead of leaving the wrapper alive for the full 900s default. + await rm(wrapper.sessionDir, { recursive: true, force: true }); + await Promise.race([ + wrapper.exited, + delay(4_000).then(() => { + throw new Error("The wrapper did not exit after its session directory disappeared."); + }), + ]); + }, 30_000); }); diff --git a/packages/adapter-utils/src/execution-target.ts b/packages/adapter-utils/src/execution-target.ts index 801af1ab554..6406e48403a 100644 --- a/packages/adapter-utils/src/execution-target.ts +++ b/packages/adapter-utils/src/execution-target.ts @@ -1603,6 +1603,39 @@ const PROCESS_SESSION_AUTH_TIMEOUT_MS = 5_000; // not the child's full shutdown. const DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS = 3_000; +// The host-refreshed lease that bounds the wrapper's own lifetime, so a host +// crash cannot orphan it. See PROCESS_SESSION_STDIN_POLL_TAIL below for the +// sandbox side of this contract: the wrapper reads the lease as filesystem +// metadata only, never as content, and it never creates, opens, writes, or +// removes the lease file itself. This host writes and refreshes the lease; it +// never reads the lease back and never gates `sessionDir` cleanup on it, so +// this adds no new sandbox-to-host data flow. +const DEFAULT_PROCESS_SESSION_LEASE_REFRESH_MS = 60_000; +const DEFAULT_PROCESS_SESSION_LEASE_TTL_MS = 900_000; +// The name does not end in ".json", so no `.json`-only reader (the stdin +// poller) ever lists it. +const PROCESS_SESSION_LEASE_FILE_NAME = ".paperclip-session-lease"; + +// Reads the refresh period and the lease duration from the environment. +// Accept an override pair only when both values are positive finite integers +// AND the duration exceeds the refresh period; reject the pair to both +// defaults together, so an accepted override never pairs with a rejected one +// into an unvalidated combination. The wrapper validates its own copy of the +// resolved values with the same rule (see PROCESS_SESSION_STDIN_POLL_TAIL). +function resolveProcessSessionLeaseTiming(): { refreshMs: number; ttlMs: number } { + const parsePositiveInt = (raw: string | undefined): number | null => { + if (!raw) return null; + const value = Number(raw); + return Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : null; + }; + const refreshOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS); + const ttlOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS); + if (refreshOverride !== null && ttlOverride !== null && ttlOverride > refreshOverride) { + return { refreshMs: refreshOverride, ttlMs: ttlOverride }; + } + return { refreshMs: DEFAULT_PROCESS_SESSION_LEASE_REFRESH_MS, ttlMs: DEFAULT_PROCESS_SESSION_LEASE_TTL_MS }; +} + function jsonLine(value: unknown): string { return `${JSON.stringify(value)}\n`; } @@ -1749,6 +1782,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { const sessionDir = path.posix.join(bridgeRuntimeDir, sessionId); const stdinDir = path.posix.join(sessionDir, "stdin"); const eventsDir = path.posix.join(sessionDir, "events"); + const leaseTiming = resolveProcessSessionLeaseTiming(); + const leasePath = path.posix.join(sessionDir, PROCESS_SESSION_LEASE_FILE_NAME); // The streamed wrapper writes its frames to stdout and rides a separate remote // path, so a warm sandbox can hold both wrapper scripts without the content // hash-skip gate thrashing when a run switches output mode. @@ -1802,6 +1837,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { // wrapper and let it go; do not capture `$!`. `PAPERCLIP_PROCESS_SESSION_DIR=${shellQuote(sessionDir)} ` + `PAPERCLIP_PROCESS_SESSION_COMMAND_B64=${shellQuote(commandPayload)} ` + + `PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS=${shellQuote(String(leaseTiming.refreshMs))} ` + + `PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS=${shellQuote(String(leaseTiming.ttlMs))} ` + `nohup node ${shellQuote(remoteScriptPath)} >/dev/null 2>&1 < /dev/null &`, ].join("\n"), ), @@ -1838,6 +1875,66 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { // corrupts a large prompt on the stdin path. let stdinWriteChain: Promise = Promise.resolve(); let pollTimer: NodeJS.Timeout | null = null; + // Set permanently by `stop()`, before it clears the timer below and awaits + // the refresh chain. Once true, a refresh that is already chained (queued + // by the last timer tick before `stop()` ran) still resolves, but it + // returns without executing the remote command -- so no refresh can start + // after `stop()` begins removing `sessionDir`. + let leaseRefreshClosed = false; + let leaseRefreshTimer: NodeJS.Timeout | null = null; + // Serializes every refresh onto one chain, so `stop()` can await the whole + // active/queued chain before it removes `sessionDir`. Two concurrent + // refreshes could otherwise race a `stop()` removal and recreate + // `sessionDir` right after cleanup -- a fresh orphan on the exact path + // PAP-5307 made deterministic. + let leaseRefreshChain: Promise = Promise.resolve(); + const refreshProcessSessionLeaseOnce = async (): Promise => { + if (leaseRefreshClosed) return; + // A fresh, unpredictable temporary name per refresh, so a peer cannot + // pre-create it and defeat the exclusive create below. + const tempPath = `${leasePath}.${randomBytes(9).toString("hex")}.tmp`; + const script = [ + // `set -C` (noclobber) makes the shell open the redirect target with + // O_CREAT|O_EXCL: it fails on an existing file and on a symbolic + // link, so it can neither follow a link nor truncate a peer's file. + "set -C", + `: > ${shellQuote(tempPath)}`, + // `mv` replaces the lease path by rename. A rename does not follow a + // symbolic link at the target, so a peer that pre-creates a link at + // the fixed lease path cannot redirect this write. This command + // contains no `mkdir`: when `stop()` already removed `sessionDir`, + // the create above fails, this refresh fails, and the wrapper + // expires -- the wanted outcome, not a defect. + `mv ${shellQuote(tempPath)} ${shellQuote(leasePath)}`, + ].join("\n"); + try { + await runner.execute({ + command: shellCommand, + args: shellCommandArgs(script), + cwd: target.remoteCwd, + env: { PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge" }, + timeoutMs, + // A busy persistent session must never queue this behind an + // in-run session command and starve the lease. + bypassSession: true, + }); + } catch { + // Best-effort and non-fatal: a failed refresh leaves the wrapper's + // deadline unreset this cycle. Never retry with a following write; + // never throw out of the timer. + } + }; + const queueProcessSessionLeaseRefresh = (): void => { + leaseRefreshChain = leaseRefreshChain.then(refreshProcessSessionLeaseOnce, refreshProcessSessionLeaseOnce); + }; + const scheduleProcessSessionLeaseRefresh = (): void => { + if (leaseRefreshClosed) return; + leaseRefreshTimer = setTimeout(() => { + queueProcessSessionLeaseRefresh(); + scheduleProcessSessionLeaseRefresh(); + }, leaseTiming.refreshMs); + leaseRefreshTimer.unref?.(); + }; const pendingRemoteEvents: Array<{ type?: string; stream?: "stdout" | "stderr"; @@ -2127,6 +2224,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { env: { PAPERCLIP_PROCESS_SESSION_DIR: sessionDir, PAPERCLIP_PROCESS_SESSION_COMMAND_B64: streamCommandPayload, + PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: String(leaseTiming.refreshMs), + PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: String(leaseTiming.ttlMs), PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge", }, timeoutMs, @@ -2157,6 +2256,15 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { schedulePoll(); } + // Write the lease once at bridge start, on both variants, then keep it + // refreshing on an unreferenced timer. The event-file variant's `mkdir -p` + // launch exec above already created `sessionDir`; the streamed variant's + // wrapper creates it inside the sandbox, so this first write can race that + // creation and fail. That failure is harmless: it leaves the deadline + // unreset for one cycle, well inside the default 900s lease duration. + queueProcessSessionLeaseRefresh(); + scheduleProcessSessionLeaseRefresh(); + // `stop()` cannot rely on the long-lived poll above to observe a late // `shutdownAck`: that poll stops re-arming as soon as it forwards a // terminal `exit`/`error` event, and `stop()` itself sets `stopping` on @@ -2198,6 +2306,13 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { agentCommand, stop: async () => { stopping = true; + // Close the lease refresh state permanently and clear its timer first, + // before any other teardown step. A refresh already chained by the + // last timer tick still resolves, but `refreshProcessSessionLeaseOnce` + // reads this flag and returns without running the remote command, so + // no refresh can start after this line. + leaseRefreshClosed = true; + if (leaseRefreshTimer) clearTimeout(leaseRefreshTimer); // End the `sandbox.agentProcess` span now, before the caller ends the run // root span, even if the remote command has not resolved yet. signalStopped(); @@ -2261,6 +2376,14 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: { `[paperclip] ACP process session wrapper did not acknowledge shutdown within ${DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS}ms; removing the session directory anyway.\n`, ).catch(() => undefined); } + // Await the entire active/queued lease refresh chain before the + // removal below. `leaseRefreshClosed` (set at the top of `stop()`) + // already stops a chained refresh from running its remote command, but + // an in-flight one (its exec already sent) still needs to settle + // first, or its `mv` could land after this removal and recreate + // `sessionDir` -- a fresh orphan on the exact path PAP-5307 made + // deterministic. + await leaseRefreshChain.catch(() => undefined); // Unconditional: this removal runs whether or not the wrapper // acknowledged, and whether or not any event (real or forged) arrived // under `sessionDir`. `stop()` runs during run teardown and must stay @@ -2363,6 +2486,90 @@ const terminateGraceMs = (() => { return Number.isFinite(raw) && raw > 0 ? raw : 3000; })(); +// Host lease that bounds this wrapper's own lifetime when a host crash +// leaves sessionDir intact. The host writes and refreshes +// ".paperclip-session-lease" under sessionDir; this wrapper never creates, +// opens, writes, or removes that file. It reads only the file's OS-level +// metadata (device, inode, and modification time), never its content: a +// content timestamp would compare the host clock against the sandbox +// clock, and a clock step would then cause a false expiry or a false +// extension. The deadline runs on a MONOTONIC clock +// (process.hrtime.bigint()), so it is immune to both a wall-clock step and +// an NTP adjustment. +// +// Accepted residuals (PAP-5354): a sandbox peer with access to +// sessionDir can touch the lease path and defer this wrapper's expiry, or +// remove the lease path and force an earlier expiry. Neither grants a +// capability the peer lacks: it can already spawn its own long-lived work, +// and it can already force a faster kill by deleting sessionDir outright +// (the session-identity latch elsewhere in this wrapper already covers that +// path). A peer that replaces sessionDir with a symbolic link still makes +// the host's exclusive create land an empty, unpredictably named file in +// the link target; the create truncates nothing and writes no content, and +// the session-identity latch terminates this wrapper on that same event. A +// peer that puts a directory at the lease path makes the host's \`mv\` move +// its temporary file into that directory instead of replacing the lease +// path, so this wrapper's own \`lstat\` sees a directory, never resets the +// deadline, and this wrapper still expires -- an availability loss, not an +// escape. +const leaseFileName = ".paperclip-session-lease"; +const leasePath = path.posix.join(sessionDir, leaseFileName); +const defaultLeaseRefreshMs = 60_000; +const defaultLeaseTtlMs = 900_000; + +// Validated the same way as the host resolver: accept the override pair +// only when both values are positive finite integers AND the duration +// exceeds the refresh period. Reject the pair to both defaults together, so +// one accepted override never combines with one rejected override into an +// unvalidated pair. +function resolveLeaseTiming() { + function parsePositiveInt(raw) { + if (!raw) return null; + const value = Number(raw); + return Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : null; + } + const refreshOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS); + const ttlOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS); + if (refreshOverride !== null && ttlOverride !== null && ttlOverride > refreshOverride) { + return { refreshMs: refreshOverride, ttlMs: ttlOverride }; + } + return { refreshMs: defaultLeaseRefreshMs, ttlMs: defaultLeaseTtlMs }; +} +const leaseTiming = resolveLeaseTiming(); + +// The deadline starts at "now plus the lease duration", before the first +// poll cycle, so a lease that never appears (a slow or crashed host) still +// bounds this wrapper. \`leaseStamp\` holds the last observed identity of the +// lease file; only a changed stamp resets the deadline. +let leaseDeadline = process.hrtime.bigint() + BigInt(leaseTiming.ttlMs) * 1000000n; +let leaseStamp = null; + +// Runs at the top of every poll cycle, before this wrapper reads stdinDir. +// This wrapper never creates, opens, writes, or removes the lease file -- +// it only lstats it. Any lstat error (a missing file, a permission error, or +// any other failure) and any non-regular-file object (a symbolic link, a +// directory, a socket) leave the deadline untouched: neither resets it nor +// expires it early. Only a changed identity on a regular file resets it. +async function checkProcessSessionLease() { + let stats; + try { + stats = await fs.lstat(leasePath); + } catch { + return; + } + if (!stats.isFile()) return; + const stamp = { dev: stats.dev, ino: stats.ino, mtimeMs: stats.mtimeMs }; + if ( + !leaseStamp || + stamp.dev !== leaseStamp.dev || + stamp.ino !== leaseStamp.ino || + stamp.mtimeMs !== leaseStamp.mtimeMs + ) { + leaseStamp = stamp; + leaseDeadline = process.hrtime.bigint() + BigInt(leaseTiming.ttlMs) * 1000000n; + } +} + // I2: terminate() is the only function in this wrapper that calls // child.kill(). No child event handler and no sibling callback calls it. // terminate() is idempotent: a second call, or a first call after the child @@ -2620,6 +2827,15 @@ async function readStdinDirNames() { async function pollStdin() { while (!shuttingDown) { + // Check the lease before this cycle reads stdinDir. This reuses + // latchAndTerminate() -- the same one-shot gate the session-identity + // check above uses -- so this wrapper still has exactly one termination + // path and no new child.kill() caller. + await checkProcessSessionLease(); + if (process.hrtime.bigint() >= leaseDeadline) { + process.stderr.write("Process session lease expired; terminating.\\n"); + await latchAndTerminate(); + } const entries = (await readStdinDirNames()).filter((name) => name.endsWith(".json")).sort(); for (const name of entries) { if (shuttingDown) break;