fix(adapter-utils): bound the process-session wrapper with a host lease

A host crash left the scratch root on disk, so the remote process-session
wrapper polled its unchanged control directory forever and kept its child
alive. The host now writes and refreshes a lease file under sessionDir on
an unreferenced timer; the wrapper reads the lease as filesystem metadata
only (device, inode, modification time) on a monotonic clock, and
terminates itself once the lease stops moving past its duration.

Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Priya RamanandPaperclip committed 2026-08-26 23:06:46 +00:00
1 parent 4277ecbb2e
commit 97879de951
2 files changed
+704

No files matched your search

@@ -766,6 +766,10 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", ()
// real "no usable creation time" filesystem is not reachable in this
// sandbox, so the test simulates the exact Stats shape instead.
fakeBirthtime?: { target: "sessionDir" | "stdinDir"; mode: "zero" | "followCtime" };
// Extra environment variables for this wrapper process, merged in last.
// Used to drive the lease timing overrides directly, with no host bridge
// in front of the wrapper.
env?: Record<string, string>;
}) {
const sessionDir = await mkdtemp(path.join(options?.parentDir ?? os.tmpdir(), "paperclip-wrapper-lifecycle-"));
cleanupDirs.push(sessionDir);
@@ -787,6 +791,7 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", ()
};
if (options?.maxRetries != null) env.PAPERCLIP_PROCESS_SESSION_STDIN_MAX_RETRIES = String(options.maxRetries);
if (options?.terminateGraceMs != null) env.PAPERCLIP_PROCESS_SESSION_TERMINATE_GRACE_MS = String(options.terminateGraceMs);
if (options?.env) Object.assign(env, options.env);
const execArgv: string[] = [];
if (options?.fakeBirthtime) {
@@ -1814,4 +1819,487 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", ()
}
expect(wrapper.stderrText()).toMatch(/Latching on a lost process session identity/);
}, 15_000);
// ---- PAP-5361: a host-refreshed lease bounds the wrapper's own lifetime,
// so a host crash cannot orphan it. The host writes and refreshes
// ".paperclip-session-lease" under sessionDir; the wrapper reads only its
// filesystem metadata (device, inode, modification time) on a monotonic
// clock, and never its content. -----------------------------------------
const LEASE_FILE_NAME = ".paperclip-session-lease";
it("T21 no refresh: the wrapper terminates itself and its child after the lease duration elapses", async () => {
for (const outputToStdout of [false, true]) {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-no-refresh-"));
cleanupDirs.push(rootDir);
const pidFile = path.join(rootDir, "t21-child.pid");
const childPath = path.join(rootDir, "t21-child.mjs");
await writeFile(childPath, trackedChildSource(pidFile), "utf8");
const wrapper = await startWrapperProcess({
outputToStdout,
command: process.execPath,
args: [childPath],
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
},
});
const pid = await waitForTrackedChildPid(pidFile);
// No lease file is ever written under sessionDir.
await expectWrapperAndTrackedChildToDie(wrapper, pid);
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
}
}, 20_000);
it("T22 a valid rename-based refresh resets the deadline, and the wrapper stays alive past the lease duration", async () => {
for (const outputToStdout of [false, true]) {
const wrapper = await startWrapperProcess({
outputToStdout,
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
},
});
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
let refreshing = true;
let refreshCount = 0;
const refresh = (async () => {
while (refreshing) {
// Mirrors the production shape: a fresh temp name, then a rename
// onto the fixed lease path.
const tempPath = `${leasePath}.${refreshCount}.tmp`;
await writeFile(tempPath, "", "utf8");
await rename(tempPath, leasePath);
refreshCount += 1;
await delay(120);
}
})();
// Refresh faster than the 400ms lease duration, for more than twice
// the duration in total. A wrapper that ignored the refresh would
// already have expired well before this returns.
await delay(950);
refreshing = false;
await refresh;
expect(refreshCount).toBeGreaterThan(2);
expect(wrapper.exitInfo().code).toBeNull();
// Let the wrapper expire on its own now that the refresh stopped, so
// the test leaves no lingering process.
await Promise.race([
wrapper.exited,
delay(4_000).then(() => {
throw new Error("The wrapper did not expire once the refresh stopped.");
}),
]);
}
}, 20_000);
it("T23 a missing lease never resets the deadline, and ordinary stdin traffic does not substitute for it", async () => {
for (const outputToStdout of [false, true]) {
const wrapper = await startWrapperProcess({
outputToStdout,
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
},
});
// Ordinary stdin traffic while the lease stays absent. Writing and
// deleting a stdin file changes stdinDir's own metadata, not the
// lease's, so this must not reset the deadline.
const finalPath = path.join(wrapper.stdinDir, "000000000001.json");
const tempPath = `${finalPath}.writing`;
await writeFile(
tempPath,
`${JSON.stringify({ type: "stdin", data: Buffer.from("hi", "utf8").toString("base64") })}\n`,
"utf8",
);
await rename(tempPath, finalPath);
await Promise.race([
wrapper.exited,
delay(4_000).then(() => {
throw new Error("The wrapper did not expire with a missing lease.");
}),
]);
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
}
}, 20_000);
it("T24 a symbolic link at the lease path is never followed, its target stays unchanged, and the wrapper expires", async () => {
const { symlink } = await import("node:fs/promises");
for (const outputToStdout of [false, true]) {
const wrapper = await startWrapperProcess({
outputToStdout,
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
},
});
const targetPath = path.join(wrapper.sessionDir, "lease-link-target");
await writeFile(targetPath, "untouched", "utf8");
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
await symlink(targetPath, leasePath);
await Promise.race([
wrapper.exited,
delay(4_000).then(() => {
throw new Error("The wrapper did not expire with a symbolic link at the lease path.");
}),
]);
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
expect(await readFile(targetPath, "utf8")).toBe("untouched");
}
}, 20_000);
it("T25 a directory at the lease path is never treated as a valid lease, and the wrapper expires", async () => {
for (const outputToStdout of [false, true]) {
const wrapper = await startWrapperProcess({
outputToStdout,
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
},
});
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
await mkdir(leasePath, { recursive: true });
await Promise.race([
wrapper.exited,
delay(4_000).then(() => {
throw new Error("The wrapper did not expire with a directory at the lease path.");
}),
]);
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
}
}, 20_000);
// Pins the accepted residual (PAP-5354): a sandbox peer that
// touches the lease path defers this wrapper's expiry. This is not a
// capability the peer lacked -- it can already spawn its own long-lived
// work -- so a later change must not silently alter this shape.
it("T26 a peer that keeps touching the lease file defers this wrapper's expiry", async () => {
const { utimes } = await import("node:fs/promises");
for (const outputToStdout of [false, true]) {
const wrapper = await startWrapperProcess({
outputToStdout,
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
},
});
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
await writeFile(leasePath, "", "utf8");
let churning = true;
const churn = (async () => {
while (churning) {
await utimes(leasePath, new Date(), new Date()).catch(() => undefined);
await delay(120);
}
})();
// Keep touching for more than twice the lease duration; a wrapper
// that ignored the touch would already have expired.
await delay(950);
churning = false;
await churn;
expect(wrapper.exitInfo().code).toBeNull();
// Stop touching and let the wrapper expire normally, so the test
// leaves no lingering process.
await Promise.race([
wrapper.exited,
delay(4_000).then(() => {
throw new Error("The wrapper did not expire once the peer stopped touching the lease.");
}),
]);
}
}, 20_000);
it("T27 each wrapper source has exactly one lease deadline declaration and one lease check call site, and the lease code adds no kill caller or numeric process identifier", () => {
for (const outputToStdout of [true, false]) {
const src = getProcessSessionRemoteSource({ outputToStdout });
const code = src
.split("\n")
.map((line) => line.replace(/\/\/.*$/, ""))
.join("\n");
const leaseDeadlineDeclarations = code.match(/\blet leaseDeadline = /g) ?? [];
expect(leaseDeadlineDeclarations.length).toBe(1);
const leaseCheckCallSites = code.match(/\bawait checkProcessSessionLease\(\)/g) ?? [];
expect(leaseCheckCallSites.length).toBe(1);
expect(code.indexOf("await checkProcessSessionLease();")).toBeLessThan(code.indexOf("await readStdinDirNames()"));
const killCallSites = [...code.matchAll(/[A-Za-z0-9_.$]*kill\(/g)].map((match) => match[0]);
expect(killCallSites.length).toBeGreaterThan(0);
for (const site of killCallSites) {
expect(site).toBe("child.kill(");
}
const leaseSectionStart = code.indexOf("const leaseFileName =");
const leaseSectionEnd = code.indexOf("async function terminate()");
expect(leaseSectionStart).toBeGreaterThan(0);
expect(leaseSectionEnd).toBeGreaterThan(leaseSectionStart);
const leaseSection = code.slice(leaseSectionStart, leaseSectionEnd);
expect(leaseSection).not.toContain("kill(");
expect(leaseSection).not.toMatch(/\bpid\b/i);
}
});
it("T28 an injected slow lease refresh in flight during stop() cannot recreate sessionDir after removal", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-host-race-"));
cleanupDirs.push(rootDir);
const childPath = path.join(rootDir, "quiet-child.mjs");
await writeFile(childPath, "process.stdin.resume();\n", "utf8");
const completionOrder: string[] = [];
let counter = 0;
let sessionDir = "";
const runner = {
execute: async (input: {
command: string;
args?: string[];
cwd?: string;
env?: Record<string, string>;
stdin?: string;
timeoutMs?: number;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
}): Promise<RunProcessResult> => {
const script = input.args?.[1] ?? "";
if (script.startsWith("mkdir -p") && !sessionDir) {
const dirsMatch = /mkdir -p '([^']+)' '([^']+)'/.exec(script);
if (dirsMatch) sessionDir = path.posix.dirname(dirsMatch[1]);
}
if (script.includes("nohup node")) {
return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null };
}
const isLeaseRefresh = script.includes("set -C") && script.includes(LEASE_FILE_NAME);
const isRemove = script.trim().startsWith("rm -rf");
if (isLeaseRefresh) {
// Simulate a slow remote exec for the refresh command: it is
// already in flight when stop() runs below.
await delay(500);
}
counter += 1;
const command =
input.command === "bash" ? "/bin/bash" : input.command === "sh" ? "/bin/sh" : input.command;
const result = await runChildProcess(`lease-race-run-${counter}`, command, input.args ?? [], {
cwd: input.cwd ?? process.cwd(),
env: input.env ?? {},
stdin: input.stdin,
timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),
graceSec: 5,
onLog: input.onLog ?? (async () => {}),
});
if (isLeaseRefresh) completionOrder.push("refresh");
if (isRemove) completionOrder.push("remove");
return result;
},
};
const target: AdapterSandboxExecutionTarget = {
kind: "remote",
transport: "sandbox",
providerKey: "local-test",
remoteCwd: rootDir,
timeoutMs: 30_000,
runner,
};
const bridge = await startAdapterExecutionTargetProcessSessionBridge({
runId: "run-lease-race",
target,
runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"),
adapterKey: "acpx",
command: process.execPath,
args: [childPath],
cwd: rootDir,
env: {},
timeoutSec: 5,
onLog: async () => {},
});
expect(bridge).not.toBeNull();
// Give the initial lease write (fired right after launch) time to start
// before stop() runs, so this exercises a refresh already in flight, not
// one stop() outruns before it ever starts.
await waitFor(() => completionOrder.length === 0 && sessionDir !== "", 4_000);
await delay(50);
await bridge!.stop();
// stop() awaits the whole refresh chain before it removes sessionDir, so
// the delayed refresh's own `mv` always finishes first.
expect(completionOrder).toEqual(["refresh", "remove"]);
expect(sessionDir).not.toBe("");
const stillThere = await readdir(sessionDir).catch(() => null);
expect(stillThere).toBeNull();
}, 15_000);
it("T29 the lease refresh command carries bypassSession and contains no mkdir", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-command-shape-"));
cleanupDirs.push(rootDir);
const childPath = path.join(rootDir, "quiet-child.mjs");
await writeFile(childPath, "process.stdin.resume();\n", "utf8");
const calls: Array<{ script: string; bypassSession?: boolean }> = [];
let counter = 0;
const runner = {
execute: async (input: {
command: string;
args?: string[];
cwd?: string;
env?: Record<string, string>;
stdin?: string;
timeoutMs?: number;
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
bypassSession?: boolean;
}): Promise<RunProcessResult> => {
const script = input.args?.[1] ?? "";
calls.push({ script, bypassSession: input.bypassSession });
if (script.includes("nohup node")) {
return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null };
}
counter += 1;
const command =
input.command === "bash" ? "/bin/bash" : input.command === "sh" ? "/bin/sh" : input.command;
return runChildProcess(`lease-shape-run-${counter}`, command, input.args ?? [], {
cwd: input.cwd ?? process.cwd(),
env: input.env ?? {},
stdin: input.stdin,
timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),
graceSec: 5,
onLog: input.onLog ?? (async () => {}),
});
},
};
const target: AdapterSandboxExecutionTarget = {
kind: "remote",
transport: "sandbox",
providerKey: "local-test",
remoteCwd: rootDir,
timeoutMs: 30_000,
runner,
};
const bridge = await startAdapterExecutionTargetProcessSessionBridge({
runId: "run-lease-shape",
target,
runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"),
adapterKey: "acpx",
command: process.execPath,
args: [childPath],
cwd: rootDir,
env: {},
timeoutSec: 5,
onLog: async () => {},
});
expect(bridge).not.toBeNull();
await waitFor(() => calls.some((call) => call.script.includes("set -C")), 4_000);
await bridge!.stop();
const refreshCalls = calls.filter((call) => call.script.includes("set -C"));
expect(refreshCalls.length).toBeGreaterThan(0);
for (const call of refreshCalls) {
expect(call.bypassSession).toBe(true);
expect(call.script).not.toContain("mkdir");
expect(call.script).toContain("mv ");
}
}, 10_000);
it("T30 validation: a zero, a negative, a non-numeric, and a D<=P lease override pair all fall back to the defaults on the host and in the wrapper", async () => {
const invalidPairs: Array<[string, string]> = [
["0", "900000"],
["-1000", "900000"],
["not-a-number", "900000"],
["600000", "300000"],
];
for (const [refreshRaw, ttlRaw] of invalidPairs) {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-validate-"));
cleanupDirs.push(rootDir);
const scripts: string[] = [];
const runner = createLocalSandboxRunner(async (script) => {
scripts.push(script);
});
const target: AdapterSandboxExecutionTarget = {
kind: "remote",
transport: "sandbox",
providerKey: "local-test",
remoteCwd: rootDir,
timeoutMs: 30_000,
runner,
};
const prevRefresh = process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS;
const prevTtl = process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS;
process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS = refreshRaw;
process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS = ttlRaw;
let bridge;
try {
bridge = await startAdapterExecutionTargetProcessSessionBridge({
runId: `run-validate-${refreshRaw}-${ttlRaw}`,
target,
runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"),
adapterKey: "acpx",
command: process.execPath,
args: ["-e", "process.stdin.resume();"],
cwd: rootDir,
env: {},
timeoutSec: 5,
onLog: async () => {},
});
} finally {
if (prevRefresh === undefined) delete process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS;
else process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS = prevRefresh;
if (prevTtl === undefined) delete process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS;
else process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS = prevTtl;
}
expect(bridge).not.toBeNull();
const nohupScript = scripts.find((script) => script.includes("nohup node"));
expect(nohupScript).toBeDefined();
expect(nohupScript).toContain("PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS='60000'");
expect(nohupScript).toContain("PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS='900000'");
await bridge!.stop();
}
// The wrapper side: an invalid override pair (P invalid, D alone would
// be a very short deadline if the wrapper only clamped D and ignored an
// invalid P) must fall the whole pair back to the 900s default, not a
// 50ms deadline.
const wrapper = await startWrapperProcess({
outputToStdout: false,
terminateGraceMs: 200,
env: {
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "0",
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "50",
},
});
await delay(400);
expect(wrapper.exitInfo().code).toBeNull();
// Force a clean exit through the existing session-identity latch,
// instead of leaving the wrapper alive for the full 900s default.
await rm(wrapper.sessionDir, { recursive: true, force: true });
await Promise.race([
wrapper.exited,
delay(4_000).then(() => {
throw new Error("The wrapper did not exit after its session directory disappeared.");
}),
]);
}, 30_000);
});
@@ -1603,6 +1603,39 @@ const PROCESS_SESSION_AUTH_TIMEOUT_MS = 5_000;
// not the child's full shutdown.
const DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS = 3_000;
// The host-refreshed lease that bounds the wrapper's own lifetime, so a host
// crash cannot orphan it. See PROCESS_SESSION_STDIN_POLL_TAIL below for the
// sandbox side of this contract: the wrapper reads the lease as filesystem
// metadata only, never as content, and it never creates, opens, writes, or
// removes the lease file itself. This host writes and refreshes the lease; it
// never reads the lease back and never gates `sessionDir` cleanup on it, so
// this adds no new sandbox-to-host data flow.
const DEFAULT_PROCESS_SESSION_LEASE_REFRESH_MS = 60_000;
const DEFAULT_PROCESS_SESSION_LEASE_TTL_MS = 900_000;
// The name does not end in ".json", so no `.json`-only reader (the stdin
// poller) ever lists it.
const PROCESS_SESSION_LEASE_FILE_NAME = ".paperclip-session-lease";
// Reads the refresh period and the lease duration from the environment.
// Accept an override pair only when both values are positive finite integers
// AND the duration exceeds the refresh period; reject the pair to both
// defaults together, so an accepted override never pairs with a rejected one
// into an unvalidated combination. The wrapper validates its own copy of the
// resolved values with the same rule (see PROCESS_SESSION_STDIN_POLL_TAIL).
function resolveProcessSessionLeaseTiming(): { refreshMs: number; ttlMs: number } {
const parsePositiveInt = (raw: string | undefined): number | null => {
if (!raw) return null;
const value = Number(raw);
return Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : null;
};
const refreshOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS);
const ttlOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS);
if (refreshOverride !== null && ttlOverride !== null && ttlOverride > refreshOverride) {
return { refreshMs: refreshOverride, ttlMs: ttlOverride };
}
return { refreshMs: DEFAULT_PROCESS_SESSION_LEASE_REFRESH_MS, ttlMs: DEFAULT_PROCESS_SESSION_LEASE_TTL_MS };
}
function jsonLine(value: unknown): string {
return `${JSON.stringify(value)}\n`;
}
@@ -1749,6 +1782,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
const sessionDir = path.posix.join(bridgeRuntimeDir, sessionId);
const stdinDir = path.posix.join(sessionDir, "stdin");
const eventsDir = path.posix.join(sessionDir, "events");
const leaseTiming = resolveProcessSessionLeaseTiming();
const leasePath = path.posix.join(sessionDir, PROCESS_SESSION_LEASE_FILE_NAME);
// The streamed wrapper writes its frames to stdout and rides a separate remote
// path, so a warm sandbox can hold both wrapper scripts without the content
// hash-skip gate thrashing when a run switches output mode.
@@ -1802,6 +1837,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
// wrapper and let it go; do not capture `$!`.
`PAPERCLIP_PROCESS_SESSION_DIR=${shellQuote(sessionDir)} ` +
`PAPERCLIP_PROCESS_SESSION_COMMAND_B64=${shellQuote(commandPayload)} ` +
`PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS=${shellQuote(String(leaseTiming.refreshMs))} ` +
`PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS=${shellQuote(String(leaseTiming.ttlMs))} ` +
`nohup node ${shellQuote(remoteScriptPath)} >/dev/null 2>&1 < /dev/null &`,
].join("\n"),
),
@@ -1838,6 +1875,66 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
// corrupts a large prompt on the stdin path.
let stdinWriteChain: Promise<void> = Promise.resolve();
let pollTimer: NodeJS.Timeout | null = null;
// Set permanently by `stop()`, before it clears the timer below and awaits
// the refresh chain. Once true, a refresh that is already chained (queued
// by the last timer tick before `stop()` ran) still resolves, but it
// returns without executing the remote command -- so no refresh can start
// after `stop()` begins removing `sessionDir`.
let leaseRefreshClosed = false;
let leaseRefreshTimer: NodeJS.Timeout | null = null;
// Serializes every refresh onto one chain, so `stop()` can await the whole
// active/queued chain before it removes `sessionDir`. Two concurrent
// refreshes could otherwise race a `stop()` removal and recreate
// `sessionDir` right after cleanup -- a fresh orphan on the exact path
// PAP-5307 made deterministic.
let leaseRefreshChain: Promise<void> = Promise.resolve();
const refreshProcessSessionLeaseOnce = async (): Promise<void> => {
if (leaseRefreshClosed) return;
// A fresh, unpredictable temporary name per refresh, so a peer cannot
// pre-create it and defeat the exclusive create below.
const tempPath = `${leasePath}.${randomBytes(9).toString("hex")}.tmp`;
const script = [
// `set -C` (noclobber) makes the shell open the redirect target with
// O_CREAT|O_EXCL: it fails on an existing file and on a symbolic
// link, so it can neither follow a link nor truncate a peer's file.
"set -C",
`: > ${shellQuote(tempPath)}`,
// `mv` replaces the lease path by rename. A rename does not follow a
// symbolic link at the target, so a peer that pre-creates a link at
// the fixed lease path cannot redirect this write. This command
// contains no `mkdir`: when `stop()` already removed `sessionDir`,
// the create above fails, this refresh fails, and the wrapper
// expires -- the wanted outcome, not a defect.
`mv ${shellQuote(tempPath)} ${shellQuote(leasePath)}`,
].join("\n");
try {
await runner.execute({
command: shellCommand,
args: shellCommandArgs(script),
cwd: target.remoteCwd,
env: { PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge" },
timeoutMs,
// A busy persistent session must never queue this behind an
// in-run session command and starve the lease.
bypassSession: true,
});
} catch {
// Best-effort and non-fatal: a failed refresh leaves the wrapper's
// deadline unreset this cycle. Never retry with a following write;
// never throw out of the timer.
}
};
const queueProcessSessionLeaseRefresh = (): void => {
leaseRefreshChain = leaseRefreshChain.then(refreshProcessSessionLeaseOnce, refreshProcessSessionLeaseOnce);
};
const scheduleProcessSessionLeaseRefresh = (): void => {
if (leaseRefreshClosed) return;
leaseRefreshTimer = setTimeout(() => {
queueProcessSessionLeaseRefresh();
scheduleProcessSessionLeaseRefresh();
}, leaseTiming.refreshMs);
leaseRefreshTimer.unref?.();
};
const pendingRemoteEvents: Array<{
type?: string;
stream?: "stdout" | "stderr";
@@ -2127,6 +2224,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
env: {
PAPERCLIP_PROCESS_SESSION_DIR: sessionDir,
PAPERCLIP_PROCESS_SESSION_COMMAND_B64: streamCommandPayload,
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: String(leaseTiming.refreshMs),
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: String(leaseTiming.ttlMs),
PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge",
},
timeoutMs,
@@ -2157,6 +2256,15 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
schedulePoll();
}
// Write the lease once at bridge start, on both variants, then keep it
// refreshing on an unreferenced timer. The event-file variant's `mkdir -p`
// launch exec above already created `sessionDir`; the streamed variant's
// wrapper creates it inside the sandbox, so this first write can race that
// creation and fail. That failure is harmless: it leaves the deadline
// unreset for one cycle, well inside the default 900s lease duration.
queueProcessSessionLeaseRefresh();
scheduleProcessSessionLeaseRefresh();
// `stop()` cannot rely on the long-lived poll above to observe a late
// `shutdownAck`: that poll stops re-arming as soon as it forwards a
// terminal `exit`/`error` event, and `stop()` itself sets `stopping` on
@@ -2198,6 +2306,13 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
agentCommand,
stop: async () => {
stopping = true;
// Close the lease refresh state permanently and clear its timer first,
// before any other teardown step. A refresh already chained by the
// last timer tick still resolves, but `refreshProcessSessionLeaseOnce`
// reads this flag and returns without running the remote command, so
// no refresh can start after this line.
leaseRefreshClosed = true;
if (leaseRefreshTimer) clearTimeout(leaseRefreshTimer);
// End the `sandbox.agentProcess` span now, before the caller ends the run
// root span, even if the remote command has not resolved yet.
signalStopped();
@@ -2261,6 +2376,14 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
`[paperclip] ACP process session wrapper did not acknowledge shutdown within ${DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS}ms; removing the session directory anyway.\n`,
).catch(() => undefined);
}
// Await the entire active/queued lease refresh chain before the
// removal below. `leaseRefreshClosed` (set at the top of `stop()`)
// already stops a chained refresh from running its remote command, but
// an in-flight one (its exec already sent) still needs to settle
// first, or its `mv` could land after this removal and recreate
// `sessionDir` -- a fresh orphan on the exact path PAP-5307 made
// deterministic.
await leaseRefreshChain.catch(() => undefined);
// Unconditional: this removal runs whether or not the wrapper
// acknowledged, and whether or not any event (real or forged) arrived
// under `sessionDir`. `stop()` runs during run teardown and must stay
@@ -2363,6 +2486,90 @@ const terminateGraceMs = (() => {
return Number.isFinite(raw) && raw > 0 ? raw : 3000;
})();
// Host lease that bounds this wrapper's own lifetime when a host crash
// leaves sessionDir intact. The host writes and refreshes
// ".paperclip-session-lease" under sessionDir; this wrapper never creates,
// opens, writes, or removes that file. It reads only the file's OS-level
// metadata (device, inode, and modification time), never its content: a
// content timestamp would compare the host clock against the sandbox
// clock, and a clock step would then cause a false expiry or a false
// extension. The deadline runs on a MONOTONIC clock
// (process.hrtime.bigint()), so it is immune to both a wall-clock step and
// an NTP adjustment.
//
// Accepted residuals (PAP-5354): a sandbox peer with access to
// sessionDir can touch the lease path and defer this wrapper's expiry, or
// remove the lease path and force an earlier expiry. Neither grants a
// capability the peer lacks: it can already spawn its own long-lived work,
// and it can already force a faster kill by deleting sessionDir outright
// (the session-identity latch elsewhere in this wrapper already covers that
// path). A peer that replaces sessionDir with a symbolic link still makes
// the host's exclusive create land an empty, unpredictably named file in
// the link target; the create truncates nothing and writes no content, and
// the session-identity latch terminates this wrapper on that same event. A
// peer that puts a directory at the lease path makes the host's \`mv\` move
// its temporary file into that directory instead of replacing the lease
// path, so this wrapper's own \`lstat\` sees a directory, never resets the
// deadline, and this wrapper still expires -- an availability loss, not an
// escape.
const leaseFileName = ".paperclip-session-lease";
const leasePath = path.posix.join(sessionDir, leaseFileName);
const defaultLeaseRefreshMs = 60_000;
const defaultLeaseTtlMs = 900_000;
// Validated the same way as the host resolver: accept the override pair
// only when both values are positive finite integers AND the duration
// exceeds the refresh period. Reject the pair to both defaults together, so
// one accepted override never combines with one rejected override into an
// unvalidated pair.
function resolveLeaseTiming() {
function parsePositiveInt(raw) {
if (!raw) return null;
const value = Number(raw);
return Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : null;
}
const refreshOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS);
const ttlOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS);
if (refreshOverride !== null && ttlOverride !== null && ttlOverride > refreshOverride) {
return { refreshMs: refreshOverride, ttlMs: ttlOverride };
}
return { refreshMs: defaultLeaseRefreshMs, ttlMs: defaultLeaseTtlMs };
}
const leaseTiming = resolveLeaseTiming();
// The deadline starts at "now plus the lease duration", before the first
// poll cycle, so a lease that never appears (a slow or crashed host) still
// bounds this wrapper. \`leaseStamp\` holds the last observed identity of the
// lease file; only a changed stamp resets the deadline.
let leaseDeadline = process.hrtime.bigint() + BigInt(leaseTiming.ttlMs) * 1000000n;
let leaseStamp = null;
// Runs at the top of every poll cycle, before this wrapper reads stdinDir.
// This wrapper never creates, opens, writes, or removes the lease file --
// it only lstats it. Any lstat error (a missing file, a permission error, or
// any other failure) and any non-regular-file object (a symbolic link, a
// directory, a socket) leave the deadline untouched: neither resets it nor
// expires it early. Only a changed identity on a regular file resets it.
async function checkProcessSessionLease() {
let stats;
try {
stats = await fs.lstat(leasePath);
} catch {
return;
}
if (!stats.isFile()) return;
const stamp = { dev: stats.dev, ino: stats.ino, mtimeMs: stats.mtimeMs };
if (
!leaseStamp ||
stamp.dev !== leaseStamp.dev ||
stamp.ino !== leaseStamp.ino ||
stamp.mtimeMs !== leaseStamp.mtimeMs
) {
leaseStamp = stamp;
leaseDeadline = process.hrtime.bigint() + BigInt(leaseTiming.ttlMs) * 1000000n;
}
}
// I2: terminate() is the only function in this wrapper that calls
// child.kill(). No child event handler and no sibling callback calls it.
// terminate() is idempotent: a second call, or a first call after the child
@@ -2620,6 +2827,15 @@ async function readStdinDirNames() {
async function pollStdin() {
while (!shuttingDown) {
// Check the lease before this cycle reads stdinDir. This reuses
// latchAndTerminate() -- the same one-shot gate the session-identity
// check above uses -- so this wrapper still has exactly one termination
// path and no new child.kill() caller.
await checkProcessSessionLease();
if (process.hrtime.bigint() >= leaseDeadline) {
process.stderr.write("Process session lease expired; terminating.\\n");
await latchAndTerminate();
}
const entries = (await readStdinDirNames()).filter((name) => name.endsWith(".json")).sort();
for (const name of entries) {
if (shuttingDown) break;