mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-25 17:45:53 +01:00
fix(adapter-utils): bound the process-session wrapper with a host lease
A host crash left the scratch root on disk, so the remote process-session wrapper polled its unchanged control directory forever and kept its child alive. The host now writes and refreshes a lease file under sessionDir on an unreferenced timer; the wrapper reads the lease as filesystem metadata only (device, inode, modification time) on a monotonic clock, and terminates itself once the lease stops moving past its duration. Co-authored-by: Paperclip <noreply@paperclip.ing>
This commit is contained in:
2 files changed
+704
No files matched your search
@@ -766,6 +766,10 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", ()
|
||||
// real "no usable creation time" filesystem is not reachable in this
|
||||
// sandbox, so the test simulates the exact Stats shape instead.
|
||||
fakeBirthtime?: { target: "sessionDir" | "stdinDir"; mode: "zero" | "followCtime" };
|
||||
// Extra environment variables for this wrapper process, merged in last.
|
||||
// Used to drive the lease timing overrides directly, with no host bridge
|
||||
// in front of the wrapper.
|
||||
env?: Record<string, string>;
|
||||
}) {
|
||||
const sessionDir = await mkdtemp(path.join(options?.parentDir ?? os.tmpdir(), "paperclip-wrapper-lifecycle-"));
|
||||
cleanupDirs.push(sessionDir);
|
||||
@@ -787,6 +791,7 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", ()
|
||||
};
|
||||
if (options?.maxRetries != null) env.PAPERCLIP_PROCESS_SESSION_STDIN_MAX_RETRIES = String(options.maxRetries);
|
||||
if (options?.terminateGraceMs != null) env.PAPERCLIP_PROCESS_SESSION_TERMINATE_GRACE_MS = String(options.terminateGraceMs);
|
||||
if (options?.env) Object.assign(env, options.env);
|
||||
|
||||
const execArgv: string[] = [];
|
||||
if (options?.fakeBirthtime) {
|
||||
@@ -1814,4 +1819,487 @@ describe("deterministic remote process-session wrapper shutdown (PAP-5316)", ()
|
||||
}
|
||||
expect(wrapper.stderrText()).toMatch(/Latching on a lost process session identity/);
|
||||
}, 15_000);
|
||||
|
||||
// ---- PAP-5361: a host-refreshed lease bounds the wrapper's own lifetime,
|
||||
// so a host crash cannot orphan it. The host writes and refreshes
|
||||
// ".paperclip-session-lease" under sessionDir; the wrapper reads only its
|
||||
// filesystem metadata (device, inode, modification time) on a monotonic
|
||||
// clock, and never its content. -----------------------------------------
|
||||
|
||||
const LEASE_FILE_NAME = ".paperclip-session-lease";
|
||||
|
||||
it("T21 no refresh: the wrapper terminates itself and its child after the lease duration elapses", async () => {
|
||||
for (const outputToStdout of [false, true]) {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-no-refresh-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const pidFile = path.join(rootDir, "t21-child.pid");
|
||||
const childPath = path.join(rootDir, "t21-child.mjs");
|
||||
await writeFile(childPath, trackedChildSource(pidFile), "utf8");
|
||||
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout,
|
||||
command: process.execPath,
|
||||
args: [childPath],
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
|
||||
},
|
||||
});
|
||||
|
||||
const pid = await waitForTrackedChildPid(pidFile);
|
||||
// No lease file is ever written under sessionDir.
|
||||
await expectWrapperAndTrackedChildToDie(wrapper, pid);
|
||||
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("T22 a valid rename-based refresh resets the deadline, and the wrapper stays alive past the lease duration", async () => {
|
||||
for (const outputToStdout of [false, true]) {
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout,
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
|
||||
},
|
||||
});
|
||||
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
|
||||
|
||||
let refreshing = true;
|
||||
let refreshCount = 0;
|
||||
const refresh = (async () => {
|
||||
while (refreshing) {
|
||||
// Mirrors the production shape: a fresh temp name, then a rename
|
||||
// onto the fixed lease path.
|
||||
const tempPath = `${leasePath}.${refreshCount}.tmp`;
|
||||
await writeFile(tempPath, "", "utf8");
|
||||
await rename(tempPath, leasePath);
|
||||
refreshCount += 1;
|
||||
await delay(120);
|
||||
}
|
||||
})();
|
||||
|
||||
// Refresh faster than the 400ms lease duration, for more than twice
|
||||
// the duration in total. A wrapper that ignored the refresh would
|
||||
// already have expired well before this returns.
|
||||
await delay(950);
|
||||
refreshing = false;
|
||||
await refresh;
|
||||
|
||||
expect(refreshCount).toBeGreaterThan(2);
|
||||
expect(wrapper.exitInfo().code).toBeNull();
|
||||
|
||||
// Let the wrapper expire on its own now that the refresh stopped, so
|
||||
// the test leaves no lingering process.
|
||||
await Promise.race([
|
||||
wrapper.exited,
|
||||
delay(4_000).then(() => {
|
||||
throw new Error("The wrapper did not expire once the refresh stopped.");
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("T23 a missing lease never resets the deadline, and ordinary stdin traffic does not substitute for it", async () => {
|
||||
for (const outputToStdout of [false, true]) {
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout,
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
|
||||
},
|
||||
});
|
||||
|
||||
// Ordinary stdin traffic while the lease stays absent. Writing and
|
||||
// deleting a stdin file changes stdinDir's own metadata, not the
|
||||
// lease's, so this must not reset the deadline.
|
||||
const finalPath = path.join(wrapper.stdinDir, "000000000001.json");
|
||||
const tempPath = `${finalPath}.writing`;
|
||||
await writeFile(
|
||||
tempPath,
|
||||
`${JSON.stringify({ type: "stdin", data: Buffer.from("hi", "utf8").toString("base64") })}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await rename(tempPath, finalPath);
|
||||
|
||||
await Promise.race([
|
||||
wrapper.exited,
|
||||
delay(4_000).then(() => {
|
||||
throw new Error("The wrapper did not expire with a missing lease.");
|
||||
}),
|
||||
]);
|
||||
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("T24 a symbolic link at the lease path is never followed, its target stays unchanged, and the wrapper expires", async () => {
|
||||
const { symlink } = await import("node:fs/promises");
|
||||
for (const outputToStdout of [false, true]) {
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout,
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
|
||||
},
|
||||
});
|
||||
const targetPath = path.join(wrapper.sessionDir, "lease-link-target");
|
||||
await writeFile(targetPath, "untouched", "utf8");
|
||||
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
|
||||
await symlink(targetPath, leasePath);
|
||||
|
||||
await Promise.race([
|
||||
wrapper.exited,
|
||||
delay(4_000).then(() => {
|
||||
throw new Error("The wrapper did not expire with a symbolic link at the lease path.");
|
||||
}),
|
||||
]);
|
||||
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
|
||||
expect(await readFile(targetPath, "utf8")).toBe("untouched");
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("T25 a directory at the lease path is never treated as a valid lease, and the wrapper expires", async () => {
|
||||
for (const outputToStdout of [false, true]) {
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout,
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
|
||||
},
|
||||
});
|
||||
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
|
||||
await mkdir(leasePath, { recursive: true });
|
||||
|
||||
await Promise.race([
|
||||
wrapper.exited,
|
||||
delay(4_000).then(() => {
|
||||
throw new Error("The wrapper did not expire with a directory at the lease path.");
|
||||
}),
|
||||
]);
|
||||
expect(wrapper.stderrText()).toMatch(/[Ll]ease expired/);
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
// Pins the accepted residual (PAP-5354): a sandbox peer that
|
||||
// touches the lease path defers this wrapper's expiry. This is not a
|
||||
// capability the peer lacked -- it can already spawn its own long-lived
|
||||
// work -- so a later change must not silently alter this shape.
|
||||
it("T26 a peer that keeps touching the lease file defers this wrapper's expiry", async () => {
|
||||
const { utimes } = await import("node:fs/promises");
|
||||
for (const outputToStdout of [false, true]) {
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout,
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "100",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "400",
|
||||
},
|
||||
});
|
||||
const leasePath = path.join(wrapper.sessionDir, LEASE_FILE_NAME);
|
||||
await writeFile(leasePath, "", "utf8");
|
||||
|
||||
let churning = true;
|
||||
const churn = (async () => {
|
||||
while (churning) {
|
||||
await utimes(leasePath, new Date(), new Date()).catch(() => undefined);
|
||||
await delay(120);
|
||||
}
|
||||
})();
|
||||
|
||||
// Keep touching for more than twice the lease duration; a wrapper
|
||||
// that ignored the touch would already have expired.
|
||||
await delay(950);
|
||||
churning = false;
|
||||
await churn;
|
||||
expect(wrapper.exitInfo().code).toBeNull();
|
||||
|
||||
// Stop touching and let the wrapper expire normally, so the test
|
||||
// leaves no lingering process.
|
||||
await Promise.race([
|
||||
wrapper.exited,
|
||||
delay(4_000).then(() => {
|
||||
throw new Error("The wrapper did not expire once the peer stopped touching the lease.");
|
||||
}),
|
||||
]);
|
||||
}
|
||||
}, 20_000);
|
||||
|
||||
it("T27 each wrapper source has exactly one lease deadline declaration and one lease check call site, and the lease code adds no kill caller or numeric process identifier", () => {
|
||||
for (const outputToStdout of [true, false]) {
|
||||
const src = getProcessSessionRemoteSource({ outputToStdout });
|
||||
const code = src
|
||||
.split("\n")
|
||||
.map((line) => line.replace(/\/\/.*$/, ""))
|
||||
.join("\n");
|
||||
|
||||
const leaseDeadlineDeclarations = code.match(/\blet leaseDeadline = /g) ?? [];
|
||||
expect(leaseDeadlineDeclarations.length).toBe(1);
|
||||
|
||||
const leaseCheckCallSites = code.match(/\bawait checkProcessSessionLease\(\)/g) ?? [];
|
||||
expect(leaseCheckCallSites.length).toBe(1);
|
||||
expect(code.indexOf("await checkProcessSessionLease();")).toBeLessThan(code.indexOf("await readStdinDirNames()"));
|
||||
|
||||
const killCallSites = [...code.matchAll(/[A-Za-z0-9_.$]*kill\(/g)].map((match) => match[0]);
|
||||
expect(killCallSites.length).toBeGreaterThan(0);
|
||||
for (const site of killCallSites) {
|
||||
expect(site).toBe("child.kill(");
|
||||
}
|
||||
|
||||
const leaseSectionStart = code.indexOf("const leaseFileName =");
|
||||
const leaseSectionEnd = code.indexOf("async function terminate()");
|
||||
expect(leaseSectionStart).toBeGreaterThan(0);
|
||||
expect(leaseSectionEnd).toBeGreaterThan(leaseSectionStart);
|
||||
const leaseSection = code.slice(leaseSectionStart, leaseSectionEnd);
|
||||
expect(leaseSection).not.toContain("kill(");
|
||||
expect(leaseSection).not.toMatch(/\bpid\b/i);
|
||||
}
|
||||
});
|
||||
|
||||
it("T28 an injected slow lease refresh in flight during stop() cannot recreate sessionDir after removal", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-host-race-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const childPath = path.join(rootDir, "quiet-child.mjs");
|
||||
await writeFile(childPath, "process.stdin.resume();\n", "utf8");
|
||||
|
||||
const completionOrder: string[] = [];
|
||||
let counter = 0;
|
||||
let sessionDir = "";
|
||||
const runner = {
|
||||
execute: async (input: {
|
||||
command: string;
|
||||
args?: string[];
|
||||
cwd?: string;
|
||||
env?: Record<string, string>;
|
||||
stdin?: string;
|
||||
timeoutMs?: number;
|
||||
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
||||
}): Promise<RunProcessResult> => {
|
||||
const script = input.args?.[1] ?? "";
|
||||
if (script.startsWith("mkdir -p") && !sessionDir) {
|
||||
const dirsMatch = /mkdir -p '([^']+)' '([^']+)'/.exec(script);
|
||||
if (dirsMatch) sessionDir = path.posix.dirname(dirsMatch[1]);
|
||||
}
|
||||
if (script.includes("nohup node")) {
|
||||
return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null };
|
||||
}
|
||||
const isLeaseRefresh = script.includes("set -C") && script.includes(LEASE_FILE_NAME);
|
||||
const isRemove = script.trim().startsWith("rm -rf");
|
||||
if (isLeaseRefresh) {
|
||||
// Simulate a slow remote exec for the refresh command: it is
|
||||
// already in flight when stop() runs below.
|
||||
await delay(500);
|
||||
}
|
||||
counter += 1;
|
||||
const command =
|
||||
input.command === "bash" ? "/bin/bash" : input.command === "sh" ? "/bin/sh" : input.command;
|
||||
const result = await runChildProcess(`lease-race-run-${counter}`, command, input.args ?? [], {
|
||||
cwd: input.cwd ?? process.cwd(),
|
||||
env: input.env ?? {},
|
||||
stdin: input.stdin,
|
||||
timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),
|
||||
graceSec: 5,
|
||||
onLog: input.onLog ?? (async () => {}),
|
||||
});
|
||||
if (isLeaseRefresh) completionOrder.push("refresh");
|
||||
if (isRemove) completionOrder.push("remove");
|
||||
return result;
|
||||
},
|
||||
};
|
||||
const target: AdapterSandboxExecutionTarget = {
|
||||
kind: "remote",
|
||||
transport: "sandbox",
|
||||
providerKey: "local-test",
|
||||
remoteCwd: rootDir,
|
||||
timeoutMs: 30_000,
|
||||
runner,
|
||||
};
|
||||
|
||||
const bridge = await startAdapterExecutionTargetProcessSessionBridge({
|
||||
runId: "run-lease-race",
|
||||
target,
|
||||
runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"),
|
||||
adapterKey: "acpx",
|
||||
command: process.execPath,
|
||||
args: [childPath],
|
||||
cwd: rootDir,
|
||||
env: {},
|
||||
timeoutSec: 5,
|
||||
onLog: async () => {},
|
||||
});
|
||||
expect(bridge).not.toBeNull();
|
||||
|
||||
// Give the initial lease write (fired right after launch) time to start
|
||||
// before stop() runs, so this exercises a refresh already in flight, not
|
||||
// one stop() outruns before it ever starts.
|
||||
await waitFor(() => completionOrder.length === 0 && sessionDir !== "", 4_000);
|
||||
await delay(50);
|
||||
await bridge!.stop();
|
||||
|
||||
// stop() awaits the whole refresh chain before it removes sessionDir, so
|
||||
// the delayed refresh's own `mv` always finishes first.
|
||||
expect(completionOrder).toEqual(["refresh", "remove"]);
|
||||
|
||||
expect(sessionDir).not.toBe("");
|
||||
const stillThere = await readdir(sessionDir).catch(() => null);
|
||||
expect(stillThere).toBeNull();
|
||||
}, 15_000);
|
||||
|
||||
it("T29 the lease refresh command carries bypassSession and contains no mkdir", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-command-shape-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const childPath = path.join(rootDir, "quiet-child.mjs");
|
||||
await writeFile(childPath, "process.stdin.resume();\n", "utf8");
|
||||
|
||||
const calls: Array<{ script: string; bypassSession?: boolean }> = [];
|
||||
let counter = 0;
|
||||
const runner = {
|
||||
execute: async (input: {
|
||||
command: string;
|
||||
args?: string[];
|
||||
cwd?: string;
|
||||
env?: Record<string, string>;
|
||||
stdin?: string;
|
||||
timeoutMs?: number;
|
||||
onLog?: (stream: "stdout" | "stderr", chunk: string) => Promise<void>;
|
||||
bypassSession?: boolean;
|
||||
}): Promise<RunProcessResult> => {
|
||||
const script = input.args?.[1] ?? "";
|
||||
calls.push({ script, bypassSession: input.bypassSession });
|
||||
if (script.includes("nohup node")) {
|
||||
return { exitCode: 0, signal: null, timedOut: false, stdout: "", stderr: "", pid: null, startedAt: null };
|
||||
}
|
||||
counter += 1;
|
||||
const command =
|
||||
input.command === "bash" ? "/bin/bash" : input.command === "sh" ? "/bin/sh" : input.command;
|
||||
return runChildProcess(`lease-shape-run-${counter}`, command, input.args ?? [], {
|
||||
cwd: input.cwd ?? process.cwd(),
|
||||
env: input.env ?? {},
|
||||
stdin: input.stdin,
|
||||
timeoutSec: Math.max(1, Math.ceil((input.timeoutMs ?? 30_000) / 1000)),
|
||||
graceSec: 5,
|
||||
onLog: input.onLog ?? (async () => {}),
|
||||
});
|
||||
},
|
||||
};
|
||||
const target: AdapterSandboxExecutionTarget = {
|
||||
kind: "remote",
|
||||
transport: "sandbox",
|
||||
providerKey: "local-test",
|
||||
remoteCwd: rootDir,
|
||||
timeoutMs: 30_000,
|
||||
runner,
|
||||
};
|
||||
|
||||
const bridge = await startAdapterExecutionTargetProcessSessionBridge({
|
||||
runId: "run-lease-shape",
|
||||
target,
|
||||
runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"),
|
||||
adapterKey: "acpx",
|
||||
command: process.execPath,
|
||||
args: [childPath],
|
||||
cwd: rootDir,
|
||||
env: {},
|
||||
timeoutSec: 5,
|
||||
onLog: async () => {},
|
||||
});
|
||||
expect(bridge).not.toBeNull();
|
||||
await waitFor(() => calls.some((call) => call.script.includes("set -C")), 4_000);
|
||||
await bridge!.stop();
|
||||
|
||||
const refreshCalls = calls.filter((call) => call.script.includes("set -C"));
|
||||
expect(refreshCalls.length).toBeGreaterThan(0);
|
||||
for (const call of refreshCalls) {
|
||||
expect(call.bypassSession).toBe(true);
|
||||
expect(call.script).not.toContain("mkdir");
|
||||
expect(call.script).toContain("mv ");
|
||||
}
|
||||
}, 10_000);
|
||||
|
||||
it("T30 validation: a zero, a negative, a non-numeric, and a D<=P lease override pair all fall back to the defaults on the host and in the wrapper", async () => {
|
||||
const invalidPairs: Array<[string, string]> = [
|
||||
["0", "900000"],
|
||||
["-1000", "900000"],
|
||||
["not-a-number", "900000"],
|
||||
["600000", "300000"],
|
||||
];
|
||||
|
||||
for (const [refreshRaw, ttlRaw] of invalidPairs) {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "paperclip-lease-validate-"));
|
||||
cleanupDirs.push(rootDir);
|
||||
const scripts: string[] = [];
|
||||
const runner = createLocalSandboxRunner(async (script) => {
|
||||
scripts.push(script);
|
||||
});
|
||||
const target: AdapterSandboxExecutionTarget = {
|
||||
kind: "remote",
|
||||
transport: "sandbox",
|
||||
providerKey: "local-test",
|
||||
remoteCwd: rootDir,
|
||||
timeoutMs: 30_000,
|
||||
runner,
|
||||
};
|
||||
|
||||
const prevRefresh = process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS;
|
||||
const prevTtl = process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS;
|
||||
process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS = refreshRaw;
|
||||
process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS = ttlRaw;
|
||||
|
||||
let bridge;
|
||||
try {
|
||||
bridge = await startAdapterExecutionTargetProcessSessionBridge({
|
||||
runId: `run-validate-${refreshRaw}-${ttlRaw}`,
|
||||
target,
|
||||
runtimeRootDir: path.posix.join(rootDir, ".paperclip-runtime", "acpx"),
|
||||
adapterKey: "acpx",
|
||||
command: process.execPath,
|
||||
args: ["-e", "process.stdin.resume();"],
|
||||
cwd: rootDir,
|
||||
env: {},
|
||||
timeoutSec: 5,
|
||||
onLog: async () => {},
|
||||
});
|
||||
} finally {
|
||||
if (prevRefresh === undefined) delete process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS;
|
||||
else process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS = prevRefresh;
|
||||
if (prevTtl === undefined) delete process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS;
|
||||
else process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS = prevTtl;
|
||||
}
|
||||
expect(bridge).not.toBeNull();
|
||||
|
||||
const nohupScript = scripts.find((script) => script.includes("nohup node"));
|
||||
expect(nohupScript).toBeDefined();
|
||||
expect(nohupScript).toContain("PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS='60000'");
|
||||
expect(nohupScript).toContain("PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS='900000'");
|
||||
|
||||
await bridge!.stop();
|
||||
}
|
||||
|
||||
// The wrapper side: an invalid override pair (P invalid, D alone would
|
||||
// be a very short deadline if the wrapper only clamped D and ignored an
|
||||
// invalid P) must fall the whole pair back to the 900s default, not a
|
||||
// 50ms deadline.
|
||||
const wrapper = await startWrapperProcess({
|
||||
outputToStdout: false,
|
||||
terminateGraceMs: 200,
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: "0",
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: "50",
|
||||
},
|
||||
});
|
||||
await delay(400);
|
||||
expect(wrapper.exitInfo().code).toBeNull();
|
||||
// Force a clean exit through the existing session-identity latch,
|
||||
// instead of leaving the wrapper alive for the full 900s default.
|
||||
await rm(wrapper.sessionDir, { recursive: true, force: true });
|
||||
await Promise.race([
|
||||
wrapper.exited,
|
||||
delay(4_000).then(() => {
|
||||
throw new Error("The wrapper did not exit after its session directory disappeared.");
|
||||
}),
|
||||
]);
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -1603,6 +1603,39 @@ const PROCESS_SESSION_AUTH_TIMEOUT_MS = 5_000;
|
||||
// not the child's full shutdown.
|
||||
const DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS = 3_000;
|
||||
|
||||
// The host-refreshed lease that bounds the wrapper's own lifetime, so a host
|
||||
// crash cannot orphan it. See PROCESS_SESSION_STDIN_POLL_TAIL below for the
|
||||
// sandbox side of this contract: the wrapper reads the lease as filesystem
|
||||
// metadata only, never as content, and it never creates, opens, writes, or
|
||||
// removes the lease file itself. This host writes and refreshes the lease; it
|
||||
// never reads the lease back and never gates `sessionDir` cleanup on it, so
|
||||
// this adds no new sandbox-to-host data flow.
|
||||
const DEFAULT_PROCESS_SESSION_LEASE_REFRESH_MS = 60_000;
|
||||
const DEFAULT_PROCESS_SESSION_LEASE_TTL_MS = 900_000;
|
||||
// The name does not end in ".json", so no `.json`-only reader (the stdin
|
||||
// poller) ever lists it.
|
||||
const PROCESS_SESSION_LEASE_FILE_NAME = ".paperclip-session-lease";
|
||||
|
||||
// Reads the refresh period and the lease duration from the environment.
|
||||
// Accept an override pair only when both values are positive finite integers
|
||||
// AND the duration exceeds the refresh period; reject the pair to both
|
||||
// defaults together, so an accepted override never pairs with a rejected one
|
||||
// into an unvalidated combination. The wrapper validates its own copy of the
|
||||
// resolved values with the same rule (see PROCESS_SESSION_STDIN_POLL_TAIL).
|
||||
function resolveProcessSessionLeaseTiming(): { refreshMs: number; ttlMs: number } {
|
||||
const parsePositiveInt = (raw: string | undefined): number | null => {
|
||||
if (!raw) return null;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : null;
|
||||
};
|
||||
const refreshOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS);
|
||||
const ttlOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS);
|
||||
if (refreshOverride !== null && ttlOverride !== null && ttlOverride > refreshOverride) {
|
||||
return { refreshMs: refreshOverride, ttlMs: ttlOverride };
|
||||
}
|
||||
return { refreshMs: DEFAULT_PROCESS_SESSION_LEASE_REFRESH_MS, ttlMs: DEFAULT_PROCESS_SESSION_LEASE_TTL_MS };
|
||||
}
|
||||
|
||||
function jsonLine(value: unknown): string {
|
||||
return `${JSON.stringify(value)}\n`;
|
||||
}
|
||||
@@ -1749,6 +1782,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
const sessionDir = path.posix.join(bridgeRuntimeDir, sessionId);
|
||||
const stdinDir = path.posix.join(sessionDir, "stdin");
|
||||
const eventsDir = path.posix.join(sessionDir, "events");
|
||||
const leaseTiming = resolveProcessSessionLeaseTiming();
|
||||
const leasePath = path.posix.join(sessionDir, PROCESS_SESSION_LEASE_FILE_NAME);
|
||||
// The streamed wrapper writes its frames to stdout and rides a separate remote
|
||||
// path, so a warm sandbox can hold both wrapper scripts without the content
|
||||
// hash-skip gate thrashing when a run switches output mode.
|
||||
@@ -1802,6 +1837,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
// wrapper and let it go; do not capture `$!`.
|
||||
`PAPERCLIP_PROCESS_SESSION_DIR=${shellQuote(sessionDir)} ` +
|
||||
`PAPERCLIP_PROCESS_SESSION_COMMAND_B64=${shellQuote(commandPayload)} ` +
|
||||
`PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS=${shellQuote(String(leaseTiming.refreshMs))} ` +
|
||||
`PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS=${shellQuote(String(leaseTiming.ttlMs))} ` +
|
||||
`nohup node ${shellQuote(remoteScriptPath)} >/dev/null 2>&1 < /dev/null &`,
|
||||
].join("\n"),
|
||||
),
|
||||
@@ -1838,6 +1875,66 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
// corrupts a large prompt on the stdin path.
|
||||
let stdinWriteChain: Promise<void> = Promise.resolve();
|
||||
let pollTimer: NodeJS.Timeout | null = null;
|
||||
// Set permanently by `stop()`, before it clears the timer below and awaits
|
||||
// the refresh chain. Once true, a refresh that is already chained (queued
|
||||
// by the last timer tick before `stop()` ran) still resolves, but it
|
||||
// returns without executing the remote command -- so no refresh can start
|
||||
// after `stop()` begins removing `sessionDir`.
|
||||
let leaseRefreshClosed = false;
|
||||
let leaseRefreshTimer: NodeJS.Timeout | null = null;
|
||||
// Serializes every refresh onto one chain, so `stop()` can await the whole
|
||||
// active/queued chain before it removes `sessionDir`. Two concurrent
|
||||
// refreshes could otherwise race a `stop()` removal and recreate
|
||||
// `sessionDir` right after cleanup -- a fresh orphan on the exact path
|
||||
// PAP-5307 made deterministic.
|
||||
let leaseRefreshChain: Promise<void> = Promise.resolve();
|
||||
const refreshProcessSessionLeaseOnce = async (): Promise<void> => {
|
||||
if (leaseRefreshClosed) return;
|
||||
// A fresh, unpredictable temporary name per refresh, so a peer cannot
|
||||
// pre-create it and defeat the exclusive create below.
|
||||
const tempPath = `${leasePath}.${randomBytes(9).toString("hex")}.tmp`;
|
||||
const script = [
|
||||
// `set -C` (noclobber) makes the shell open the redirect target with
|
||||
// O_CREAT|O_EXCL: it fails on an existing file and on a symbolic
|
||||
// link, so it can neither follow a link nor truncate a peer's file.
|
||||
"set -C",
|
||||
`: > ${shellQuote(tempPath)}`,
|
||||
// `mv` replaces the lease path by rename. A rename does not follow a
|
||||
// symbolic link at the target, so a peer that pre-creates a link at
|
||||
// the fixed lease path cannot redirect this write. This command
|
||||
// contains no `mkdir`: when `stop()` already removed `sessionDir`,
|
||||
// the create above fails, this refresh fails, and the wrapper
|
||||
// expires -- the wanted outcome, not a defect.
|
||||
`mv ${shellQuote(tempPath)} ${shellQuote(leasePath)}`,
|
||||
].join("\n");
|
||||
try {
|
||||
await runner.execute({
|
||||
command: shellCommand,
|
||||
args: shellCommandArgs(script),
|
||||
cwd: target.remoteCwd,
|
||||
env: { PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge" },
|
||||
timeoutMs,
|
||||
// A busy persistent session must never queue this behind an
|
||||
// in-run session command and starve the lease.
|
||||
bypassSession: true,
|
||||
});
|
||||
} catch {
|
||||
// Best-effort and non-fatal: a failed refresh leaves the wrapper's
|
||||
// deadline unreset this cycle. Never retry with a following write;
|
||||
// never throw out of the timer.
|
||||
}
|
||||
};
|
||||
const queueProcessSessionLeaseRefresh = (): void => {
|
||||
leaseRefreshChain = leaseRefreshChain.then(refreshProcessSessionLeaseOnce, refreshProcessSessionLeaseOnce);
|
||||
};
|
||||
const scheduleProcessSessionLeaseRefresh = (): void => {
|
||||
if (leaseRefreshClosed) return;
|
||||
leaseRefreshTimer = setTimeout(() => {
|
||||
queueProcessSessionLeaseRefresh();
|
||||
scheduleProcessSessionLeaseRefresh();
|
||||
}, leaseTiming.refreshMs);
|
||||
leaseRefreshTimer.unref?.();
|
||||
};
|
||||
const pendingRemoteEvents: Array<{
|
||||
type?: string;
|
||||
stream?: "stdout" | "stderr";
|
||||
@@ -2127,6 +2224,8 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
env: {
|
||||
PAPERCLIP_PROCESS_SESSION_DIR: sessionDir,
|
||||
PAPERCLIP_PROCESS_SESSION_COMMAND_B64: streamCommandPayload,
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS: String(leaseTiming.refreshMs),
|
||||
PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS: String(leaseTiming.ttlMs),
|
||||
PAPERCLIP_SANDBOX_EXEC_CHANNEL: "bridge",
|
||||
},
|
||||
timeoutMs,
|
||||
@@ -2157,6 +2256,15 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
schedulePoll();
|
||||
}
|
||||
|
||||
// Write the lease once at bridge start, on both variants, then keep it
|
||||
// refreshing on an unreferenced timer. The event-file variant's `mkdir -p`
|
||||
// launch exec above already created `sessionDir`; the streamed variant's
|
||||
// wrapper creates it inside the sandbox, so this first write can race that
|
||||
// creation and fail. That failure is harmless: it leaves the deadline
|
||||
// unreset for one cycle, well inside the default 900s lease duration.
|
||||
queueProcessSessionLeaseRefresh();
|
||||
scheduleProcessSessionLeaseRefresh();
|
||||
|
||||
// `stop()` cannot rely on the long-lived poll above to observe a late
|
||||
// `shutdownAck`: that poll stops re-arming as soon as it forwards a
|
||||
// terminal `exit`/`error` event, and `stop()` itself sets `stopping` on
|
||||
@@ -2198,6 +2306,13 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
agentCommand,
|
||||
stop: async () => {
|
||||
stopping = true;
|
||||
// Close the lease refresh state permanently and clear its timer first,
|
||||
// before any other teardown step. A refresh already chained by the
|
||||
// last timer tick still resolves, but `refreshProcessSessionLeaseOnce`
|
||||
// reads this flag and returns without running the remote command, so
|
||||
// no refresh can start after this line.
|
||||
leaseRefreshClosed = true;
|
||||
if (leaseRefreshTimer) clearTimeout(leaseRefreshTimer);
|
||||
// End the `sandbox.agentProcess` span now, before the caller ends the run
|
||||
// root span, even if the remote command has not resolved yet.
|
||||
signalStopped();
|
||||
@@ -2261,6 +2376,14 @@ export async function startAdapterExecutionTargetProcessSessionBridge(input: {
|
||||
`[paperclip] ACP process session wrapper did not acknowledge shutdown within ${DEFAULT_PROCESS_SESSION_SHUTDOWN_WAIT_MS}ms; removing the session directory anyway.\n`,
|
||||
).catch(() => undefined);
|
||||
}
|
||||
// Await the entire active/queued lease refresh chain before the
|
||||
// removal below. `leaseRefreshClosed` (set at the top of `stop()`)
|
||||
// already stops a chained refresh from running its remote command, but
|
||||
// an in-flight one (its exec already sent) still needs to settle
|
||||
// first, or its `mv` could land after this removal and recreate
|
||||
// `sessionDir` -- a fresh orphan on the exact path PAP-5307 made
|
||||
// deterministic.
|
||||
await leaseRefreshChain.catch(() => undefined);
|
||||
// Unconditional: this removal runs whether or not the wrapper
|
||||
// acknowledged, and whether or not any event (real or forged) arrived
|
||||
// under `sessionDir`. `stop()` runs during run teardown and must stay
|
||||
@@ -2363,6 +2486,90 @@ const terminateGraceMs = (() => {
|
||||
return Number.isFinite(raw) && raw > 0 ? raw : 3000;
|
||||
})();
|
||||
|
||||
// Host lease that bounds this wrapper's own lifetime when a host crash
|
||||
// leaves sessionDir intact. The host writes and refreshes
|
||||
// ".paperclip-session-lease" under sessionDir; this wrapper never creates,
|
||||
// opens, writes, or removes that file. It reads only the file's OS-level
|
||||
// metadata (device, inode, and modification time), never its content: a
|
||||
// content timestamp would compare the host clock against the sandbox
|
||||
// clock, and a clock step would then cause a false expiry or a false
|
||||
// extension. The deadline runs on a MONOTONIC clock
|
||||
// (process.hrtime.bigint()), so it is immune to both a wall-clock step and
|
||||
// an NTP adjustment.
|
||||
//
|
||||
// Accepted residuals (PAP-5354): a sandbox peer with access to
|
||||
// sessionDir can touch the lease path and defer this wrapper's expiry, or
|
||||
// remove the lease path and force an earlier expiry. Neither grants a
|
||||
// capability the peer lacks: it can already spawn its own long-lived work,
|
||||
// and it can already force a faster kill by deleting sessionDir outright
|
||||
// (the session-identity latch elsewhere in this wrapper already covers that
|
||||
// path). A peer that replaces sessionDir with a symbolic link still makes
|
||||
// the host's exclusive create land an empty, unpredictably named file in
|
||||
// the link target; the create truncates nothing and writes no content, and
|
||||
// the session-identity latch terminates this wrapper on that same event. A
|
||||
// peer that puts a directory at the lease path makes the host's \`mv\` move
|
||||
// its temporary file into that directory instead of replacing the lease
|
||||
// path, so this wrapper's own \`lstat\` sees a directory, never resets the
|
||||
// deadline, and this wrapper still expires -- an availability loss, not an
|
||||
// escape.
|
||||
const leaseFileName = ".paperclip-session-lease";
|
||||
const leasePath = path.posix.join(sessionDir, leaseFileName);
|
||||
const defaultLeaseRefreshMs = 60_000;
|
||||
const defaultLeaseTtlMs = 900_000;
|
||||
|
||||
// Validated the same way as the host resolver: accept the override pair
|
||||
// only when both values are positive finite integers AND the duration
|
||||
// exceeds the refresh period. Reject the pair to both defaults together, so
|
||||
// one accepted override never combines with one rejected override into an
|
||||
// unvalidated pair.
|
||||
function resolveLeaseTiming() {
|
||||
function parsePositiveInt(raw) {
|
||||
if (!raw) return null;
|
||||
const value = Number(raw);
|
||||
return Number.isFinite(value) && Number.isInteger(value) && value > 0 ? value : null;
|
||||
}
|
||||
const refreshOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_REFRESH_MS);
|
||||
const ttlOverride = parsePositiveInt(process.env.PAPERCLIP_PROCESS_SESSION_LEASE_TTL_MS);
|
||||
if (refreshOverride !== null && ttlOverride !== null && ttlOverride > refreshOverride) {
|
||||
return { refreshMs: refreshOverride, ttlMs: ttlOverride };
|
||||
}
|
||||
return { refreshMs: defaultLeaseRefreshMs, ttlMs: defaultLeaseTtlMs };
|
||||
}
|
||||
const leaseTiming = resolveLeaseTiming();
|
||||
|
||||
// The deadline starts at "now plus the lease duration", before the first
|
||||
// poll cycle, so a lease that never appears (a slow or crashed host) still
|
||||
// bounds this wrapper. \`leaseStamp\` holds the last observed identity of the
|
||||
// lease file; only a changed stamp resets the deadline.
|
||||
let leaseDeadline = process.hrtime.bigint() + BigInt(leaseTiming.ttlMs) * 1000000n;
|
||||
let leaseStamp = null;
|
||||
|
||||
// Runs at the top of every poll cycle, before this wrapper reads stdinDir.
|
||||
// This wrapper never creates, opens, writes, or removes the lease file --
|
||||
// it only lstats it. Any lstat error (a missing file, a permission error, or
|
||||
// any other failure) and any non-regular-file object (a symbolic link, a
|
||||
// directory, a socket) leave the deadline untouched: neither resets it nor
|
||||
// expires it early. Only a changed identity on a regular file resets it.
|
||||
async function checkProcessSessionLease() {
|
||||
let stats;
|
||||
try {
|
||||
stats = await fs.lstat(leasePath);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!stats.isFile()) return;
|
||||
const stamp = { dev: stats.dev, ino: stats.ino, mtimeMs: stats.mtimeMs };
|
||||
if (
|
||||
!leaseStamp ||
|
||||
stamp.dev !== leaseStamp.dev ||
|
||||
stamp.ino !== leaseStamp.ino ||
|
||||
stamp.mtimeMs !== leaseStamp.mtimeMs
|
||||
) {
|
||||
leaseStamp = stamp;
|
||||
leaseDeadline = process.hrtime.bigint() + BigInt(leaseTiming.ttlMs) * 1000000n;
|
||||
}
|
||||
}
|
||||
|
||||
// I2: terminate() is the only function in this wrapper that calls
|
||||
// child.kill(). No child event handler and no sibling callback calls it.
|
||||
// terminate() is idempotent: a second call, or a first call after the child
|
||||
@@ -2620,6 +2827,15 @@ async function readStdinDirNames() {
|
||||
|
||||
async function pollStdin() {
|
||||
while (!shuttingDown) {
|
||||
// Check the lease before this cycle reads stdinDir. This reuses
|
||||
// latchAndTerminate() -- the same one-shot gate the session-identity
|
||||
// check above uses -- so this wrapper still has exactly one termination
|
||||
// path and no new child.kill() caller.
|
||||
await checkProcessSessionLease();
|
||||
if (process.hrtime.bigint() >= leaseDeadline) {
|
||||
process.stderr.write("Process session lease expired; terminating.\\n");
|
||||
await latchAndTerminate();
|
||||
}
|
||||
const entries = (await readStdinDirNames()).filter((name) => name.endsWith(".json")).sort();
|
||||
for (const name of entries) {
|
||||
if (shuttingDown) break;
|
||||
|
||||
Reference in new issue
Block a user