diff --git a/server/src/__tests__/agent-instructions-service.test.ts b/server/src/__tests__/agent-instructions-service.test.ts index 61c58716ed7..cf0d64c1dd6 100644 --- a/server/src/__tests__/agent-instructions-service.test.ts +++ b/server/src/__tests__/agent-instructions-service.test.ts @@ -611,7 +611,7 @@ describe("agent instructions service", () => { await fs.mkdir(lockPath, { recursive: true }); await fs.writeFile( path.join(lockPath, "owner.json"), - `${JSON.stringify({ pid: process.pid, token: "live", createdAt: "1970-01-01T00:00:00.000Z" })}\n`, + `${JSON.stringify({ pid: process.pid, token: "live", createdAt: new Date().toISOString() })}\n`, "utf8", ); @@ -670,6 +670,87 @@ describe("agent instructions service", () => { await expect(fs.stat(lockPath)).rejects.toThrow(); }); + it("recovers a probe-less legacy lock after its PID is reused", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-legacy-stale-lock-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + const managedRoot = path.join( + paperclipHome, + "instances", + "test-instance", + "companies", + "company-1", + "agents", + "agent-1", + "instructions", + ); + const lockPath = `${managedRoot}.paperclip-materialize.lock`; + const token = randomUUID(); + await fs.mkdir(lockPath, { recursive: true }); + await fs.writeFile( + path.join(lockPath, "owner.json"), + `${JSON.stringify({ pid: process.pid, token, createdAt: "1970-01-01T00:00:00.000Z" })}\n`, + "utf8", + ); + const heartbeatPath = path.join(lockPath, `heartbeat-${token}`); + await fs.writeFile(heartbeatPath, "", "utf8"); + const staleHeartbeat = new Date(Date.now() - 60_000); + await fs.utimes(heartbeatPath, staleHeartbeat, staleHeartbeat); + + const result = await svc.materializeManagedBundle(agent, { "AGENTS.md": "# Stock\n" }); + + expect(result.materialization.action).toBe("added"); + await expect(fs.stat(lockPath)).rejects.toThrow(); + }); + + it("preserves a probe-less legacy lock when its live owner heartbeat is stale", async () => { + const paperclipHome = await makeTempDir("paperclip-agent-instructions-legacy-live-lock-"); + cleanupDirs.add(paperclipHome); + process.env.PAPERCLIP_HOME = paperclipHome; + process.env.PAPERCLIP_INSTANCE_ID = "test-instance"; + const svc = agentInstructionsService(); + const agent = makeAgent({}); + const managedRoot = path.join( + paperclipHome, + "instances", + "test-instance", + "companies", + "company-1", + "agents", + "agent-1", + "instructions", + ); + const lockPath = `${managedRoot}.paperclip-materialize.lock`; + const token = randomUUID(); + await fs.mkdir(lockPath, { recursive: true }); + await fs.writeFile( + path.join(lockPath, "owner.json"), + `${JSON.stringify({ pid: process.pid, token, createdAt: new Date().toISOString() })}\n`, + "utf8", + ); + const heartbeatPath = path.join(lockPath, `heartbeat-${token}`); + await fs.writeFile(heartbeatPath, "", "utf8"); + const staleHeartbeat = new Date(Date.now() - 60_000); + await fs.utimes(heartbeatPath, staleHeartbeat, staleHeartbeat); + + const materialization = svc.materializeManagedBundle(agent, { "AGENTS.md": "# Stock\n" }); + const firstOutcome = await Promise.race([ + materialization.then(() => "materialized"), + new Promise<"waiting">((resolve) => setTimeout(() => resolve("waiting"), 75)), + ]); + + expect(firstOutcome).toBe("waiting"); + const releasedLockPath = `${lockPath}.released`; + await fs.rename(lockPath, releasedLockPath); + await fs.rm(releasedLockPath, { recursive: true, force: true }); + await expect(materialization).resolves.toMatchObject({ + materialization: { action: "added" }, + }); + }); + it.skipIf(process.platform !== "linux")( "does not evict a live lock with a stale heartbeat when its identity scheme is unavailable", async () => { diff --git a/server/src/services/agent-instructions.ts b/server/src/services/agent-instructions.ts index a41b09aafb3..20290c3c81e 100644 --- a/server/src/services/agent-instructions.ts +++ b/server/src/services/agent-instructions.ts @@ -357,6 +357,20 @@ function processStartMarkersComparable(left: string, right: string): boolean { && left.slice(0, leftSeparator) === right.slice(0, rightSeparator); } +function processStartMarkerTimestamp(marker: string | null): number | null { + if (!marker?.startsWith("ps:")) return null; + const timestamp = Date.parse(marker.slice("ps:".length)); + return Number.isFinite(timestamp) ? timestamp : null; +} + +async function processStartedAfterLockCreation(pid: number, createdAt: unknown): Promise { + if (typeof createdAt !== "string") return false; + const lockCreatedAt = Date.parse(createdAt); + if (!Number.isFinite(lockCreatedAt)) return false; + const processStartedAt = processStartMarkerTimestamp(await processStartMarker(pid, "ps")); + return processStartedAt !== null && processStartedAt > lockCreatedAt; +} + async function removeStaleManagedBundleLock(lockPath: string): Promise { let shouldRemove = false; let ownerSnapshot: string | null = null; @@ -368,6 +382,7 @@ async function removeStaleManagedBundleLock(lockPath: string): Promise token?: unknown; probePath?: unknown; processStartMarker?: unknown; + createdAt?: unknown; }; const pid = typeof owner.pid === "number" && Number.isInteger(owner.pid) && owner.pid > 0 ? owner.pid @@ -395,10 +410,11 @@ async function removeStaleManagedBundleLock(lockPath: string): Promise ) { shouldRemove = recordedStart !== currentStart; } else { - // A delayed heartbeat does not prove that a live process has stopped - // materializing. Only a comparable process-start marker can safely - // distinguish a reused PID from the original owner. - shouldRemove = false; + // Probe-less legacy locks still record when they were created. A live + // process that started later cannot be the original owner, which lets + // us recover a reused PID without ever evicting an owner merely because + // its heartbeat was delayed. + shouldRemove = await processStartedAfterLockCreation(pid, owner.createdAt); } } else if (pid !== null) { shouldRemove = true;