diff --git a/server/src/__tests__/agent-directory-remote-checkpoint-lock.test.ts b/server/src/__tests__/agent-directory-remote-checkpoint-lock.test.ts new file mode 100644 index 00000000000..c0b99e1af91 --- /dev/null +++ b/server/src/__tests__/agent-directory-remote-checkpoint-lock.test.ts @@ -0,0 +1,171 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { sql } from "drizzle-orm"; +import type { CommandManagedRuntimeRunner } from "@paperclipai/adapter-utils/command-managed-runtime"; +import { agents, authUsers, closeFileStorageDb, companies, companyMemberships, createDb, environmentLeases, environments, heartbeatRuns, + principalPermissionGrants } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { agentInstructionWorkingCopyService } from "../services/agent-instruction-working-copies.js"; +import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; + +const execFile = promisify(execFileCallback); + +describe("remote agent directory checkpoint locking", () => { + let database: Awaited>; + let db: ReturnType; + let observer: ReturnType; + let home: string, companyId: string, agentId: string, userId: string, root: string, remoteCwd: string; + const entryFile = "AGENTS.md"; + const runnerDelay = { ms: 0 }; + const calls: Array<{ checkpointScript: boolean; directoryLockHeld: boolean }> = []; + let gate: { opened: Promise; reached: () => void; reachedPromise: Promise } | null = null; + + async function directoryLockHeld(): Promise { + const key = `agent-directory:${companyId}:${agentId}`; + const [row] = await observer.execute<{ held: number }>(sql`select count(*)::int as held from pg_locks + where locktype = 'advisory' and granted and objsubid = 1 + and ((classid::bigint << 32) | objid::bigint) = hashtextextended(${key}, 0)`); + return Number(row?.held ?? 0) > 0; + } + + const runner: CommandManagedRuntimeRunner = { + execute: async input => { + const startedAt = new Date().toISOString(); + const env = { ...process.env, ...input.env }; + const args = [...(input.args ?? [])]; + const checkpointScript = args.some(arg => arg.includes("--checkpoint")); + calls.push({ checkpointScript, directoryLockHeld: await directoryLockHeld() }); + if (checkpointScript && gate) { + const current = gate; + gate = null; + current.reached(); + await current.opened; + } + if (runnerDelay.ms) await new Promise(resolve => setTimeout(resolve, runnerDelay.ms)); + if (input.stdin != null && (args[0] === "-c" || args[0] === "-lc")) { + env.PAPERCLIP_TEST_STDIN = input.stdin; + args[1] = `printf '%s' "$PAPERCLIP_TEST_STDIN" | (${args[1]})`; + } + try { + const result = await execFile(input.command, args, { cwd: input.cwd, env, timeout: input.timeoutMs, maxBuffer: 32 * 1024 * 1024 }); + return { exitCode: 0, signal: null, timedOut: false, stdout: result.stdout, stderr: result.stderr, pid: null, startedAt }; + } catch (error) { + const e = error as { code?: number; signal?: NodeJS.Signals; stdout?: string; stderr?: string }; + return { exitCode: typeof e.code === "number" ? e.code : 1, signal: e.signal ?? null, timedOut: false, stdout: e.stdout ?? "", stderr: e.stderr ?? "", pid: null, startedAt }; + } + }, + }; + + async function warmRemoteRun() { + const executionTarget = { kind: "remote" as const, transport: "sandbox" as const, environmentId: randomUUID(), leaseId: randomUUID(), remoteCwd, runner }; + await db.insert(environments).values({ id: executionTarget.environmentId, name: `Checkpoint fixture ${executionTarget.environmentId}`, driver: "sandbox" }); + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId }); + await db.insert(environmentLeases).values({ id: executionTarget.leaseId, companyId, environmentId: executionTarget.environmentId, + heartbeatRunId: runId, provider: "daytona", providerLeaseId: "checkpoint-fixture-allocation" }); + const copies = agentInstructionWorkingCopyService(db); + const copy = (await copies.prepare({ companyId, agentId, runId, cwd: home, target: executionTarget, warm: true }))!; + return { copies, copy, executionTarget }; + } + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("agent-directory-remote-checkpoint-"); + db = createDb(database.connectionString); + observer = createDb(database.connectionString); + }, 90_000); + afterAll(async () => { + await observer?.$client.end({ timeout: 1 }); + if (db) { await closeFileStorageDb(db, 1); await db.$client.end({ timeout: 1 }); } + await database?.cleanup(); + }); + beforeEach(async () => { + home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "agent-directory-remote-checkpoint-"))); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_EPHEMERAL_SERVER", "true"); + runnerDelay.ms = 0; calls.length = 0; gate = null; + companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID(); + root = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "Owner", adapterConfig: {} }); + remoteCwd = path.join(home, "remote-task"); + await fs.mkdir(remoteCwd, { recursive: true }); + await db.insert(companies).values({ id: companyId, name: "Remote checkpoint locking", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(authUsers).values({ id: userId, name: "Editor", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(agents).values({ id: agentId, companyId, name: "Owner", + adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root, instructionsEntryFile: entryFile } }); + await db.insert(companyMemberships).values([ + { companyId, principalType: "user", principalId: userId, membershipRole: "operator" }, + { companyId, principalType: "agent", principalId: agentId, membershipRole: "member" }, + ]); + await db.insert(principalPermissionGrants).values({ companyId, principalType: "user", principalId: userId, + permissionKey: "agents:configure", scope: { agentIds: [agentId] } }); + await fs.mkdir(root, { recursive: true }); + await fs.writeFile(path.join(root, entryFile), "# Policy\n"); + }); + afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(home, { recursive: true, force: true }); + }); + + it("captures a turn's remote files without holding the directory lock and its database connection", async () => { + const { copies, copy, executionTarget } = await warmRemoteRun(); + await fs.writeFile(path.join(copy.executionRoot, "memory.txt"), "turn one\n"); + runnerDelay.ms = 200; + calls.length = 0; + const startedAt = performance.now(); + let lockedMs = 0; + let sampling = true; + const sampler = (async () => { + let last = performance.now(); + while (sampling) { + const held = await directoryLockHeld(); + const now = performance.now(); + if (held) lockedMs += now - last; + last = now; + await new Promise(resolve => setTimeout(resolve, 10)); + } + })(); + let saved: Awaited>; + let elapsedMs: number; + try { + saved = await copies.checkpointWarm({ companyId, runId: copy.runId, target: executionTarget }); + elapsedMs = performance.now() - startedAt; + } finally { + sampling = false; + await sampler; + } + expect(saved, JSON.stringify({ code: saved?.errorCode, message: saved?.errorMessage })).toMatchObject({ state: "warm_saved", errorCode: null }); + expect(await fs.readFile(path.join(root, "memory.txt"), "utf8")).toBe("turn one\n"); + console.info(JSON.stringify({ metric: "remote_checkpoint_directory_lock", runnerCalls: calls.length, + runnerCallsWhileLocked: calls.filter(call => call.directoryLockHeld).length, lockedMs: Math.round(lockedMs), elapsedMs: Math.round(elapsedMs) })); + expect(calls.filter(call => call.checkpointScript)).toHaveLength(1); + expect(calls.filter(call => call.directoryLockHeld)).toEqual([]); + expect(lockedMs).toBeLessThan(elapsedMs / 2); + }); + + it("discards an early capture when a competing collector changes the copy, and keeps the collector's outcome", async () => { + const { copies, copy, executionTarget } = await warmRemoteRun(); + await fs.writeFile(path.join(copy.executionRoot, "memory.txt"), "final turn\n"); + let open!: () => void, reached!: () => void; + const opened = new Promise(resolve => { open = resolve; }); + const reachedPromise = new Promise(resolve => { reached = resolve; }); + gate = { opened, reached, reachedPromise }; + const checkpoint = copies.checkpointWarm({ companyId, runId: copy.runId, target: executionTarget }); + await reachedPromise; + try { + const competitor = agentInstructionWorkingCopyService(db); + const collected = await competitor.collectStopped({ companyId, runId: copy.runId, target: executionTarget }); + expect(collected).toMatchObject({ state: "saved", errorCode: null }); + } finally { open(); } + const result = await checkpoint; + const final = await copies.get(companyId, copy.runId); + expect(result).toEqual(final); + expect(final).toMatchObject({ state: "saved", errorCode: null, receipt: { cleanupPending: false } }); + expect(await fs.readFile(path.join(root, "memory.txt"), "utf8")).toBe("final turn\n"); + await expect(fs.stat(copy.executionRoot)).rejects.toMatchObject({ code: "ENOENT" }); + expect((await fs.readdir(path.dirname(copy.localRoot)).catch(() => [])).filter(name => name.startsWith("checkpoint-"))).toEqual([]); + }); +}); diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index 5582092c74a..31c36c79183 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -404,9 +404,25 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string } else await replacement?.cleanupWorkspaceSnapshot?.(); return adopted; } - async function checkpoint(row: Copy, target?: AdapterExecutionTarget | null, stopped = false): Promise { + type Capture = Awaited>; + type Precapture = { row: Copy; target: AdapterExecutionTarget; captured: Capture }; + async function precapture(row: Copy, target?: AdapterExecutionTarget | null): Promise { + if (!disposableRemoteCopy(row) || row.receipt?.warm !== true || !await owns(row) || row.state === "superseded") return undefined; + const resolved = target ?? transports.get(key(row))?.target; + if (resolved?.kind !== "remote" || row.localRoot !== projectionRoot(row)) return undefined; + try { + const previous = row.receipt?.checkpointManifest as AgentFileManifest | undefined ?? checkpointBaseline(baseline(row)); + const captured = await captureAgentFileCheckpoint({ runId: row.runId, localRoot: row.localRoot, executionRoot: row.executionRoot, previous, target: resolved }); + return { row, target: resolved, captured }; + } catch (error) { + logger.warn({ err: error, runId: row.runId }, "Agent file checkpoint capture before the directory lock failed; capturing under the lock"); + return undefined; + } + } + async function checkpoint(row: Copy, target?: AdapterExecutionTarget | null, stopped = false, early?: Precapture): Promise { if (!await owns(row) || row.state === "superseded") return row; target ??= transports.get(key(row))?.target; + const reusable = early && early.target === target && isDeepStrictEqual(adoptionAuthority(early.row), adoptionAuthority(row)) ? early.captured : undefined; if (row.location !== "local" && target?.kind !== "remote") { // A missing controller cache is not the remote directory. Retain the // pending save until its exact remote collector or destruction proof exists. @@ -416,10 +432,11 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string } let failure: unknown; for (let attempt = 0; attempt < 3; attempt++) { - let captured: Awaited> | undefined; + let captured: Capture | undefined; try { const previous = row.receipt?.checkpointManifest as AgentFileManifest | undefined ?? checkpointBaseline(baseline(row)); - captured = await captureAgentFileCheckpoint({ runId: row.runId, localRoot: row.localRoot, executionRoot: row.executionRoot, previous, target }); + captured = attempt === 0 && reusable ? reusable + : await captureAgentFileCheckpoint({ runId: row.runId, localRoot: row.localRoot, executionRoot: row.executionRoot, previous, target }); const snapshot = checkpointSnapshot(captured.manifest); const nextHash = directorySnapshotSha256(snapshot); const changed = nextHash !== row.baseHash; @@ -442,7 +459,7 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string if (error instanceof HttpError && error.status < 500) break; } finally { - await captured?.cleanup().catch(error => logger.warn({ err: error, runId: row.runId }, "Agent checkpoint scratch cleanup deferred to session retirement")); + if (captured !== early?.captured) await captured?.cleanup().catch(error => logger.warn({ err: error, runId: row.runId }, "Agent checkpoint scratch cleanup deferred to session retirement")); } } const rejection = failure instanceof HttpError && failure.status === 403 ? failure.message : null; @@ -671,7 +688,14 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string } return { prepare: (input: Parameters[0]) => prepareCopy(input), hasChanges, adopt, canReuse, recoverUnavailable, recoverStoppedRemote: (row: Copy) => serial(row, current => current.receipt?.retainedByRunId ? Promise.resolve(current) : recoverStoppedRemote(current), "agent_directory_collect"), - checkpointWarm: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => current.receipt?.warm === true ? checkpoint(current, target) : Promise.resolve(current), "agent_directory_checkpoint"), + checkpointWarm: async (row: Copy, target?: AdapterExecutionTarget | null) => { + const early = await precapture(row, target); + try { + return await serial(row, current => current.receipt?.warm === true ? checkpoint(current, target, false, early) : Promise.resolve(current), "agent_directory_checkpoint"); + } finally { + await early?.captured.cleanup().catch(error => logger.warn({ err: error, runId: row.runId }, "Agent checkpoint scratch cleanup deferred to session retirement")); + } + }, collectStopped: (row: Copy, target?: AdapterExecutionTarget | null) => serial(row, current => collectStopped(current, target), "agent_directory_collect"), release: async (row: Copy) => { let current = await get(row.companyId, row.runId);