diff --git a/server/src/__tests__/agent-file-restore-locking.test.ts b/server/src/__tests__/agent-file-restore-locking.test.ts new file mode 100644 index 00000000000..e804283d0d3 --- /dev/null +++ b/server/src/__tests__/agent-file-restore-locking.test.ts @@ -0,0 +1,199 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { Readable } from "node:stream"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { agentFileRoots, agents, authUsers, closeFileStorageDb, companies, companyMemberships, createDb, principalPermissionGrants } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { agentFileStore, fileHash } from "../services/agent-file-store.js"; +import { agentInstructionRevisionService } from "../services/agent-instruction-revisions.js"; +import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; + +const objects = vi.hoisted(() => ({ + bytes: new Map(), + downloads: [] as Array<{ objectKey: string; agentRowLocked: boolean | null }>, + probe: null as null | (() => Promise), + gate: null as null | { opened: Promise; reached: () => void }, + delayMs: 0, +})); +vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ + provider: "s3", + async putFile(input: { companyId: string; namespace: string; body: Readable }) { + const chunks: Buffer[] = []; + for await (const chunk of input.body) chunks.push(Buffer.from(chunk)); + const objectKey = `${input.companyId}/${input.namespace}/${randomUUID()}`; + objects.bytes.set(objectKey, Buffer.concat(chunks)); + return { objectKey }; + }, + async getObject(_companyId: string, objectKey: string) { + const body = objects.bytes.get(objectKey); + if (!body) throw new Error("fixture object missing"); + objects.downloads.push({ objectKey, agentRowLocked: objects.probe ? await objects.probe() : null }); + const gate = objects.gate; + if (gate) { objects.gate = null; gate.reached(); await gate.opened; } + if (objects.delayMs) await new Promise(resolve => setTimeout(resolve, objects.delayMs)); + return { stream: Readable.from([body]) }; + }, +}) })); + +describe("agent file restore locking", () => { + let database: Awaited>; + let db: ReturnType; + let observer: ReturnType; + let home: string, companyId: string, agentId: string, userId: string, root: string; + const entryFile = "AGENTS.md"; + const board = () => ({ type: "board" as const, userId, source: "session" as const }); + const target = () => ({ companyId, agentId }); + + async function agentRowLocked(): Promise { + try { + await observer.transaction(async tx => { await tx.execute(sql`select id from agents where id = ${agentId} for update nowait`); }); + return false; + } catch (error) { + const code = (error as { code?: string; cause?: { code?: string } }).code ?? (error as { cause?: { code?: string } }).cause?.code; + if (code === "55P03") return true; + throw error; + } + } + async function head() { + const [row] = await db.select().from(agentFileRoots).where(and(eq(agentFileRoots.companyId, companyId), eq(agentFileRoots.agentId, agentId))); + return row!; + } + async function loseLocalProjection() { + await fs.rm(root, { recursive: true, force: true }); + await fs.rm(path.join(path.dirname(root), "file-sync", "durable-generation"), { force: true }); + } + async function stagingLeftovers() { + return (await fs.readdir(path.dirname(root))).filter(name => name.startsWith(".agent-files-")); + } + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("agent-file-restore-locking-"); + db = createDb(database.connectionString); + observer = createDb(database.connectionString); + }, 90_000); + afterAll(async () => { + await observer?.$client.end({ timeout: 1 }); + if (db) { await closeFileStorageDb(db, 1); await db.$client.end({ timeout: 1 }); } + await database?.cleanup(); + }); + beforeEach(async () => { + home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "agent-file-restore-locking-"))); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + objects.bytes.clear(); objects.downloads.length = 0; objects.probe = null; objects.gate = null; objects.delayMs = 0; + companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID(); + root = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "Owner", adapterConfig: {} }); + await db.insert(companies).values({ id: companyId, name: "Restore locking", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(authUsers).values({ id: userId, name: "Editor", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(agents).values({ id: agentId, companyId, name: "Owner", + adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root, instructionsEntryFile: entryFile } }); + await db.insert(companyMemberships).values([ + { companyId, principalType: "user", principalId: userId, membershipRole: "operator" }, + { companyId, principalType: "agent", principalId: agentId, membershipRole: "member" }, + ]); + await db.insert(principalPermissionGrants).values({ companyId, principalType: "user", principalId: userId, + permissionKey: "agents:configure", scope: { agentIds: [agentId] } }); + await fs.mkdir(root, { recursive: true }); + await fs.writeFile(path.join(root, entryFile), "# Policy\n"); + expect((await agentInstructionRevisionService(db).readCommittedForRuntime(target()))?.content).toBe("# Policy\n"); + expect((await head()).entries.map(entry => entry.path)).toEqual([entryFile]); + }); + afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(home, { recursive: true, force: true }); + }); + + it("downloads a stale projection without holding the agent row lock", async () => { + await loseLocalProjection(); + objects.probe = agentRowLocked; + objects.delayMs = 300; + let lockedMs = 0; + let sampling = true; + const sampler = (async () => { + let last = performance.now(); + while (sampling) { + const locked = await agentRowLocked(); + const now = performance.now(); + if (locked) lockedMs += now - last; + last = now; + await new Promise(resolve => setTimeout(resolve, 10)); + } + })(); + const startedAt = performance.now(); + let current: Awaited["readCommittedForRuntime"]>>; + let elapsedMs: number; + try { + current = await agentInstructionRevisionService(db).readCommittedForRuntime(target()); + elapsedMs = performance.now() - startedAt; + } finally { + sampling = false; + await sampler; + } + console.info(JSON.stringify({ metric: "agent_file_restore_row_lock", downloads: objects.downloads.length, + downloadsWhileLocked: objects.downloads.filter(download => download.agentRowLocked).length, lockedMs: Math.round(lockedMs), elapsedMs: Math.round(elapsedMs) })); + expect(current?.content).toBe("# Policy\n"); + expect(objects.downloads).toEqual([{ objectKey: expect.any(String), agentRowLocked: false }]); + expect(lockedMs).toBeLessThan(150); + expect(await fs.readFile(path.join(path.dirname(root), "file-sync", "durable-generation"), "utf8")).toBe((await head()).generation); + expect(await stagingLeftovers()).toEqual([]); + }); + + it("lets a writer commit during the download and then serves the writer's head, not the stale download", async () => { + await loseLocalProjection(); + let open!: () => void, reached!: () => void; + const opened = new Promise(resolve => { open = resolve; }); + const reachedPromise = new Promise(resolve => { reached = resolve; }); + objects.gate = { opened, reached }; + const before = (await head()).generation; + const reader = agentInstructionRevisionService(db).readCommittedForRuntime(target()); + await reachedPromise; + try { + const written = await Promise.race([ + agentFileStore(db).write({ companyId, agentId, path: entryFile, bytes: Buffer.from("# Policy v2\n"), baseHash: fileHash(Buffer.from("# Policy\n")) }, board()), + new Promise((_, reject) => setTimeout(() => reject(new Error("the writer waited for the reader's download")), 5_000)), + ]); + expect(written).toMatchObject({ changed: true }); + } finally { open(); } + const current = await reader; + const after = await head(); + expect(after.generation).not.toBe(before); + expect(current?.content).toBe("# Policy v2\n"); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe("# Policy v2\n"); + expect(await fs.readFile(path.join(path.dirname(root), "file-sync", "durable-generation"), "utf8")).toBe(after.generation); + expect(await stagingLeftovers()).toEqual([]); + }); + + it("fails once, without the row lock and without retrying, when the staged download fails", async () => { + await loseLocalProjection(); + const attempts: boolean[] = []; + objects.probe = async () => { + attempts.push(await agentRowLocked()); + throw new Error("object storage unavailable"); + }; + await expect(agentInstructionRevisionService(db).readCommittedForRuntime(target())).rejects.toThrow("object storage unavailable"); + expect(attempts).toEqual([false]); + expect(await stagingLeftovers()).toEqual([]); + }); + + it("restores under the lock as before when the head keeps changing during every download", async () => { + await loseLocalProjection(); + const store = agentFileStore(db); + let version = 0; + const realGet = objects.probe; + objects.probe = async () => { + if (version < 2) { + version++; + await db.update(agentFileRoots).set({ generation: randomUUID() }) + .where(and(eq(agentFileRoots.companyId, companyId), eq(agentFileRoots.agentId, agentId))); + } + return realGet ? realGet() : false; + }; + expect((await store.read(companyId, agentId, entryFile, board()))?.toString("utf8")).toBe("# Policy\n"); + expect(objects.downloads).toHaveLength(3); + expect(await fs.readFile(path.join(path.dirname(root), "file-sync", "durable-generation"), "utf8")).toBe((await head()).generation); + expect(await stagingLeftovers()).toEqual([]); + }); +}); diff --git a/server/src/services/agent-file-durability.ts b/server/src/services/agent-file-durability.ts index 058175f6c19..4ba1f60e058 100644 --- a/server/src/services/agent-file-durability.ts +++ b/server/src/services/agent-file-durability.ts @@ -1,3 +1,4 @@ +import { AsyncLocalStorage } from "node:async_hooks"; import { createHash, randomUUID } from "node:crypto"; import fs from "node:fs/promises"; import { constants } from "node:fs"; @@ -104,52 +105,25 @@ export async function restoreDurableAgentFiles(tx: Tx, owner: Owner, root: strin throw error; }); if (generation === head.generation && await fs.stat(root).then(s => s.isDirectory()).catch(() => false)) return true; + const scope = restoreScope.getStore(); + const deferrable = scope?.deferring === true && !scope.used; + if (scope) scope.used = true; + const stageKey = `${owner.companyId}:${owner.id}:${root}`; + const staged = scope?.staged.get(stageKey); + if (staged?.generation === head.generation) { + scope!.staged.delete(stageKey); + try { + await invalidateAgentFileProjection(root); + await fs.rm(root, { recursive: true, force: true }); + await fs.rename(staged.directory, root); + await mark(root, head.generation); + return true; + } finally { await fs.rm(staged.directory, { recursive: true, force: true }); } + } + if (deferrable) throw new AgentFilesRestoreDeferred(owner, root, head); const temporary = path.join(path.dirname(root), `.agent-files-${randomUUID()}`); - await fs.mkdir(temporary, { recursive: true, mode: 0o700 }); - const failed = new AbortController(); - const signal = AbortSignal.any([AbortSignal.timeout(30_000), failed.signal]); try { - // Bound open streams and file handles while avoiding one network round trip - // per file in series. Await every worker before removing a failed projection. - const entries = head.entries; - let next = 0; - const restoreEntry = async (entry: DurableAgentFileEntry) => { - signal.throwIfAborted(); - const relative = instructionPath(entry.path); - if (entry.kind === "dir") { - await assertInstructionPathSafe(temporary, `${relative}/.path-check`); - await fs.mkdir(path.join(temporary, relative), { recursive: true, mode: 0o700 }); return; - } - const filename = await assertInstructionPathSafe(temporary, relative); - await fs.mkdir(path.dirname(filename), { recursive: true, mode: 0o700 }); - const handle = await fs.open(filename, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, entry.mode & 0o777); - try { - const hash = createHash("sha256"); - let size = 0; - if (entry.objectKey !== null) { - const object = await service.getObject(owner.companyId, entry.objectKey, { signal }); - await pipeline(object.stream, new Transform({ transform(chunk: Buffer, _encoding, callback) { - size += chunk.length; - if (size > entry.size) return callback(new Error("Agent file exceeds its committed size")); - hash.update(chunk); callback(null, chunk); - } }), handle.createWriteStream(), { signal }); - } - if (size !== entry.size || hash.digest("hex") !== entry.sha256) throw new Error("Agent file does not match its committed digest"); - } finally { await handle.close(); } - }; - const workers = Array.from({ length: Math.min(8, entries.length) }, async () => { - try { - while (next < entries.length) { - signal.throwIfAborted(); - await restoreEntry(entries[next++]!); - } - } catch (error) { - failed.abort(error); - throw error; - } - }); - await Promise.allSettled(workers); - signal.throwIfAborted(); + await downloadAgentFiles(service, owner, head.entries, temporary); await invalidateAgentFileProjection(root); await fs.rm(root, { recursive: true, force: true }); await fs.rename(temporary, root); @@ -158,6 +132,88 @@ export async function restoreDurableAgentFiles(tx: Tx, owner: Owner, root: strin } finally { await fs.rm(temporary, { recursive: true, force: true }); } } +async function downloadAgentFiles(service: ReturnType, owner: Pick, + entries: DurableAgentFileEntry[], temporary: string) { + await fs.mkdir(temporary, { recursive: true, mode: 0o700 }); + const failed = new AbortController(); + const signal = AbortSignal.any([AbortSignal.timeout(30_000), failed.signal]); + let next = 0; + const restoreEntry = async (entry: DurableAgentFileEntry) => { + signal.throwIfAborted(); + const relative = instructionPath(entry.path); + if (entry.kind === "dir") { + await assertInstructionPathSafe(temporary, `${relative}/.path-check`); + await fs.mkdir(path.join(temporary, relative), { recursive: true, mode: 0o700 }); return; + } + const filename = await assertInstructionPathSafe(temporary, relative); + await fs.mkdir(path.dirname(filename), { recursive: true, mode: 0o700 }); + const handle = await fs.open(filename, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, entry.mode & 0o777); + try { + const hash = createHash("sha256"); + let size = 0; + if (entry.objectKey !== null) { + const object = await service.getObject(owner.companyId, entry.objectKey, { signal }); + await pipeline(object.stream, new Transform({ transform(chunk: Buffer, _encoding, callback) { + size += chunk.length; + if (size > entry.size) return callback(new Error("Agent file exceeds its committed size")); + hash.update(chunk); callback(null, chunk); + } }), handle.createWriteStream(), { signal }); + } + if (size !== entry.size || hash.digest("hex") !== entry.sha256) throw new Error("Agent file does not match its committed digest"); + } finally { await handle.close(); } + }; + const workers = Array.from({ length: Math.min(8, entries.length) }, async () => { + try { + while (next < entries.length) { + signal.throwIfAborted(); + await restoreEntry(entries[next++]!); + } + } catch (error) { + failed.abort(error); + throw error; + } + }); + await Promise.allSettled(workers); + signal.throwIfAborted(); +} + +class AgentFilesRestoreDeferred extends Error { + constructor(readonly owner: Owner, readonly root: string, readonly head: Head) { super("agent_files_restore_deferred"); } +} +type StagedAgentFiles = { generation: string; directory: string }; +type RestoreScope = { staged: Map; deferring: boolean; used: boolean }; +const restoreScope = new AsyncLocalStorage(); +const RESTORE_DEFERRALS = 2; + +export async function withAgentFilesRestoredBeforeLock(work: () => Promise): Promise { + const scope: RestoreScope = { staged: new Map(), deferring: true, used: false }; + try { + for (let attempt = 0; ; attempt++) { + scope.deferring = attempt < RESTORE_DEFERRALS; + scope.used = false; + try { + return await restoreScope.run(scope, work); + } catch (error) { + if (!(error instanceof AgentFilesRestoreDeferred)) throw error; + const stageKey = `${error.owner.companyId}:${error.owner.id}:${error.root}`; + const previous = scope.staged.get(stageKey); + scope.staged.delete(stageKey); + if (previous) await fs.rm(previous.directory, { recursive: true, force: true }); + const directory = path.join(path.dirname(error.root), `.agent-files-${randomUUID()}`); + try { + await downloadAgentFiles(storage(), error.owner, error.head.entries, directory); + scope.staged.set(stageKey, { generation: error.head.generation, directory }); + } catch (failure) { + await fs.rm(directory, { recursive: true, force: true }); + throw failure; + } + } + } + } finally { + await Promise.all([...scope.staged.values()].map(staged => fs.rm(staged.directory, { recursive: true, force: true }))); + } +} + /** Objects uploaded before the agent row lock, by content (`sha256:size`). */ export type PreparedAgentFileObjects = ReadonlyMap; const contentKey = (sha256: string, size: number) => `${sha256}:${size}`; diff --git a/server/src/services/agent-file-store.ts b/server/src/services/agent-file-store.ts index a5813d54317..06074ce50b9 100644 --- a/server/src/services/agent-file-store.ts +++ b/server/src/services/agent-file-store.ts @@ -16,7 +16,7 @@ import { authorizeInstructionCommit, authorizeInstructionRead } from "./agent-in import { assertInstructionPathSafe, instructionPath, instructionBytes, materializeInstructionBytes, readInstructionBytes, MAX_INSTRUCTION_BYTES } from "./agent-instruction-files.js"; import { agentInstructionsBundleMode, deriveBundleState, resolveManagedInstructionsRoot } from "./agent-instructions.js"; import type { AuthorizationActor } from "./authorization.js"; -import { agentFileProjectionCurrent, durableAgentFilesEnabled, invalidateAgentFileProjection, persistDurableAgentFiles, restoreDurableAgentFiles, uploadAgentFilesBeforeCommit, type PreparedAgentFileObjects } from "./agent-file-durability.js"; +import { agentFileProjectionCurrent, durableAgentFilesEnabled, invalidateAgentFileProjection, persistDurableAgentFiles, restoreDurableAgentFiles, uploadAgentFilesBeforeCommit, withAgentFilesRestoredBeforeLock, type PreparedAgentFileObjects } from "./agent-file-durability.js"; type Tx = Parameters[0]>[0]; type Agent = typeof agents.$inferSelect; @@ -195,7 +195,7 @@ export function agentFileStore(db: Db) { async function locked(companyId: string, agentId: string, actor: AuthorizationActor, write: boolean, fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor, onRollback: (restore: () => Promise) => void) => Promise, texts: string[] = [], prepared?: PreparedAgentFileObjects) { - return db.transaction(async tx => { + return withAgentFilesRestoredBeforeLock(() => db.transaction(async tx => { if (write) await assertAgentInteraction(tx, { companyId, actorAgentId: actor.type === "agent" ? actor.agentId : null, operation: "configure", targetAgentId: agentId, lockAgentIds: texts.flatMap(extractAgentMentionIds), lock: "update" }); const [agent] = await tx.select().from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))).for("update"); @@ -212,7 +212,7 @@ export function agentFileStore(db: Db) { await restore?.(); throw error; } - }); + })); } async function audit(tx: Tx, agent: Agent, actor: AuthorizationActor, details: Record) { const origin = currentPluginInteractionOrigin(); diff --git a/server/src/services/agent-instruction-revisions.ts b/server/src/services/agent-instruction-revisions.ts index babbbc11e4d..cffe2bca42e 100644 --- a/server/src/services/agent-instruction-revisions.ts +++ b/server/src/services/agent-instruction-revisions.ts @@ -4,6 +4,7 @@ import { assertAgentInteraction } from "./agent-interaction-policy.js"; import { updateAgentConfigurationInTransaction } from "./agent-configuration-transaction.js"; import { createHash } from "node:crypto"; import { adoptAgentFiles, agentFileToken, isAgentFileSaveRecorded } from "./agent-file-store.js"; +import { withAgentFilesRestoredBeforeLock } from "./agent-file-durability.js"; import { invalidateAgentFileProjection, persistDurableAgentFiles } from "./agent-file-durability.js"; import { and, desc, eq, getTableColumns, lt, or, sql } from "drizzle-orm"; import { @@ -193,28 +194,28 @@ export function agentInstructionRevisionService(db: Db) { target: InstructionTarget, actor: AuthorizationActor, ): Promise { - return db.transaction(async (tx) => { + return withAgentFilesRestoredBeforeLock(() => db.transaction(async (tx) => { const state = await lockTarget(tx, target, actor); const bound = await authorizeRead(tx, actor, target); const row = await currentFile(tx, target, state, bound); return row ? snapshot(row) : null; - }); + })); } /** Trusted orchestration read for the already authorized run target. This * never seeds or writes, and is not exposed through HTTP or model tools. */ async function readCommittedForRuntime(target: InstructionTarget): Promise { - return db.transaction(async (tx) => { + return withAgentFilesRestoredBeforeLock(() => db.transaction(async (tx) => { const state = await lockTarget(tx, target); const current = await currentFile(tx, target, state); return current ? snapshot(current) : null; - }); + })); } /** Rebuild disk from the current committed head under the same lock as commits. Safe after restart. */ async function materializeCurrent(target: InstructionTarget): Promise { - await db.transaction(async (tx) => { + await withAgentFilesRestoredBeforeLock(() => db.transaction(async (tx) => { const state = await lockTarget(tx, target); await adoptAgentFiles(tx, state.agent); - }); + })); } async function commitInternal( input: