diff --git a/server/src/__tests__/agent-file-checkpoint-locking.test.ts b/server/src/__tests__/agent-file-checkpoint-locking.test.ts new file mode 100644 index 00000000000..aed57fd0181 --- /dev/null +++ b/server/src/__tests__/agent-file-checkpoint-locking.test.ts @@ -0,0 +1,359 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { randomUUID } from "node:crypto"; +import { Readable } from "node:stream"; +import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import { and, eq, sql } from "drizzle-orm"; +import { agentFileRoots, agents, authUsers, closeFileStorageDb, companies, companyMemberships, createDb, fileStorageDb, heartbeatRuns, + issueThreadInteractions, issues, principalPermissionGrants } from "@paperclipai/db"; +import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { agentFileStore, fileHash } from "../services/agent-file-store.js"; +import { agentInstructionWorkingCopyService } from "../services/agent-instruction-working-copies.js"; +import { resolveManagedInstructionsRoot } from "../services/agent-instructions.js"; + +/** Object storage fixture. Each upload records whether the agent row was locked while it ran. */ +const objects = vi.hoisted(() => ({ + bytes: new Map(), + uploads: [] as Array<{ objectKey: string; sha256: string; agentRowLocked: boolean | null }>, + probe: null as null | (() => Promise), + gate: null as null | Promise, + started: null as null | (() => void), + delayMs: 0, +})); +vi.mock("../storage/index.js", () => ({ getStorageService: () => ({ + provider: "s3", + async putFile(input: { companyId: string; namespace: string; body: Readable; byteSize: number; sha256: string }) { + const chunks: Buffer[] = []; + for await (const chunk of input.body) chunks.push(Buffer.from(chunk)); + const body = Buffer.concat(chunks); + expect(body.length).toBe(input.byteSize); + expect(fileHash(body)).toBe(input.sha256); + const agentRowLocked = objects.probe ? await objects.probe() : null; + objects.started?.(); + if (objects.gate) await objects.gate; + if (objects.delayMs) await new Promise(resolve => setTimeout(resolve, objects.delayMs)); + const objectKey = `${input.companyId}/${input.namespace}/${randomUUID()}`; + objects.bytes.set(objectKey, body); + objects.uploads.push({ objectKey, sha256: input.sha256, agentRowLocked }); + return { objectKey }; + }, + async getObject(_companyId: string, objectKey: string) { + const body = objects.bytes.get(objectKey); + if (!body) throw new Error("fixture object missing"); + return { stream: Readable.from([body]) }; + }, +}) })); + +describe("agent file checkpoint locking", () => { + let database: Awaited>; + let db: ReturnType; + let observer: ReturnType; + let copies: ReturnType; + let home: string, companyId: string, agentId: string, userId: string, root: string; + const entryFile = "AGENTS.md"; + const board = () => ({ type: "board" as const, userId, source: "session" as const }); + + /** True when another transaction holds the agent row lock. */ + async function agentRowLocked(): Promise { + try { + await observer.transaction(async tx => { await tx.execute(sql`select id from agents where id = ${agentId} for update nowait`); }); + return false; + } catch (error) { + const code = (error as { code?: string; cause?: { code?: string } }).code ?? (error as { cause?: { code?: string } }).cause?.code; + if (code === "55P03") return true; + throw error; + } + } + async function warmRun() { + const runId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: runId, companyId, agentId, invocationSource: "on_demand", responsibleUserId: userId }); + return (await copies.prepare({ companyId, agentId, runId, cwd: home, warm: true }))!; + } + async function head() { + const [row] = await db.select().from(agentFileRoots).where(and(eq(agentFileRoots.companyId, companyId), eq(agentFileRoots.agentId, agentId))); + return row!; + } + /** Every committed file must resolve to stored bytes with its digest. */ + async function expectHeadRestorable() { + for (const entry of (await head()).entries) { + if (entry.kind !== "file" || entry.size === 0) continue; + const bytes = objects.bytes.get(entry.objectKey!); + expect(bytes, entry.path).toBeDefined(); + expect(fileHash(bytes!)).toBe(entry.sha256); + } + } + /** Statements sent by both pools while the callback runs. */ + async function recordStatements(fn: () => Promise): Promise<{ result: T; statements: string[] }> { + const statements: string[] = []; + const clients = [db.$client, fileStorageDb(db).$client] as unknown as Array<{ options: { debug?: unknown } }>; + for (const client of clients) client.options.debug = (_connection: number, query: string) => { statements.push(query); }; + try { return { result: await fn(), statements }; } + finally { for (const client of clients) delete client.options.debug; } + } + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("agent-file-checkpoint-locking-"); + db = createDb(database.connectionString); + observer = createDb(database.connectionString); + copies = agentInstructionWorkingCopyService(db); + }, 90_000); + afterAll(async () => { + await observer?.$client.end({ timeout: 1 }); + if (db) { await closeFileStorageDb(db, 1); await db.$client.end({ timeout: 1 }); } + await database?.cleanup(); + }); + beforeEach(async () => { + home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "agent-file-checkpoint-locking-"))); + vi.stubEnv("PAPERCLIP_HOME", home); + vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true"); + objects.bytes.clear(); objects.uploads.length = 0; objects.probe = null; objects.gate = null; objects.started = null; objects.delayMs = 0; + companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID(); + root = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "Owner", adapterConfig: {} }); + await db.insert(companies).values({ id: companyId, name: "Checkpoint locking", issuePrefix: randomUUID().slice(0, 8) }); + await db.insert(authUsers).values({ id: userId, name: "Editor", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() }); + await db.insert(agents).values({ id: agentId, companyId, name: "Owner", + adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root, instructionsEntryFile: entryFile } }); + await db.insert(companyMemberships).values([ + { companyId, principalType: "user", principalId: userId, membershipRole: "operator" }, + { companyId, principalType: "agent", principalId: agentId, membershipRole: "member" }, + ]); + await db.insert(principalPermissionGrants).values({ companyId, principalType: "user", principalId: userId, + permissionKey: "agents:configure", scope: { agentIds: [agentId] } }); + await fs.mkdir(root, { recursive: true }); + await fs.writeFile(path.join(root, entryFile), "# Policy\n"); + }); + afterEach(async () => { + vi.unstubAllEnvs(); + await fs.rm(home, { recursive: true, force: true }); + }); + + it("saves an unchanged warm checkpoint without the agent row lock or a head write", async () => { + const copy = await warmRun(); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + const generation = (await head()).generation; + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let locked!: () => void; + const lockTaken = new Promise(resolve => { locked = resolve; }); + const holder = observer.transaction(async tx => { + await tx.execute(sql`select id from agents where id = ${agentId} for update`); + locked(); + await held; + }); + await lockTaken; + try { + const { result, statements } = await recordStatements(() => Promise.race([ + copies.checkpointWarm({ companyId, runId: copy.runId }), + new Promise((_, reject) => setTimeout(() => reject(new Error("checkpoint waited for the agent row lock")), 5_000)), + ])); + expect(result?.state).toBe("warm_saved"); + expect(result?.receipt?.checkpointState).toBe("unchanged"); + expect(statements.filter(statement => /for (update|no key update|share)/i.test(statement))).toEqual([]); + // One unlocked read of the head generation; no head write. + expect(statements.filter(statement => statement.includes("agent_file_roots")).map(statement => statement.trim().split(/\s/)[0]!.toLowerCase())).toEqual(["select"]); + } finally { + release(); + await holder; + } + expect((await head()).generation).toBe(generation); + }); + + it("adopts the agent's files into a durable head on an unchanged checkpoint when no head exists", async () => { + const copy = await warmRun(); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + // Durable files were turned on after this run started: the agent has local files and no head. + await db.delete(agentFileRoots).where(and(eq(agentFileRoots.companyId, companyId), eq(agentFileRoots.agentId, agentId))); + await fs.rm(path.join(path.dirname(root), "file-sync", "durable-adoption.json"), { force: true }); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + expect((await head())?.entries.map(entry => entry.path)).toContain(entryFile); + await expectHeadRestorable(); + }); + + it("takes the locked path for an unchanged checkpoint when the local projection is not marked at the head", async () => { + const copy = await warmRun(); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + await fs.rm(path.join(path.dirname(root), "file-sync", "durable-generation"), { force: true }); + const { result, statements } = await recordStatements(() => copies.checkpointWarm({ companyId, runId: copy.runId })); + expect(result?.state).toBe("warm_saved"); + expect(statements.some(statement => /for update/i.test(statement))).toBe(true); + expect(await fs.readFile(path.join(path.dirname(root), "file-sync", "durable-generation"), "utf8")).toBe((await head()).generation); + }); + + it("uploads changed bytes before it takes the agent row lock, then commits the uploaded object", async () => { + const copy = await warmRun(); + objects.uploads.length = 0; + objects.probe = agentRowLocked; + await fs.writeFile(path.join(copy.localRoot, "notes.md"), "turn one"); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + expect(objects.uploads).toEqual([{ objectKey: expect.any(String), sha256: fileHash(Buffer.from("turn one")), agentRowLocked: false }]); + const entry = (await head()).entries.find(candidate => candidate.path === "notes.md"); + expect(entry).toMatchObject({ kind: "file", sha256: fileHash(Buffer.from("turn one")), objectKey: objects.uploads[0]!.objectKey }); + await expectHeadRestorable(); + }); + + it("uploads nothing when the run lost its save permission before the checkpoint", async () => { + const copy = await warmRun(); + const before = await head(); + objects.uploads.length = 0; + await db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, companyId)); + await db.update(companyMemberships).set({ membershipRole: "member" }).where(eq(companyMemberships.companyId, companyId)); + await fs.writeFile(path.join(copy.localRoot, "refused.md"), "must not upload"); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.errorCode).toBe("AGENT_FILES_CHECKPOINT_UNSTABLE"); + expect(objects.uploads).toEqual([]); + expect((await head()).generation).toBe(before.generation); + }); + + it("consumes an accepted change confirmation once, in the commit of the changed checkpoint only", async () => { + const copy = await warmRun(); + const sourceRunId = randomUUID(), issueId = randomUUID(), interactionId = randomUUID(); + await db.insert(heartbeatRuns).values({ id: sourceRunId, companyId, agentId, status: "succeeded" }); + await db.insert(issues).values({ id: issueId, companyId, title: "Review instructions" }); + await db.delete(principalPermissionGrants).where(eq(principalPermissionGrants.companyId, companyId)); + await db.update(companyMemberships).set({ membershipRole: "member" }).where(eq(companyMemberships.companyId, companyId)); + await db.insert(principalPermissionGrants).values([ + { companyId, principalType: "agent", principalId: agentId, permissionKey: "agents:suggest-changes" }, + { companyId, principalType: "user", principalId: userId, permissionKey: "agents:suggest-changes" }, + ]); + await db.insert(issueThreadInteractions).values({ id: interactionId, companyId, issueId, kind: "request_confirmation", status: "accepted", sourceRunId, createdByAgentId: agentId, + payload: { version: 1, prompt: "Apply the file change?", detailsMarkdown: "```diff\n+approved\n```", target: { type: "custom", key: `agent:${agentId}:instructions`, revisionId: "proposal" } }, + result: { version: 1, outcome: "accepted" }, resolvedByUserId: userId, resolvedAt: new Date(), + }); + const confirmation = async () => (await db.select().from(issueThreadInteractions).where(eq(issueThreadInteractions.id, interactionId)))[0]!.result; + // A turn that changes no file keeps the confirmation for the change it approves. + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))).toMatchObject({ state: "warm_saved", errorCode: null }); + expect(await confirmation()).not.toHaveProperty("consumedByRunId"); + objects.uploads.length = 0; + await fs.writeFile(path.join(copy.localRoot, "approved.md"), "approved change"); + const saved = (await copies.checkpointWarm({ companyId, runId: copy.runId }))!; + expect(saved).toMatchObject({ state: "warm_saved", errorCode: null }); + expect(await confirmation()).toMatchObject({ consumedByRunId: copy.runId }); + expect((await head()).entries.find(entry => entry.path === "approved.md")).toMatchObject({ sha256: fileHash(Buffer.from("approved change")) }); + expect(objects.uploads).toHaveLength(1); + // The confirmation is spent: the next change is refused before it uploads. + await fs.writeFile(path.join(copy.localRoot, "unapproved.md"), "second change"); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.errorCode).toBe("AGENT_FILES_CHECKPOINT_UNSTABLE"); + expect(objects.uploads).toHaveLength(1); + }); + + it("commits both runs when two warm checkpoints of one agent race", async () => { + const first = await warmRun(); + const second = await warmRun(); + objects.uploads.length = 0; + objects.probe = agentRowLocked; + objects.delayMs = 100; + await fs.writeFile(path.join(first.localRoot, "first.md"), "from the first run"); + await fs.writeFile(path.join(second.localRoot, "second.md"), "from the second run"); + const [a, b] = await Promise.all([ + copies.checkpointWarm({ companyId, runId: first.runId }), + copies.checkpointWarm({ companyId, runId: second.runId }), + ]); + expect([a?.state, b?.state]).toEqual(["warm_saved", "warm_saved"]); + expect(objects.uploads.map(upload => upload.agentRowLocked)).toEqual([false, false]); + const paths = (await head()).entries.map(entry => entry.path); + expect(paths).toEqual(expect.arrayContaining(["first.md", "second.md", entryFile])); + expect(await fs.readFile(path.join(root, "first.md"), "utf8")).toBe("from the first run"); + expect(await fs.readFile(path.join(root, "second.md"), "utf8")).toBe("from the second run"); + await expectHeadRestorable(); + }); + + it("lets an instruction edit commit while a checkpoint uploads, then keeps both changes", async () => { + const copy = await warmRun(); + objects.uploads.length = 0; + let open!: () => void; + objects.gate = new Promise(resolve => { open = resolve; }); + const uploading = new Promise(resolve => { objects.started = resolve; }); + await fs.writeFile(path.join(copy.localRoot, "memory.md"), "from the run"); + const checkpoint = copies.checkpointWarm({ companyId, runId: copy.runId }); + await uploading; + // The checkpoint waits inside its upload. The editor must not wait for it. + objects.started = null; + const editorGate = objects.gate; + objects.gate = null; + const edited = await Promise.race([ + agentFileStore(db).write({ companyId, agentId, path: entryFile, bytes: Buffer.from("# Edited policy\n"), baseHash: fileHash(Buffer.from("# Policy\n")) }, board()), + new Promise((_, reject) => setTimeout(() => reject(new Error("editor waited for the checkpoint upload")), 10_000)), + ]); + expect(edited.changed).toBe(true); + open(); + await editorGate; + expect((await checkpoint)?.state).toBe("warm_saved"); + expect(await fs.readFile(path.join(root, entryFile), "utf8")).toBe("# Edited policy\n"); + expect(await fs.readFile(path.join(root, "memory.md"), "utf8")).toBe("from the run"); + const entries = new Map((await head()).entries.map(entry => [entry.path, entry])); + expect(entries.get(entryFile)).toMatchObject({ sha256: fileHash(Buffer.from("# Edited policy\n")) }); + expect(entries.get("memory.md")).toMatchObject({ sha256: fileHash(Buffer.from("from the run")) }); + await expectHeadRestorable(); + }); + + it("keeps the committed files when the save stops between upload and commit", async () => { + const copy = await warmRun(); + const before = await head(); + objects.uploads.length = 0; + let open!: () => void; + objects.gate = new Promise(resolve => { open = resolve; }); + await fs.writeFile(path.join(copy.localRoot, "draft.md"), "uploaded but not committed"); + const uploading = new Promise(resolve => { objects.started = resolve; }); + const checkpoint = copies.checkpointWarm({ companyId, runId: copy.runId }); + await Promise.race([uploading, checkpoint]); + objects.started = null; + // Change the instruction entry while the bytes upload. The commit then refuses the save. + await db.update(agents).set({ adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root, instructionsEntryFile: "OTHER.md" } }) + .where(eq(agents.id, agentId)); + objects.gate = null; + open(); + const failed = (await checkpoint)!; + expect(failed.errorCode).toBe("AGENT_FILES_CHECKPOINT_UNSTABLE"); + expect(objects.uploads.length).toBeGreaterThan(0); + const after = await head(); + expect(after.generation).toBe(before.generation); + expect(after.entries.some(entry => entry.path === "draft.md")).toBe(false); + await expectHeadRestorable(); + // The next save after the entry is restored commits the same change. + await db.update(agents).set({ adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root, instructionsEntryFile: entryFile } }) + .where(eq(agents.id, agentId)); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + expect((await head()).entries.find(entry => entry.path === "draft.md")).toMatchObject({ sha256: fileHash(Buffer.from("uploaded but not committed")) }); + await expectHeadRestorable(); + }); + + // Opt in: PAPERCLIP_AGENT_FILE_CHECKPOINT_BENCHMARK=1 pnpm exec vitest run src/__tests__/agent-file-checkpoint-locking.test.ts -t measures + // Wall time shares the host's load; statement counts and lock observations are exact. + it.skipIf(process.env.PAPERCLIP_AGENT_FILE_CHECKPOINT_BENCHMARK !== "1")("measures checkpoint statements and agent row lock time", async () => { + const copy = await warmRun(); + for (let file = 0; file < 50; file += 1) await fs.writeFile(path.join(copy.localRoot, `file-${file}.md`), `content ${file}`); + expect((await copies.checkpointWarm({ companyId, runId: copy.runId }))?.state).toBe("warm_saved"); + const unchanged: Array<{ ms: number; statements: number; rowLocks: number }> = []; + for (let attempt = 0; attempt < 10; attempt += 1) { + const started = performance.now(); + const { statements } = await recordStatements(() => copies.checkpointWarm({ companyId, runId: copy.runId })); + unchanged.push({ ms: performance.now() - started, statements: statements.length, + rowLocks: statements.filter(statement => /for (update|no key update|share)/i.test(statement)).length }); + } + // One changed file per save, with 250 ms object-store latency. Observe the agent row lock from another connection. + objects.delayMs = 250; + const changed: Array<{ ms: number; statements: number; lockedMs: number }> = []; + for (let attempt = 0; attempt < 5; attempt += 1) { + await fs.writeFile(path.join(copy.localRoot, "changed.md"), `changed ${attempt}`); + let polling = true, firstLocked = 0, lastLocked = 0; + const poller = (async () => { + while (polling) { + if (await agentRowLocked()) { const now = performance.now(); firstLocked ||= now; lastLocked = now; } + await new Promise(resolve => setTimeout(resolve, 5)); + } + })(); + const started = performance.now(); + const { statements } = await recordStatements(() => copies.checkpointWarm({ companyId, runId: copy.runId })); + const ms = performance.now() - started; + polling = false; + await poller; + changed.push({ ms, statements: statements.length, lockedMs: firstLocked ? lastLocked - firstLocked : 0 }); + } + const median = (values: number[]) => Math.round([...values].sort((a, b) => a - b)[Math.floor(values.length / 2)]!); + console.info(JSON.stringify({ + unchanged: { medianMs: median(unchanged.map(value => value.ms)), statements: unchanged[0]!.statements, rowLockStatements: unchanged[0]!.rowLocks }, + changed: { medianMs: median(changed.map(value => value.ms)), statements: changed[0]!.statements, + medianAgentRowLockObservedMs: median(changed.map(value => value.lockedMs)), lockObservedMs: changed.map(value => Math.round(value.lockedMs)) }, + })); + }, 120_000); +}); diff --git a/server/src/services/agent-directory-working-copies.ts b/server/src/services/agent-directory-working-copies.ts index 98ec9ab4ec8..5582092c74a 100644 --- a/server/src/services/agent-directory-working-copies.ts +++ b/server/src/services/agent-directory-working-copies.ts @@ -423,9 +423,11 @@ export function agentDirectoryWorkingCopyService(db: Db, get: (companyId: string const snapshot = checkpointSnapshot(captured.manifest); const nextHash = directorySnapshotSha256(snapshot); const changed = nextHash !== row.baseHash; + // An unchanged run writes no agent file. Its save skips the row lock + // when the durable head is already current for the local projection. const { storageWarning } = changed ? await store.apply({ companyId: row.companyId, agentId: row.agentId, sourceDir: path.join(captured.directory, "files"), baseline: baseline(row), checkpoint: captured.manifest }, actor(row)) - : await store.locked(row.companyId, row.agentId, actor(row), true, async () => ({ + : await store.saveUnchanged(row.companyId, row.agentId, actor(row)).then(() => ({ storageWarning: typeof row.receipt?.storageWarning === "string" ? row.receipt.storageWarning : null, })); row = await patch(row, { state: stopped ? (changed ? "saved" : "unchanged") : "warm_saved", baseHash: nextHash, diff --git a/server/src/services/agent-file-durability.ts b/server/src/services/agent-file-durability.ts index 1fbb927fd9d..058175f6c19 100644 --- a/server/src/services/agent-file-durability.ts +++ b/server/src/services/agent-file-durability.ts @@ -15,7 +15,7 @@ import { AgentFileLimitError, MAX_AGENT_DIRECTORY_BYTES, MAX_AGENT_DIRECTORY_ENT type Tx = Parameters[0]>[0]; type Owner = { companyId: string; id: string; createdAt: Date }; type Head = typeof agentFileRoots.$inferSelect; -const where = (owner: Owner) => and(eq(agentFileRoots.companyId, owner.companyId), eq(agentFileRoots.agentId, owner.id)); +const where = (owner: Pick) => and(eq(agentFileRoots.companyId, owner.companyId), eq(agentFileRoots.agentId, owner.id)); const marker = (root: string) => path.join(path.dirname(root), "file-sync", "durable-generation"); const adoptionReceipt = (root: string) => path.join(path.dirname(root), "file-sync", "durable-adoption.json"); export const durableAgentFilesEnabled = () => process.env.PAPERCLIP_DURABLE_AGENT_FILES === "true"; @@ -51,6 +51,28 @@ async function atomicPrivateWrite(filename: string, contents: string) { } finally { await fs.rm(temporary, { force: true }); } } +/** + * True when the locked file-tree step of a save would change nothing for an + * unchanged run: the durable head exists and the local projection is marked at + * its generation (so restore is a no-op and persist only re-marks it), or + * durable files are off, no head exists and the one-time local adoption is + * already recorded (so persist returns at once). A false result asks the caller + * to take the locked path, which adopts, restores or heals the head. + */ +export async function agentFileProjectionCurrent(db: Db, owner: { companyId: string; id: string }, root: string): Promise { + const [head] = await db.select({ generation: agentFileRoots.generation }).from(agentFileRoots).where(where(owner)); + const exists = (filename: string) => fs.stat(filename).then(() => true, (error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return false; + throw error; + }); + if (!head) return !durableAgentFilesEnabled() && await exists(path.join(path.dirname(root), "file-sync", "adopted.json")); + const generation = await fs.readFile(marker(root), "utf8").catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + return generation === head.generation && await fs.stat(root).then(info => info.isDirectory(), () => false); +} + /** Under the agent row lock. A durable head remains authoritative even after * the opt-in is removed; silently falling back would lose acknowledged writes. */ export async function restoreDurableAgentFiles(tx: Tx, owner: Owner, root: string): Promise { @@ -136,9 +158,69 @@ export async function restoreDurableAgentFiles(tx: Tx, owner: Owner, root: strin } finally { await fs.rm(temporary, { recursive: true, force: true }); } } +/** Objects uploaded before the agent row lock, by content (`sha256:size`). */ +export type PreparedAgentFileObjects = ReadonlyMap; +const contentKey = (sha256: string, size: number) => `${sha256}:${size}`; + +/** Stream one file to object storage. The upload fails unless the bytes match + * the expected size and digest, so a stored object always has that content. */ +async function uploadAgentFile(service: ReturnType, owner: { companyId: string; id: string }, + filename: string, size: number, sha256: string, signal: AbortSignal): Promise { + const handle = await fs.open(filename, constants.O_RDONLY | constants.O_NOFOLLOW); + const source = handle.createReadStream(); + const hash = createHash("sha256"); + let read = 0; + const body = new Transform({ transform(chunk: Buffer, _encoding, callback) { + read += chunk.length; + if (read > size) return callback(new Error("Agent file changed during upload")); + hash.update(chunk); callback(null, chunk); + }, flush(callback) { + callback(read === size && hash.digest("hex") === sha256 ? null : new Error("Agent file changed during upload")); + } }); + try { + const [result] = await Promise.all([service.putFile({ companyId: owner.companyId, namespace: `agent-files/${owner.id}`, + originalFilename: null, contentType: "application/octet-stream", body, + byteSize: size, sha256, signal }), pipeline(source, body, { signal })]); + return result.objectKey; + } finally { source.destroy(); body.destroy(); await handle.close(); } +} + +/** + * Upload the files a run changed before its save takes the agent row lock, so + * the locked commit only records object keys. Objects are immutable and + * verified by digest; the commit uses one only for a file with the same digest + * and size. An object that no commit uses (a refused save, or a crash before + * the commit) is never authoritative and is removed by the object collector + * after its grace period. This step is an optimization: a file it could not + * upload is uploaded by the locked commit as before. + */ +export async function uploadAgentFilesBeforeCommit(db: Db, owner: { companyId: string; id: string }, sourceDir: string, + files: ReadonlyArray<{ path: string; size: number; sha256: string }>): Promise { + const prepared = new Map(); + if (files.length === 0) return prepared; + try { + const [head] = await db.select({ entries: agentFileRoots.entries }).from(agentFileRoots).where(where(owner)); + if (!head && !durableAgentFilesEnabled()) return prepared; + const service = storage(); + const committed = new Map(head?.entries.map(entry => [entry.path, entry])); + const signal = AbortSignal.timeout(30_000); + for (const file of files) { + const key = contentKey(file.sha256, file.size); + const old = committed.get(file.path); + if (file.size === 0 || prepared.has(key) || (old?.kind === "file" && old.sha256 === file.sha256 && old.size === file.size)) continue; + const filename = await assertInstructionPathSafe(sourceDir, instructionPath(file.path)); + prepared.set(key, await uploadAgentFile(service, owner, filename, file.size, file.sha256, signal)); + } + } catch { + // The locked commit uploads whatever is missing and reports its own errors. + } + return prepared; +} + /** Publish immutable bytes before the database head. A failed/uncertain database * commit never removes uploaded bytes that may have become authoritative. */ -export async function persistDurableAgentFiles(tx: Tx, owner: Owner, root: string, importingLegacy = false) { +export async function persistDurableAgentFiles(tx: Tx, owner: Owner, root: string, importingLegacy = false, + prepared: PreparedAgentFileObjects = new Map()) { const [previous] = await tx.select().from(agentFileRoots).where(where(owner)); if (!previous && !durableAgentFilesEnabled()) return; const service = storage(); @@ -161,26 +243,10 @@ export async function persistDurableAgentFiles(tx: Tx, owner: Owner, root: strin if (entry.kind !== "file") throw new Error("Agent files cannot contain symbolic links"); if (entry.size === undefined || entry.mode === undefined || !entry.hash) throw new Error("Agent file metadata is incomplete"); const old = before.get(relative); - let objectKey = old?.kind === "file" && old.sha256 === entry.hash && old.size === entry.size ? old.objectKey : null; + let objectKey = old?.kind === "file" && old.sha256 === entry.hash && old.size === entry.size ? old.objectKey + : prepared.get(contentKey(entry.hash, entry.size)) ?? null; if (!objectKey && entry.size !== 0) { - const filename = await assertInstructionPathSafe(root, relative); - const handle = await fs.open(filename, constants.O_RDONLY | constants.O_NOFOLLOW); - const source = handle.createReadStream(); - const hash = createHash("sha256"); - let size = 0; - const body = new Transform({ transform(chunk: Buffer, _encoding, callback) { - size += chunk.length; - if (size > entry.size!) return callback(new Error("Agent file changed during upload")); - hash.update(chunk); callback(null, chunk); - }, flush(callback) { - callback(size === entry.size && hash.digest("hex") === entry.hash ? null : new Error("Agent file changed during upload")); - } }); - try { - const [result] = await Promise.all([service.putFile({ companyId: owner.companyId, namespace: `agent-files/${owner.id}`, - originalFilename: null, contentType: "application/octet-stream", body, - byteSize: entry.size, sha256: entry.hash, signal }), pipeline(source, body, { signal })]); - objectKey = result.objectKey; - } finally { source.destroy(); body.destroy(); await handle.close(); } + objectKey = await uploadAgentFile(service, owner, await assertInstructionPathSafe(root, relative), entry.size, entry.hash, signal); } entries.push({ path: relative, kind: "file", mode: entry.mode, size: entry.size, sha256: entry.hash, objectKey }); } diff --git a/server/src/services/agent-file-store.ts b/server/src/services/agent-file-store.ts index 0d5c3672040..a5813d54317 100644 --- a/server/src/services/agent-file-store.ts +++ b/server/src/services/agent-file-store.ts @@ -16,7 +16,7 @@ import { authorizeInstructionCommit, authorizeInstructionRead } from "./agent-in import { assertInstructionPathSafe, instructionPath, instructionBytes, materializeInstructionBytes, readInstructionBytes, MAX_INSTRUCTION_BYTES } from "./agent-instruction-files.js"; import { agentInstructionsBundleMode, deriveBundleState, resolveManagedInstructionsRoot } from "./agent-instructions.js"; import type { AuthorizationActor } from "./authorization.js"; -import { durableAgentFilesEnabled, invalidateAgentFileProjection, persistDurableAgentFiles, restoreDurableAgentFiles } from "./agent-file-durability.js"; +import { agentFileProjectionCurrent, durableAgentFilesEnabled, invalidateAgentFileProjection, persistDurableAgentFiles, restoreDurableAgentFiles, uploadAgentFilesBeforeCommit, type PreparedAgentFileObjects } from "./agent-file-durability.js"; type Tx = Parameters[0]>[0]; type Agent = typeof agents.$inferSelect; @@ -193,7 +193,8 @@ export async function isAgentFileSaveRecorded(tx: Tx, agent: Agent, relative: st export function agentFileStore(db: Db) { db = fileStorageDb(db); async function locked(companyId: string, agentId: string, actor: AuthorizationActor, write: boolean, - fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor, onRollback: (restore: () => Promise) => void) => Promise, texts: string[] = []) { + fn: (tx: Tx, agent: Agent, root: string, bound: AuthorizationActor, onRollback: (restore: () => Promise) => void) => Promise, texts: string[] = [], + prepared?: PreparedAgentFileObjects) { return db.transaction(async tx => { if (write) await assertAgentInteraction(tx, { companyId, actorAgentId: actor.type === "agent" ? actor.agentId : null, operation: "configure", targetAgentId: agentId, lockAgentIds: texts.flatMap(extractAgentMentionIds), lock: "update" }); @@ -205,7 +206,7 @@ export function agentFileStore(db: Db) { let restore: (() => Promise) | undefined; try { const result = await fn(tx, agent, root, bound, callback => { restore = callback; }); - if (write) await persistDurableAgentFiles(tx, agent, root); + if (write) await persistDurableAgentFiles(tx, agent, root, false, prepared); return result; } catch (error) { await restore?.(); @@ -220,8 +221,42 @@ export function agentFileStore(db: Db) { runId: origin ? origin.runId : actor.runId, responsibleUserId: origin ? null : actor.type === "board" ? actor.userId : actor.onBehalfOfUserId, action: "agent.files_updated", entityType: "agent", entityId: agent.id, details }); } + /** The write authorization of `locked`, without the row lock or the file + * tree, in a transaction that always rolls back. It writes nothing: an + * accepted change confirmation stays unconsumed, so only the locked commit + * of a save consumes it. It is enough for a save that changes no file, and + * it stops a save that will be refused before that save uploads any bytes. */ + async function authorizeSave(companyId: string, agentId: string, actor: AuthorizationActor) { + const rollback = Symbol("authorization preview"); + await db.transaction(async tx => { + await assertAgentInteraction(tx, { companyId, actorAgentId: actor.type === "agent" ? actor.agentId : null, + operation: "configure", targetAgentId: agentId }); + const [agent] = await tx.select().from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))); + if (!agent) throw notFound("Agent not found"); + await authorizeInstructionCommit(tx, actor, agent); + throw rollback; + }).catch(error => { if (error !== rollback) throw error; }); + } + /** + * The save of a run that changed no file. When the locked file-tree step + * would change nothing (see `agentFileProjectionCurrent`), only the + * read-only authorization runs, without the row lock. Otherwise the locked + * write runs as for any save, so it adopts, restores or heals the durable + * head exactly as before. + */ + async function saveUnchanged(companyId: string, agentId: string, actor: AuthorizationActor) { + const [agent] = await db.select().from(agents).where(and(eq(agents.companyId, companyId), eq(agents.id, agentId))); + if (agent && agentInstructionsBundleMode(agent) !== "external" + && await agentFileProjectionCurrent(db, agent, resolveManagedInstructionsRoot(agent))) { + await authorizeSave(companyId, agentId, actor); + return; + } + await locked(companyId, agentId, actor, true, async () => undefined); + } return { locked, + authorizeSave, + saveUnchanged, download: async (companyId: string, agentId: string, relative: string, actor: AuthorizationActor) => { const opened: { file: Awaited> } = { file: null }; try { @@ -324,6 +359,13 @@ export function agentFileStore(db: Db) { if (mentions.length) changedMentions.set(name, { hash: fileHash(bytes!), text: mentions.map(id => `[@agent](agent://${id})`).join(" ") }); } } + // Upload the run's changed bytes before the row lock, after a preview of + // the same write authorization. The commit below runs that authorization, + // validates the tree and records only objects with matching digests. + if (input.checkpoint) await authorizeSave(input.companyId, input.agentId, actor); + const prepared = input.checkpoint ? await uploadAgentFilesBeforeCommit(db, target, input.sourceDir, input.checkpoint.entries + .filter(([name, entry]) => entry.kind === "file" && JSON.stringify(incoming.entries.get(name)) !== JSON.stringify(input.baseline.entries.get(name))) + .map(([name, entry]) => ({ path: name, size: entry.size!, sha256: entry.hash! }))) : undefined; return locked(input.companyId, input.agentId, actor, true, async (tx, agent, root, bound) => { if (deriveBundleState(agent).entryFile !== requestedEntry) throw conflict("Agent instructions entry changed; retry the file synchronization"); const entryPath = deriveBundleState(agent).entryFile; @@ -392,7 +434,7 @@ export function agentFileStore(db: Db) { ...(input.checkpoint ? { snapshots: { source: incoming, current } } : {}) }); await audit(tx, agent, bound, { sourceRunId: currentPluginInteractionOrigin()?.runId ?? actor.runId, contract: AGENT_FILES_CONTRACT }); return { storageWarning: storageWarning(total, finalEntries.size, fullFile) }; - }, [...changedMentions.values()].map(value => value.text)); + }, [...changedMentions.values()].map(value => value.text), prepared); }, }; }