Files
PaperClipAI/server/src/__tests__/agent-file-durability.test.ts
T
TonioandPaperclip 9dfdeaad86 Refine agent workspaces and organization charts (#20973)
Refine agent details, identity editing, responsive controls, and organization charts with three card tiers, clearer save feedback, and stable filtering.

Guard instruction-file renames and root configuration updates in one transaction through the agent lifecycle integration, preserving conflict checks, rollback, and configuration history.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
2026-11-12 01:23:59 +00:00

343 lines
23 KiB
TypeScript

import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { randomUUID } from "node:crypto";
import { Readable } from "node:stream";
import { setTimeout as delay } from "node:timers/promises";
import { beforeAll, beforeEach, afterEach, afterAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { agents, agentConfigRevisions, agentFileRoots, agentInstructionWorkingCopies, authUsers, companies, companyMemberships, principalPermissionGrants, heartbeatRuns, createDb, closeFileStorageDb } from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { agentFileStore, adoptAgentFiles, fileHash } from "../services/agent-file-store.js";
import { invalidateAgentFileProjection, persistDurableAgentFiles } from "../services/agent-file-durability.js";
import { materializeInstructionBytes } from "../services/agent-instruction-files.js";
import { agentInstructionsService, resolveManagedInstructionsRoot } from "../services/agent-instructions.js";
import { AGENT_FILE_OBJECT_GRACE_MS, collectAgentFileObjects, migrateAgentFiles } from "../services/agent-file-maintenance.js";
import { getStorageService } from "../storage/index.js";
const objects = vi.hoisted(() => ({ bytes: new Map<string, Buffer>(), rejectWrites: false, corruptReads: false, readDelayMs: 0, activeReads: 0, peakReads: 0 }));
vi.mock("../storage/index.js", () => ({ getStorageService: () => ({
provider: "s3",
async putFile(input: { companyId: string; namespace: string; body: Readable; byteSize: number; sha256: string }) {
if (objects.rejectWrites) throw new Error("fixture object-store outage");
const chunks: Buffer[] = [];
for await (const chunk of input.body) chunks.push(Buffer.from(chunk));
const body = Buffer.concat(chunks);
expect(body.length).toBe(input.byteSize);
expect(fileHash(body)).toBe(input.sha256);
const objectKey = `${input.companyId}/${input.namespace}/${randomUUID()}`;
objects.bytes.set(objectKey, body);
return { objectKey };
},
async getObject(companyId: string, objectKey: string, options: { signal?: AbortSignal } = {}) {
expect(objectKey.startsWith(`${companyId}/`)).toBe(true);
const body = objects.bytes.get(objectKey);
if (!body) throw new Error("fixture object missing");
return { stream: Readable.from((async function* () {
objects.activeReads++;
objects.peakReads = Math.max(objects.peakReads, objects.activeReads);
try {
if (objects.readDelayMs) await delay(objects.readDelayMs, undefined, { signal: options.signal });
yield objects.corruptReads ? Buffer.from("wrong bytes") : body;
} finally { objects.activeReads--; }
})()) };
},
async headObject(_companyId: string, key: string) { return { exists: objects.bytes.has(key), lastModified: new Date(0) }; },
async deleteObject(companyId: string, key: string, options: { allVersions: boolean }) {
expect(key.startsWith(`${companyId}/`)).toBe(true);
expect(options.allVersions).toBe(true);
objects.bytes.delete(key);
},
}) }));
describe("durable agent file authority", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let home: string, root: string, companyId: string, agentId: string, userId: string;
const actor = () => ({ type: "board" as const, userId, source: "session" as const });
const store = () => agentFileStore(db);
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("durable-agent-files-");
db = createDb(database.connectionString);
}, 90_000);
afterAll(async () => {
if (db) { await closeFileStorageDb(db, 1); await db.$client.end({ timeout: 1 }); }
await database?.cleanup();
});
beforeEach(async () => {
home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "durable-agent-files-")));
vi.stubEnv("PAPERCLIP_HOME", home); vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "true");
objects.bytes.clear(); objects.rejectWrites = false; objects.corruptReads = false;
objects.readDelayMs = 0; objects.activeReads = 0; objects.peakReads = 0;
companyId = randomUUID(); agentId = randomUUID(); userId = randomUUID();
await db.insert(companies).values({ id: companyId, name: "File authority", issuePrefix: randomUUID().slice(0, 8) });
await db.insert(authUsers).values({ id: userId, name: "Editor", email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
await db.insert(agents).values({ id: agentId, companyId, name: "File owner" });
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, membershipRole: "operator" });
await db.insert(principalPermissionGrants).values({ companyId, principalType: "user", principalId: userId,
permissionKey: "agents:configure", scope: { agentIds: [agentId] } });
root = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "File owner", adapterConfig: {} });
});
afterEach(async () => {
vi.unstubAllEnvs();
await fs.rm(home, { recursive: true, force: true });
});
async function replaceFilesystem() {
await fs.rm(home, { recursive: true, force: true });
home = await fs.realpath(await fs.mkdtemp(path.join(os.tmpdir(), "durable-agent-replacement-")));
vi.stubEnv("PAPERCLIP_HOME", home);
root = resolveManagedInstructionsRoot({ id: agentId, companyId, name: "File owner", adapterConfig: {} });
}
it("renames the active entry and edited content in the same committed file snapshot", async () => {
await store().write({ companyId, agentId, path: "AGENTS.md", bytes: Buffer.from("Original"), baseHash: null }, actor());
await store().rename({ companyId, agentId, path: "AGENTS.md", renameTo: "ROOT.md", bytes: Buffer.from("Edited"), baseHash: fileHash(Buffer.from("Original")) }, actor());
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
expect(agent.adapterConfig.instructionsEntryFile).toBe("ROOT.md");
const [revision] = await db.select().from(agentConfigRevisions).where(eq(agentConfigRevisions.agentId, agentId));
expect(revision).toMatchObject({ source: "instructions_rename", createdByUserId: userId,
afterConfig: { adapterConfig: { instructionsEntryFile: "ROOT.md" } } });
await replaceFilesystem();
expect(await store().read(companyId, agentId, "ROOT.md", actor())).toEqual(Buffer.from("Edited"));
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toBeNull();
});
it("creates a renamed missing root without replacing a concurrently created source", async () => {
await store().rename({ companyId, agentId, path: "AGENTS.md", renameTo: "ROOT.md", bytes: Buffer.from("New root"), baseHash: null }, actor());
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
expect(agent.adapterConfig.instructionsEntryFile).toBe("ROOT.md");
expect(await store().read(companyId, agentId, "ROOT.md", actor())).toEqual(Buffer.from("New root"));
await expect(store().rename({ companyId, agentId, path: "ROOT.md", renameTo: "OTHER.md", bytes: Buffer.from("Stale draft"), baseHash: null }, actor())).rejects.toMatchObject({ status: 409 });
expect(await store().read(companyId, agentId, "OTHER.md", actor())).toBeNull();
});
it("rejects a rename that waits behind a concurrent source edit", async () => {
await store().write({ companyId, agentId, path: "AGENTS.md", bytes: Buffer.from("Original"), baseHash: null }, actor());
let enter!: () => void, release!: () => void;
const entered = new Promise<void>(resolve => { enter = resolve; });
const released = new Promise<void>(resolve => { release = resolve; });
const writer = store().locked(companyId, agentId, actor(), true, async (_tx, _agent, folder) => {
enter();
await released;
await materializeInstructionBytes(folder, "AGENTS.md", Buffer.from("Concurrent edit"));
});
await entered;
const rename = store().rename({ companyId, agentId, path: "AGENTS.md", renameTo: "ROOT.md", bytes: Buffer.from("Original"), baseHash: fileHash(Buffer.from("Original")) }, actor());
const rejected = expect(rename).rejects.toMatchObject({ status: 409 });
release();
await writer;
await rejected;
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
expect(agent.adapterConfig.instructionsEntryFile).toBeUndefined();
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Concurrent edit"));
expect(await store().read(companyId, agentId, "ROOT.md", actor())).toBeNull();
});
it("preserves the active root and source when a rename cannot persist its content", async () => {
await store().write({ companyId, agentId, path: "AGENTS.md", bytes: Buffer.from("Original"), baseHash: null }, actor());
objects.rejectWrites = true;
await expect(store().rename({ companyId, agentId, path: "AGENTS.md", renameTo: "ROOT.md", bytes: Buffer.from("Edited"), baseHash: fileHash(Buffer.from("Original")) }, actor())).rejects.toThrow("fixture object-store outage");
objects.rejectWrites = false;
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
expect(agent.adapterConfig.instructionsEntryFile).toBeUndefined();
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Original"));
expect(await store().read(companyId, agentId, "ROOT.md", actor())).toBeNull();
});
it("reads instruction compatibility from the committed head after deleting the entire home", async () => {
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Original"));
await db.update(agents).set({ adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root,
instructionsFilePath: path.join(root, "AGENTS.md"), instructionsEntryFile: "AGENTS.md" } }).where(eq(agents.id, agentId));
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
const instructions = agentInstructionsService(db);
expect(await instructions.getManagedEntryFingerprint(agent)).toBeNull();
await store().read(companyId, agentId, "AGENTS.md", actor());
const expected = { entryFile: "AGENTS.md", contentHash: `sha256:${fileHash(Buffer.from("Original"))}` };
expect(await instructions.getManagedEntryFingerprint(agent)).toEqual(expected);
await replaceFilesystem();
expect(await instructions.getManagedEntryFingerprint(agent)).toEqual(expected);
await expect(fs.stat(root)).rejects.toMatchObject({ code: "ENOENT" });
expect(await instructions.getManagedEntryFingerprint({ ...agent, companyId: randomUUID() })).toBeNull();
expect(await instructions.getManagedEntryFingerprint({ ...agent, adapterConfig: {
...agent.adapterConfig, instructionsBundleMode: "external",
} })).toBeNull();
expect(await instructions.getManagedEntryFingerprint({ ...agent, adapterConfig: {
...agent.adapterConfig, instructionsEntryFile: "missing.md",
} })).toEqual({ entryFile: "missing.md", contentHash: null });
});
it("keeps recorded file dates through restore without treating projection times as edits", async () => {
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Historical instructions"));
await db.update(agents).set({ adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root,
instructionsEntryFile: "AGENTS.md" } }).where(eq(agents.id, agentId));
const [agent] = await db.select().from(agents).where(eq(agents.id, agentId));
await store().read(companyId, agentId, "AGENTS.md", actor());
await store().write({ companyId, agentId, path: "NOTES.md", bytes: Buffer.from("Recorded creation"), baseHash: null }, actor());
const instructions = agentInstructionsService(db);
const created = await instructions.readFile(agent, "NOTES.md");
expect(Number.isFinite(Date.parse(created.createdAt!))).toBe(true);
await replaceFilesystem();
vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "false");
const restored = await instructions.readFile(agent, "AGENTS.md");
expect(restored.content).toBe("Historical instructions");
expect(restored.createdAt).toBeUndefined();
expect(restored.updatedAt).toBeUndefined();
const summary = (await instructions.getBundle(agent)).files.find(file => file.path === "AGENTS.md");
expect(summary).toBeDefined();
expect(summary?.createdAt).toBeUndefined();
expect(summary?.updatedAt).toBeUndefined();
const recorded = await instructions.readFile(agent, "NOTES.md");
expect(recorded.createdAt).toBe(created.createdAt);
expect(recorded.updatedAt).toBeUndefined();
});
it("imports legacy bytes, modes and empty entries, then restores without any old filesystem", async () => {
const binary = Buffer.alloc(2 * 1024 * 1024, 0x81);
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Original"));
await materializeInstructionBytes(root, "nested/data.bin", binary);
await materializeInstructionBytes(root, "empty.txt", Buffer.alloc(0));
await materializeInstructionBytes(root, "script.sh", Buffer.from("#!/bin/sh\n"));
await fs.chmod(path.join(root, "script.sh"), 0o755);
await fs.mkdir(path.join(root, "empty-folder"));
await db.update(agents).set({ adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root } }).where(eq(agents.id, agentId));
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Original"));
await replaceFilesystem();
vi.stubEnv("PAPERCLIP_DURABLE_AGENT_FILES", "false");
expect(await store().read(companyId, agentId, "nested/data.bin", actor())).toEqual(binary);
expect(await store().read(companyId, agentId, "empty.txt", actor())).toEqual(Buffer.alloc(0));
expect((await fs.stat(path.join(root, "script.sh"))).mode & 0o777).toBe(0o755);
expect((await fs.stat(path.join(root, "empty-folder"))).isDirectory()).toBe(true);
});
it("restores a cold folder with bounded parallel streams and settles all failed work before cleanup", async () => {
const contents = Buffer.alloc(128 * 1024, 0x81);
for (let index = 0; index < 40; index++) {
await materializeInstructionBytes(root, `nested/file-${index}.bin`, contents);
}
expect(await migrateAgentFiles(db, { companyId, apply: true, writersStopped: true })).toMatchObject({ pageReady: true });
await replaceFilesystem();
objects.readDelayMs = 25;
expect((await store().read(companyId, agentId, "nested/file-39.bin", actor()))?.equals(contents)).toBe(true);
expect(objects.peakReads).toBeGreaterThan(1);
expect(objects.peakReads).toBeLessThanOrEqual(8);
expect(objects.activeReads).toBe(0);
for (let index = 0; index < 40; index++) {
expect((await fs.readFile(path.join(root, `nested/file-${index}.bin`))).equals(contents)).toBe(true);
}
// A corrupt replacement must preserve the old projection and await every
// cancelled sibling stream before deleting its temporary directory.
await invalidateAgentFileProjection(root);
objects.corruptReads = true;
await expect(store().read(companyId, agentId, "nested/file-39.bin", actor())).rejects.toThrow(/committed (size|digest)/);
expect(objects.activeReads).toBe(0);
expect((await fs.readdir(path.dirname(root))).filter(name => name.startsWith(".agent-files-"))).toEqual([]);
expect((await fs.readFile(path.join(root, "nested/file-39.bin"))).equals(contents)).toBe(true);
objects.corruptReads = false;
expect((await store().read(companyId, agentId, "nested/file-39.bin", actor()))?.equals(contents)).toBe(true);
});
it("does not promote a rejected first save through subsequent local adoption", async () => {
objects.rejectWrites = true;
await expect(store().write({ companyId, agentId, path: "failed.txt", bytes: Buffer.from("uncommitted"), baseHash: null }, actor()))
.rejects.toThrow("fixture object-store outage");
expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId))).toEqual([]);
objects.rejectWrites = false;
expect(await store().read(companyId, agentId, "failed.txt", actor())).toBeNull();
await replaceFilesystem();
expect(await store().read(companyId, agentId, "failed.txt", actor())).toBeNull();
});
it("dry-runs without adopting files and publishes settled legacy files online", async () => {
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Legacy"));
const input = { companyId, apply: false, writersStopped: false };
expect(await migrateAgentFiles(db, input)).toMatchObject({ results: [{ agentId, state: "needs_import" }], pageReady: false });
expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId))).toEqual([]);
expect(await migrateAgentFiles(db, { ...input, apply: true })).toMatchObject({ pageReady: true });
await replaceFilesystem();
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Legacy"));
});
it("recognizes an active agent's existing durable head without restoring its filesystem projection", async () => {
await store().write({ companyId, agentId, path: "AGENTS.md", bytes: Buffer.from("Durable"), baseHash: null }, actor());
const [active] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running", runtimeMode: "native" }).returning();
const before = await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId));
await fs.rm(root, { recursive: true });
expect(await migrateAgentFiles(db, { companyId, apply: true, writersStopped: false }))
.toMatchObject({ results: [{ agentId, state: "durable" }], pageReady: true });
expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId))).toEqual(before);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, active!.id)))[0]).toEqual(active);
await expect(fs.stat(root)).rejects.toMatchObject({ code: "ENOENT" });
});
it.each(["pending_collection", "unavailable"])("preserves a %s working copy and keeps first adoption blocked", async state => {
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Canonical files"));
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "failed", finishedAt: new Date() }).returning();
const [copy] = await db.insert(agentInstructionWorkingCopies).values({ companyId, agentId, runId: run!.id,
responsibleUserId: userId, entryFile: "AGENTS.md", baseHash: "a".repeat(64), localRoot: path.join(home, "retained-copy"),
executionRoot: "/retained/agent-files", location: "local", state,
errorCode: "INSTRUCTION_COLLECTION_UNAVAILABLE", receipt: { schema: "paperclip.agent-files.v1" } }).returning();
expect(await migrateAgentFiles(db, { companyId, apply: true, writersStopped: false }))
.toMatchObject({ results: [{ agentId, state: "working_copy_unsettled" }], pageReady: false });
expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId))).toEqual([]);
expect((await db.select().from(agentInstructionWorkingCopies).where(eq(agentInstructionWorkingCopies.runId, run!.id)))[0]).toEqual(copy);
expect(await fs.readFile(path.join(root, "AGENTS.md"), "utf8")).toBe("Canonical files");
});
it("refuses migration while the agent has queued or running execution", async () => {
const [run] = await db.insert(heartbeatRuns).values({ companyId, agentId, status: "running" }).returning();
for (const status of ["running", "queued"]) {
await db.update(heartbeatRuns).set({ status }).where(eq(heartbeatRuns.id, run!.id));
expect(await migrateAgentFiles(db, { companyId, apply: true, writersStopped: true }))
.toMatchObject({ results: [{ agentId, state: "active_execution" }], pageReady: false });
expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId))).toEqual([]);
}
});
it("retains current and unadopted objects, then collects expired unreferenced and deleted-owner objects", async () => {
const prefix = `${companyId}/agent-files/${agentId}/`;
const abandoned = `${prefix}abandoned`;
objects.bytes.set(abandoned, Buffer.from("uncertain upload"));
const provider = { id: "s3" as const, listObjects: async () => ({ keys: [...objects.bytes.keys()].filter(key => key.startsWith(prefix)) }) };
const input = { companyId, agentId, apply: true, now: AGENT_FILE_OBJECT_GRACE_MS + 1 };
const collect = (options = input) => collectAgentFileObjects(db, provider, getStorageService(), options);
expect(await collect()).toMatchObject({ retained: 1, deleted: 0 });
await store().write({ companyId, agentId, path: "AGENTS.md", bytes: Buffer.from("Current"), baseHash: null }, actor());
expect(await collect({ ...input, now: 1 })).toMatchObject({ retained: 2, deleted: 0 });
expect(await collect({ ...input, apply: false })).toMatchObject({ retained: 1, eligible: 1, deleted: 0 });
expect(objects.bytes.has(abandoned)).toBe(true);
expect(await collect()).toMatchObject({ retained: 1, deleted: 1 });
await replaceFilesystem();
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Current"));
await db.delete(agents).where(eq(agents.id, agentId));
expect(await collect()).toMatchObject({ retained: 0, deleted: 1 });
expect(objects.bytes.size).toBe(0);
});
it("refuses to treat a missing legacy filesystem as an empty durable folder", async () => {
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Not migrated yet"));
await db.update(agents).set({ adapterConfig: { instructionsBundleMode: "managed", instructionsRootPath: root } }).where(eq(agents.id, agentId));
await replaceFilesystem();
await expect(store().read(companyId, agentId, "AGENTS.md", actor()))
.rejects.toMatchObject({ status: 503, details: { code: "AGENT_FILES_MIGRATION_REQUIRED" } });
expect(await db.select().from(agentFileRoots).where(eq(agentFileRoots.agentId, agentId))).toEqual([]);
});
it("restores the committed head after SQL rollback and rejects corrupted object bytes", async () => {
await store().write({ companyId, agentId, path: "AGENTS.md", bytes: Buffer.from("Committed"), baseHash: null }, actor());
await expect(db.transaction(async tx => {
const [owner] = await tx.select().from(agents).where(eq(agents.id, agentId)).for("update");
await adoptAgentFiles(tx, owner!);
await invalidateAgentFileProjection(root);
await materializeInstructionBytes(root, "AGENTS.md", Buffer.from("Rolled back"));
await persistDurableAgentFiles(tx, owner!, root);
throw new Error("fixture transaction rollback");
})).rejects.toThrow("fixture transaction rollback");
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Committed"));
await replaceFilesystem(); objects.corruptReads = true;
await expect(store().read(companyId, agentId, "AGENTS.md", actor())).rejects.toThrow(/committed (size|digest)/);
objects.corruptReads = false;
expect(await store().read(companyId, agentId, "AGENTS.md", actor())).toEqual(Buffer.from("Committed"));
});
});