From 01bdb33e5cf182cae472f9798acc76b837384061 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 11 Oct 2026 09:28:04 -0500 Subject: [PATCH] Preserve empty directories during persistent home seeding Stage directory entries under the existing exact upload token and no-follow directory handles before atomic publication. Preserve adoption of existing homes and bounded file streaming. Co-Authored-By: Paperclip --- .../modules/computers/adapters/boat.test.ts | 19 +++++ .../computers/adapters/remote-program.ts | 11 ++- .../computers/application/service.test.ts | 6 ++ .../modules/computers/application/service.ts | 4 +- .../services/persistent-agent-files.test.ts | 80 +++++++++++++++++++ server/src/services/persistent-agent-files.ts | 8 +- 6 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 server/src/services/persistent-agent-files.test.ts diff --git a/server/src/modules/computers/adapters/boat.test.ts b/server/src/modules/computers/adapters/boat.test.ts index e7f7c088fa1..3146a41e0ea 100644 --- a/server/src/modules/computers/adapters/boat.test.ts +++ b/server/src/modules/computers/adapters/boat.test.ts @@ -1,5 +1,6 @@ import { existsSync, + unlinkSync, readFileSync, readdirSync, statSync, @@ -511,6 +512,24 @@ describe("confined computer files", () => { error: "invalid", }); }); + it("stages empty seed directories with exact upload ownership and no-follow paths", () => { + const f = fixture(); const seedId = randomUUID(); + f.call({ action: "seed-begin", seedId }); + expect(f.call({ action: "seed-directory", seedId, path: "empty/nested" })).toEqual({}); + expect(existsSync(f.root)).toBe(false); + expect(f.call({ action: "seed-directory", seedId: randomUUID(), path: "foreign" })).toEqual({ error: "conflict" }); + expect(f.call({ action: "seed-directory", seedId, path: "../escape" })).toEqual({ error: "invalid" }); + const staging = join(f.temp, `.paperclip-seed-home-${seedId}`, "tree"); + const outside = join(f.temp, "outside"); mkdirSync(outside); + symlinkSync(outside, join(staging, "alias")); + expect(f.call({ action: "seed-directory", seedId, path: "alias/escape" })).toEqual({ error: "invalid" }); + expect(readdirSync(outside)).toEqual([]); + // A malicious link must not be included in the successful fixture publication. + unlinkSync(join(staging, "alias")); + expect(f.call({ action: "seed-commit", seedId })).toEqual({ seeded: true }); + expect(statSync(join(f.root, "empty/nested")).isDirectory()).toBe(true); + expect(statSync(join(f.root, "empty/nested")).mode & 0o777).toBe(0o700); + }); it("publishes bounded seed chunks atomically and aborts partial homes", () => { const f = fixture(); const seedId = randomUUID(); expect(f.call({ action: "seed-begin", seedId })).toEqual({ started: true }); diff --git a/server/src/modules/computers/adapters/remote-program.ts b/server/src/modules/computers/adapters/remote-program.ts index 638a267a152..9f58ed285da 100644 --- a/server/src/modules/computers/adapters/remote-program.ts +++ b/server/src/modules/computers/adapters/remote-program.ts @@ -124,7 +124,7 @@ def directory(parts,create=False,start=None): except BaseException:os.close(fd);raise # Initial uploads stay in a private sibling until a no-replace rename publishes them. # Each request carries at most one bounded chunk, never the whole home. -seed_actions=('seed-begin','seed-chunk','seed-commit','seed-abort') +seed_actions=('seed-begin','seed-directory','seed-chunk','seed-commit','seed-abort') # The durable receipt names one exact upload. Root listings (including home # admission) reclaim expired uploads even after a controller crash or publish. if act in seed_actions or (act=='list' and not p.get('path') and os.path.lexists(os.path.join(os.path.dirname(root),'.paperclip-seed-'+os.path.basename(root)+'.json'))): @@ -209,7 +209,14 @@ if act in seed_actions or (act=='list' and not p.get('path') and os.path.lexists if receipt is None or receipt['seedId']!=token:fail('conflict') stagefd=os.open(staging,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=parentfd) if read_metadata(stagefd,'owner.json')!=identity(token):fail('invalid') - if act=='seed-chunk': + if act=='seed-directory': + relative=p.get('path','');safe(relative) + parts=[part for part in relative.split('/') if part not in ('','.')] + if not parts:fail('invalid') + fd=directory(['tree']+parts,create=True,start=stagefd);os.close(fd) + receipt['updatedAt']=time.time();save_receipt(receipt) + print('{}') + elif act=='seed-chunk': relative=p.get('path','');safe(relative) parts=[part for part in relative.split('/') if part not in ('','.')] if not parts:fail('invalid') diff --git a/server/src/modules/computers/application/service.test.ts b/server/src/modules/computers/application/service.test.ts index d1526691aed..09b558e7807 100644 --- a/server/src/modules/computers/application/service.test.ts +++ b/server/src/modules/computers/application/service.test.ts @@ -770,6 +770,12 @@ with open(output,'wb') as out,open(error,'wb') as err: expect(chunks.map(input => input.offset)).toEqual([0, 1024 * 1024, 2 * 1024 * 1024]); expect(chunks.every(input => Buffer.from(input.base64 as string, "base64").length <= 1024 * 1024)).toBe(true); vi.mocked(f.backend.remote).mockClear(); + await expect(files.seedFiles((async function* () {})(), ["empty", "empty/nested"])).resolves.toEqual({ seeded: true }); + const directoryRequests = vi.mocked(f.backend.remote).mock.calls.map(([, input]) => input); + expect(directoryRequests.map(input => input.action)).toEqual(["seed-begin", "seed-directory", "seed-directory", "seed-commit"]); + expect(directoryRequests.slice(1, 3).map(input => input.path)).toEqual(["empty", "empty/nested"]); + expect(new Set(directoryRequests.map(input => input.seedId)).size).toBe(1); + vi.mocked(f.backend.remote).mockClear(); await expect(files.seedFiles((async function* () { yield { path: "first", offset: 0, bytes: Buffer.from("first") }; throw new Error("source disappeared"); diff --git a/server/src/modules/computers/application/service.ts b/server/src/modules/computers/application/service.ts index 8a2d1e85b95..ab471a74096 100644 --- a/server/src/modules/computers/application/service.ts +++ b/server/src/modules/computers/application/service.ts @@ -797,11 +797,13 @@ finally: base64: bytes.toString("base64"), expectedSha256, }); - const seedFiles = async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>): Promise<{ seeded: boolean }> => { + const seedFiles = async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>, directories: readonly string[] = []): Promise<{ seeded: boolean }> => { const seedId = randomUUID(); const started = await remote({ action: "seed-begin", seedId }); if (!started.started) return { seeded: false }; try { + if (directories.length > 100_000) throw new ComputerError("invalid", "Initial home exceeds its entry limit"); + for (const path of directories) await remote({ action: "seed-directory", seedId, path }); for await (const chunk of chunks) { if (chunk.bytes.length > 1024 * 1024) throw new ComputerError("invalid", "Initial home chunk exceeds its size limit"); await remote({ action: "seed-chunk", seedId, path: chunk.path, offset: chunk.offset, base64: chunk.bytes.toString("base64") }); diff --git a/server/src/services/persistent-agent-files.test.ts b/server/src/services/persistent-agent-files.test.ts new file mode 100644 index 00000000000..5d61f444829 --- /dev/null +++ b/server/src/services/persistent-agent-files.test.ts @@ -0,0 +1,80 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { seedPersistentAgentHome } from "./persistent-agent-files.js"; + +describe("persistent home initialization", () => { + const roots: string[] = []; + afterEach(async () => { for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); }); + async function local() { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "boat-agent-seed-")); roots.push(root); + await fs.mkdir(path.join(root, "memory")); + await fs.writeFile(path.join(root, "AGENTS.md"), "instructions"); + await fs.writeFile(path.join(root, "memory", "binary.bin"), Buffer.from([0, 255, 1])); + return root; + } + it.each([false, true])("adopts existing remote storage without reading the controller tree (truncated: %s)", async truncated => { + const seedFiles = vi.fn(); + await seedPersistentAgentHome({ listPage: async () => ({ entries: [], truncated }), seedFiles } as never, "/missing-controller-folder"); + expect(seedFiles).not.toHaveBeenCalled(); + }); + it("seeds absent personal storage once with exact binary bytes", async () => { + const received: Record = {}; + const seedFiles = vi.fn(async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>) => { + for await (const chunk of chunks) received[chunk.path] = Buffer.concat([received[chunk.path] ?? Buffer.alloc(0), chunk.bytes]); + return { seeded: true }; + }); + await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles } as never, await local()); + expect(received).toEqual({ "AGENTS.md": Buffer.from("instructions"), "memory/binary.bin": Buffer.from([0, 255, 1]) }); + }); + it("streams large files in bounded chunks without importing runtime-owned files", async () => { + const root = await local(); + const expected = Buffer.alloc(2 * 1024 * 1024 + 17, 93); + await fs.writeFile(path.join(root, "large.bin"), expected); + await fs.mkdir(path.join(root, ".paperclip-runtime")); + await fs.writeFile(path.join(root, ".paperclip-runtime", "private"), "runtime material"); + await fs.mkdir(path.join(root, "memory", ".paperclip-runtime")); + await fs.writeFile(path.join(root, "memory", ".paperclip-runtime", "user.txt"), "user content"); + const chunks: Buffer[] = []; + const paths: string[] = []; + let maxChunk = 0; + await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles: async (stream: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>) => { + for await (const chunk of stream) { + maxChunk = Math.max(maxChunk, chunk.bytes.length); paths.push(chunk.path); + if (chunk.path === "large.bin") { + expect(chunk.offset).toBe(chunks.reduce((total, b) => total + b.length, 0)); + chunks.push(Buffer.from(chunk.bytes)); + } + } + } } as never, root); + expect(maxChunk).toBe(1024 * 1024); + expect(chunks).toHaveLength(3); + expect(Buffer.concat(chunks)).toEqual(expected); + expect(paths).not.toContain(".paperclip-runtime/private"); + expect(paths).toContain("memory/.paperclip-runtime/user.txt"); + }); + it("preserves empty and nested directories in the first seed", async () => { + const root = await local(); + await fs.mkdir(path.join(root, "empty", "nested"), { recursive: true }); + await fs.mkdir(path.join(root, ".paperclip-runtime")); + const seedFiles = vi.fn(async (stream: AsyncIterable) => { + for await (const _chunk of stream) { /* consume the source */ } + return { seeded: true }; + }); + await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles } as never, root); + expect(seedFiles.mock.calls[0]).toEqual([expect.anything(), ["empty", "empty/nested", "memory"]]); + }); + it("does not interpret provider outages as an absent directory", async () => { + const seedFiles = vi.fn(); + await expect(seedPersistentAgentHome({ listPage: async () => { throw new Error("offline"); }, seedFiles } as never, await local())).rejects.toThrow("offline"); + expect(seedFiles).not.toHaveBeenCalled(); + }); + it("rejects symlinked initial files before any upload", async () => { + const root = await local(); + await fs.symlink(path.join(root, "AGENTS.md"), path.join(root, "alias.md")); + const seedFiles = vi.fn(); + await expect(seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles } as never, root)).rejects.toThrow("links or special files"); + expect(seedFiles).not.toHaveBeenCalled(); + }); +}); diff --git a/server/src/services/persistent-agent-files.ts b/server/src/services/persistent-agent-files.ts index e47d095f2cf..c8d2356daef 100644 --- a/server/src/services/persistent-agent-files.ts +++ b/server/src/services/persistent-agent-files.ts @@ -51,6 +51,7 @@ export async function seedPersistentAgentHome(files: NonNullable = []; + const initialDirectories: string[] = []; let total = 0, count = 0; async function walk(relative: string) { const entries = await fs.readdir(path.join(localRoot, relative), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { @@ -63,7 +64,10 @@ export async function seedPersistentAgentHome(files: NonNullable 100_000) throw new Error("Initial agent folder exceeds its entry limit"); const stat = await fs.lstat(path.join(localRoot, name)); - if (stat.isDirectory()) await walk(name); + if (stat.isDirectory()) { + initialDirectories.push(name); + await walk(name); + } else if (stat.isFile() && stat.nlink === 1) { total += stat.size; if (stat.size > 256 * 1024 * 1024 || total > 2 * 1024 * 1024 * 1024) throw new Error("Initial agent folder exceeds its storage limit"); @@ -88,7 +92,7 @@ export async function seedPersistentAgentHome(files: NonNullable