Preserve empty directories during persistent home seeding

Stage directory entries under the existing exact upload token and no-follow directory handles before atomic publication. Preserve adoption of existing homes and bounded file streaming.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/modules/computers/adapters/boat.test.ts

@@ -1,5 +1,6 @@
 import {
   existsSync,
+  unlinkSync,
   readFileSync,
   readdirSync,
   statSync,
@@ -511,6 +512,24 @@ describe("confined computer files", () => {
       error: "invalid",
     });
   });
+  it("stages empty seed directories with exact upload ownership and no-follow paths", () => {
+    const f = fixture(); const seedId = randomUUID();
+    f.call({ action: "seed-begin", seedId });
+    expect(f.call({ action: "seed-directory", seedId, path: "empty/nested" })).toEqual({});
+    expect(existsSync(f.root)).toBe(false);
+    expect(f.call({ action: "seed-directory", seedId: randomUUID(), path: "foreign" })).toEqual({ error: "conflict" });
+    expect(f.call({ action: "seed-directory", seedId, path: "../escape" })).toEqual({ error: "invalid" });
+    const staging = join(f.temp, `.paperclip-seed-home-${seedId}`, "tree");
+    const outside = join(f.temp, "outside"); mkdirSync(outside);
+    symlinkSync(outside, join(staging, "alias"));
+    expect(f.call({ action: "seed-directory", seedId, path: "alias/escape" })).toEqual({ error: "invalid" });
+    expect(readdirSync(outside)).toEqual([]);
+    // A malicious link must not be included in the successful fixture publication.
+    unlinkSync(join(staging, "alias"));
+    expect(f.call({ action: "seed-commit", seedId })).toEqual({ seeded: true });
+    expect(statSync(join(f.root, "empty/nested")).isDirectory()).toBe(true);
+    expect(statSync(join(f.root, "empty/nested")).mode & 0o777).toBe(0o700);
+  });
   it("publishes bounded seed chunks atomically and aborts partial homes", () => {
     const f = fixture(); const seedId = randomUUID();
     expect(f.call({ action: "seed-begin", seedId })).toEqual({ started: true });

server/src/modules/computers/adapters/remote-program.ts

@@ -124,7 +124,7 @@ def directory(parts,create=False,start=None):
  except BaseException:os.close(fd);raise
 # Initial uploads stay in a private sibling until a no-replace rename publishes them.
 # Each request carries at most one bounded chunk, never the whole home.
-seed_actions=('seed-begin','seed-chunk','seed-commit','seed-abort')
+seed_actions=('seed-begin','seed-directory','seed-chunk','seed-commit','seed-abort')
 # The durable receipt names one exact upload. Root listings (including home
 # admission) reclaim expired uploads even after a controller crash or publish.
 if act in seed_actions or (act=='list' and not p.get('path') and os.path.lexists(os.path.join(os.path.dirname(root),'.paperclip-seed-'+os.path.basename(root)+'.json'))):
@@ -209,7 +209,14 @@ if act in seed_actions or (act=='list' and not p.get('path') and os.path.lexists
     if receipt is None or receipt['seedId']!=token:fail('conflict')
     stagefd=os.open(staging,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=parentfd)
     if read_metadata(stagefd,'owner.json')!=identity(token):fail('invalid')
-    if act=='seed-chunk':
+    if act=='seed-directory':
+     relative=p.get('path','');safe(relative)
+     parts=[part for part in relative.split('/') if part not in ('','.')]
+     if not parts:fail('invalid')
+     fd=directory(['tree']+parts,create=True,start=stagefd);os.close(fd)
+     receipt['updatedAt']=time.time();save_receipt(receipt)
+     print('{}')
+    elif act=='seed-chunk':
      relative=p.get('path','');safe(relative)
      parts=[part for part in relative.split('/') if part not in ('','.')]
      if not parts:fail('invalid')

server/src/modules/computers/application/service.test.ts

@@ -770,6 +770,12 @@ with open(output,'wb') as out,open(error,'wb') as err:
     expect(chunks.map(input => input.offset)).toEqual([0, 1024 * 1024, 2 * 1024 * 1024]);
     expect(chunks.every(input => Buffer.from(input.base64 as string, "base64").length <= 1024 * 1024)).toBe(true);
     vi.mocked(f.backend.remote).mockClear();
+    await expect(files.seedFiles((async function* () {})(), ["empty", "empty/nested"])).resolves.toEqual({ seeded: true });
+    const directoryRequests = vi.mocked(f.backend.remote).mock.calls.map(([, input]) => input);
+    expect(directoryRequests.map(input => input.action)).toEqual(["seed-begin", "seed-directory", "seed-directory", "seed-commit"]);
+    expect(directoryRequests.slice(1, 3).map(input => input.path)).toEqual(["empty", "empty/nested"]);
+    expect(new Set(directoryRequests.map(input => input.seedId)).size).toBe(1);
+    vi.mocked(f.backend.remote).mockClear();
     await expect(files.seedFiles((async function* () {
       yield { path: "first", offset: 0, bytes: Buffer.from("first") };
       throw new Error("source disappeared");

server/src/modules/computers/application/service.ts

@@ -797,11 +797,13 @@ finally:
         base64: bytes.toString("base64"),
         expectedSha256,
       });
-    const seedFiles = async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>): Promise<{ seeded: boolean }> => {
+    const seedFiles = async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>, directories: readonly string[] = []): Promise<{ seeded: boolean }> => {
       const seedId = randomUUID();
       const started = await remote({ action: "seed-begin", seedId });
       if (!started.started) return { seeded: false };
       try {
+        if (directories.length > 100_000) throw new ComputerError("invalid", "Initial home exceeds its entry limit");
+        for (const path of directories) await remote({ action: "seed-directory", seedId, path });
         for await (const chunk of chunks) {
           if (chunk.bytes.length > 1024 * 1024) throw new ComputerError("invalid", "Initial home chunk exceeds its size limit");
           await remote({ action: "seed-chunk", seedId, path: chunk.path, offset: chunk.offset, base64: chunk.bytes.toString("base64") });

server/src/services/persistent-agent-files.test.ts

@@ -0,0 +1,80 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { seedPersistentAgentHome } from "./persistent-agent-files.js";
+
+describe("persistent home initialization", () => {
+  const roots: string[] = [];
+  afterEach(async () => { for (const root of roots.splice(0)) await fs.rm(root, { recursive: true, force: true }); });
+  async function local() {
+    const root = await fs.mkdtemp(path.join(os.tmpdir(), "boat-agent-seed-")); roots.push(root);
+    await fs.mkdir(path.join(root, "memory"));
+    await fs.writeFile(path.join(root, "AGENTS.md"), "instructions");
+    await fs.writeFile(path.join(root, "memory", "binary.bin"), Buffer.from([0, 255, 1]));
+    return root;
+  }
+  it.each([false, true])("adopts existing remote storage without reading the controller tree (truncated: %s)", async truncated => {
+    const seedFiles = vi.fn();
+    await seedPersistentAgentHome({ listPage: async () => ({ entries: [], truncated }), seedFiles } as never, "/missing-controller-folder");
+    expect(seedFiles).not.toHaveBeenCalled();
+  });
+  it("seeds absent personal storage once with exact binary bytes", async () => {
+    const received: Record<string, Buffer> = {};
+    const seedFiles = vi.fn(async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>) => {
+      for await (const chunk of chunks) received[chunk.path] = Buffer.concat([received[chunk.path] ?? Buffer.alloc(0), chunk.bytes]);
+      return { seeded: true };
+    });
+    await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles } as never, await local());
+    expect(received).toEqual({ "AGENTS.md": Buffer.from("instructions"), "memory/binary.bin": Buffer.from([0, 255, 1]) });
+  });
+  it("streams large files in bounded chunks without importing runtime-owned files", async () => {
+    const root = await local();
+    const expected = Buffer.alloc(2 * 1024 * 1024 + 17, 93);
+    await fs.writeFile(path.join(root, "large.bin"), expected);
+    await fs.mkdir(path.join(root, ".paperclip-runtime"));
+    await fs.writeFile(path.join(root, ".paperclip-runtime", "private"), "runtime material");
+    await fs.mkdir(path.join(root, "memory", ".paperclip-runtime"));
+    await fs.writeFile(path.join(root, "memory", ".paperclip-runtime", "user.txt"), "user content");
+    const chunks: Buffer[] = [];
+    const paths: string[] = [];
+    let maxChunk = 0;
+    await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles: async (stream: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>) => {
+      for await (const chunk of stream) {
+        maxChunk = Math.max(maxChunk, chunk.bytes.length); paths.push(chunk.path);
+        if (chunk.path === "large.bin") {
+          expect(chunk.offset).toBe(chunks.reduce((total, b) => total + b.length, 0));
+          chunks.push(Buffer.from(chunk.bytes));
+        }
+      }
+    } } as never, root);
+    expect(maxChunk).toBe(1024 * 1024);
+    expect(chunks).toHaveLength(3);
+    expect(Buffer.concat(chunks)).toEqual(expected);
+    expect(paths).not.toContain(".paperclip-runtime/private");
+    expect(paths).toContain("memory/.paperclip-runtime/user.txt");
+  });
+  it("preserves empty and nested directories in the first seed", async () => {
+    const root = await local();
+    await fs.mkdir(path.join(root, "empty", "nested"), { recursive: true });
+    await fs.mkdir(path.join(root, ".paperclip-runtime"));
+    const seedFiles = vi.fn(async (stream: AsyncIterable<unknown>) => {
+      for await (const _chunk of stream) { /* consume the source */ }
+      return { seeded: true };
+    });
+    await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles } as never, root);
+    expect(seedFiles.mock.calls[0]).toEqual([expect.anything(), ["empty", "empty/nested", "memory"]]);
+  });
+  it("does not interpret provider outages as an absent directory", async () => {
+    const seedFiles = vi.fn();
+    await expect(seedPersistentAgentHome({ listPage: async () => { throw new Error("offline"); }, seedFiles } as never, await local())).rejects.toThrow("offline");
+    expect(seedFiles).not.toHaveBeenCalled();
+  });
+  it("rejects symlinked initial files before any upload", async () => {
+    const root = await local();
+    await fs.symlink(path.join(root, "AGENTS.md"), path.join(root, "alias.md"));
+    const seedFiles = vi.fn();
+    await expect(seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedFiles } as never, root)).rejects.toThrow("links or special files");
+    expect(seedFiles).not.toHaveBeenCalled();
+  });
+});

server/src/services/persistent-agent-files.ts

@@ -51,6 +51,7 @@ export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnT
   try { await files.listPage("", { limit: 1 }); return; }
   catch (error) { if (!isMissingRemoteFile(error)) throw error; }
   const initialFiles: Array<{ name: string; size: number; dev: number; ino: number }> = [];
+  const initialDirectories: string[] = [];
   let total = 0, count = 0;
   async function walk(relative: string) {
     const entries = await fs.readdir(path.join(localRoot, relative), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
@@ -63,7 +64,10 @@ export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnT
       instructionPath(name);
       if (++count > 100_000) throw new Error("Initial agent folder exceeds its entry limit");
       const stat = await fs.lstat(path.join(localRoot, name));
-      if (stat.isDirectory()) await walk(name);
+      if (stat.isDirectory()) {
+        initialDirectories.push(name);
+        await walk(name);
+      }
       else if (stat.isFile() && stat.nlink === 1) {
         total += stat.size;
         if (stat.size > 256 * 1024 * 1024 || total > 2 * 1024 * 1024 * 1024) throw new Error("Initial agent folder exceeds its storage limit");
@@ -88,7 +92,7 @@ export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnT
         } while (offset < file.size);
       } finally { await handle.close(); }
     }
-  })());
+  })(), initialDirectories);
 }
 
 /** Translate the domain's transport-neutral errors at the HTTP service edge. */