Bound initial computer home uploads and exclude runtime state

Stream initial personal files through bounded chunks into private staging and atomically publish without replacing an existing home. Keep the home-level runtime directory out of instruction enumeration and seeding while preserving nested user content.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/agent-directory-working-copies.test.ts

@@ -78,11 +78,15 @@ describe("persistent agent directories", () => {
   it("seeds a persistent execution home before a run or instruction-copy receipt exists", async () => {
     const remoteRoot = "/home/user/paperclip/test/agents/target";
     let exists = false;
-    const seedBytes = vi.fn(async () => { exists = true; });
+    const received: Record<string, Buffer> = {};
+    const seedFiles = vi.fn(async (chunks: AsyncIterable<{ path: string; bytes: Buffer }>) => {
+      for await (const chunk of chunks) received[chunk.path] = Buffer.concat([received[chunk.path] ?? Buffer.alloc(0), chunk.bytes]);
+      exists = true;
+    });
     const remote = { root: remoteRoot, listPage: async () => {
       if (!exists) throw { code: "not_found" };
       return { entries: [], truncated: false };
-    }, seedBytes };
+    }, seedFiles };
     const lookup = vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue(remote as never);
     const environmentId = randomUUID();
     const executionTarget = { kind: "remote", transport: "computer", environmentId,
@@ -91,14 +95,14 @@ describe("persistent agent directories", () => {
       await fs.writeFile(path.join(root, "memory.bin"), Buffer.from([0, 255, 7]));
       expect(await preparePersistentAgentExecutionHome(db, { companyId, agentId, target: executionTarget })).toBe(remoteRoot);
       expect(lookup).toHaveBeenCalledWith(db, companyId, agentId, environmentId);
-      expect(seedBytes).toHaveBeenCalledOnce();
-      expect(seedBytes).toHaveBeenCalledWith(expect.objectContaining({
+      expect(seedFiles).toHaveBeenCalledOnce();
+      expect(received).toEqual(expect.objectContaining({
         [entryFile]: Buffer.from(initial), "memory.bin": Buffer.from([0, 255, 7]),
       }));
       // Subsequent setup adopts the remote tree, regardless of controller edits.
       await fs.writeFile(path.join(root, "memory.bin"), "stale controller bytes");
       await preparePersistentAgentExecutionHome(db, { companyId, agentId, target: executionTarget });
-      expect(seedBytes).toHaveBeenCalledOnce();
+      expect(seedFiles).toHaveBeenCalledOnce();
       expect(await db.select().from(agentInstructionWorkingCopies).where(eq(agentInstructionWorkingCopies.agentId, agentId))).toEqual([]);
     } finally { lookup.mockRestore(); }
   });

server/src/__tests__/agent-instructions-service.test.ts

@@ -2,6 +2,7 @@ import fs from "node:fs/promises";
 import os from "node:os";
 import path from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import * as fileStore from "../services/agent-file-store.js";
 import * as persistentFiles from "../services/persistent-agent-files.js";
 import { agentInstructionsService } from "../services/agent-instructions.js";
 
@@ -82,6 +83,26 @@ describe("agent instructions service", () => {
     expect(db.transaction).not.toHaveBeenCalled();
   });
 
+  it("excludes only home-level runtime files from remote listing and export", async () => {
+    const agent = makeAgent({ instructionsBundleMode: "managed" });
+    const list = vi.fn(async (relative = "") => {
+      if (relative === ".paperclip-runtime") throw new Error("Must not enumerate provider packs or session state");
+      if (!relative) return [{ name: "AGENTS.md", kind: "file", size: 12 }, { name: ".paperclip-runtime", kind: "directory", size: 0 }, { name: "notes", kind: "directory", size: 0 }];
+      if (relative === "notes") return [{ name: ".paperclip-runtime", kind: "directory", size: 0 }];
+      return [{ name: "user.md", kind: "file", size: 12 }];
+    });
+    const readBytes = vi.fn(async () => ({ bytes: Buffer.from("instructions"), sha256: "hash" }));
+    vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue({ root: "/remote/home", list, readBytes } as never);
+    vi.spyOn(persistentFiles, "seedPersistentAgentHome").mockResolvedValue(undefined);
+    vi.spyOn(fileStore, "adoptAgentFiles").mockResolvedValue("/controller/home");
+    const db = { transaction: async (fn: (tx: unknown) => Promise<unknown>) => fn({ select: () => ({ from: () => ({ where: async () => [agent] }) }) }) };
+    const svc = agentInstructionsService(db as never);
+    expect((await svc.getBundle(agent)).files.map(file => file.path)).toEqual(["AGENTS.md", "notes/.paperclip-runtime/user.md"]);
+    expect((await svc.exportFiles(agent)).files).toEqual({ "AGENTS.md": "instructions", "notes/.paperclip-runtime/user.md": "instructions" });
+    expect(list).not.toHaveBeenCalledWith(".paperclip-runtime");
+    expect(readBytes).not.toHaveBeenCalledWith(expect.stringMatching(/^\.paperclip-runtime\//));
+  });
+
   it("previews an explicit external-to-managed migration before adopting remote personal files", async () => {
     const root = await makeTempDir("boat-instructions-migration-"); cleanupDirs.add(root);
     await fs.writeFile(path.join(root, "AGENTS.md"), "external migration source");

server/src/modules/computers/adapters/boat.test.ts

@@ -1,14 +1,16 @@
 import {
   existsSync,
-  mkdtempSync,
   readFileSync,
+  readdirSync,
+  statSync,
+  mkdtempSync,
   realpathSync,
   rmSync,
   symlinkSync,
   writeFileSync,
   mkdirSync,
 } from "node:fs";
-import { createHash } from "node:crypto";
+import { createHash, randomUUID } from "node:crypto";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { spawnSync } from "node:child_process";
@@ -172,6 +174,37 @@ describe("confined computer files", () => {
       error: "invalid",
     });
   });
+  it("publishes bounded seed chunks atomically and aborts partial homes", () => {
+    const f = fixture(); const seedId = randomUUID();
+    expect(f.call({ action: "seed-begin", seedId })).toEqual({ started: true });
+    const chunk = Buffer.alloc(1024 * 1024, 19);
+    for (const offset of [0, chunk.length]) expect(f.call({ action: "seed-chunk", seedId, path: "memory/data", offset, base64: chunk.toString("base64") })).toEqual({});
+    expect(existsSync(f.root)).toBe(false);
+    const staging = readdirSync(f.temp).find(name => name.startsWith(".paperclip-seed-"))!;
+    expect(statSync(join(f.temp, staging)).mode & 0o777).toBe(0o700);
+    expect(statSync(join(f.temp, staging, "tree", "memory", "data")).mode & 0o777).toBe(0o600);
+    expect(f.call({ action: "seed-chunk", seedId, path: "memory/data", offset: 0, base64: "" })).toEqual({ error: "conflict" });
+    expect(f.call({ action: "seed-commit", seedId })).toEqual({ seeded: true });
+    expect(readFileSync(join(f.root, "memory", "data"))).toEqual(Buffer.concat([chunk, chunk]));
+    expect(readdirSync(f.temp)).toEqual(["home"]);
+    expect(f.call({ action: "seed-begin", seedId: randomUUID() })).toEqual({ started: false });
+    const g = fixture(); const partial = randomUUID();
+    g.call({ action: "seed-begin", seedId: partial });
+    expect(g.call({ action: "seed-chunk", seedId: partial, path: "../escape", offset: 0, base64: "" })).toEqual({ error: "invalid" });
+    expect(g.call({ action: "seed-chunk", seedId: partial, path: "large", offset: 0, base64: Buffer.alloc(1024 * 1024 + 1).toString("base64") })).toEqual({ error: "invalid" });
+    expect(g.call({ action: "seed-abort", seedId: partial })).toEqual({});
+    expect(existsSync(g.root)).toBe(false);
+    expect(readdirSync(g.temp)).toEqual([]);
+  });
+  it("does not replace a home created while its initial upload was in progress", () => {
+    const f = fixture(); const seedId = randomUUID();
+    f.call({ action: "seed-begin", seedId });
+    f.call({ action: "seed-chunk", seedId, path: "initial", offset: 0, base64: Buffer.from("staged").toString("base64") });
+    mkdirSync(f.root);
+    expect(f.call({ action: "seed-commit", seedId })).toEqual({ seeded: false });
+    expect(readdirSync(f.root)).toEqual([]);
+    expect(readdirSync(f.temp)).toEqual(["home"]);
+  });
   it("hashes and deletes files over the read limit while fencing changed bytes", () => {
     const f = fixture();
     f.call({ action: "seed", files: {} });

server/src/modules/computers/adapters/remote-program.ts

@@ -107,6 +107,70 @@ def directory(parts,create=False,start=None):
    os.close(fd);fd=nxt
   return fd
  except BaseException:os.close(fd);raise
+# Initial uploads stay in a private sibling until a no-replace rename publishes them.
+# Each request carries at most one bounded chunk, never the whole home.
+if act in ('seed-begin','seed-chunk','seed-commit','seed-abort'):
+ import uuid,ctypes,errno
+ token=p.get('seedId','')
+ try:
+  if str(uuid.UUID(token))!=token:fail('invalid')
+ except (ValueError,TypeError,AttributeError):fail('invalid')
+ parentfd=directory(os.path.dirname(root).split('/'),create=True)
+ name=os.path.basename(root);staging='.paperclip-seed-'+name+'-'+token
+ stagefd=None;lockfd=None
+ try:
+  if act=='seed-abort':
+   try:shutil.rmtree(staging,dir_fd=parentfd)
+   except FileNotFoundError:pass
+   print('{}');sys.exit(0)
+  if act=='seed-begin':
+   try:
+    existing=os.stat(name,dir_fd=parentfd,follow_symlinks=False)
+    if not stat.S_ISDIR(existing.st_mode):fail('invalid')
+    print(json.dumps({'started':False}));sys.exit(0)
+   except FileNotFoundError:pass
+   os.mkdir(staging,0o700,dir_fd=parentfd)
+  stagefd=os.open(staging,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=parentfd)
+  lockfd=os.open('lock',os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600,dir_fd=stagefd)
+  fcntl.flock(lockfd,fcntl.LOCK_EX)
+  if act=='seed-begin':
+   os.mkdir('tree',0o700,dir_fd=stagefd)
+   print(json.dumps({'started':True}))
+  elif act=='seed-chunk':
+   relative=p.get('path','');safe(relative)
+   parts=[part for part in relative.split('/') if part not in ('','.')]
+   if not parts:fail('invalid')
+   data=base64.b64decode(p['base64'],validate=True);offset=p.get('offset')
+   if len(data)>1024*1024 or type(offset)!=int or offset<0 or offset+len(data)>256*1024*1024:fail('invalid')
+   destfd=directory(['tree']+parts[:-1],create=True,start=stagefd)
+   try:fd=os.open(parts[-1],os.O_WRONLY|os.O_CREAT|os.O_NOFOLLOW,0o600,dir_fd=destfd)
+   finally:os.close(destfd)
+   with os.fdopen(fd,'r+b') as f:
+    info=os.fstat(f.fileno())
+    if not stat.S_ISREG(info.st_mode) or info.st_nlink!=1:fail('invalid')
+    if info.st_size!=offset:fail('conflict')
+    f.seek(offset);f.write(data)
+   print('{}')
+  else:
+   libc=ctypes.CDLL(None,use_errno=True)
+   if hasattr(libc,'renameat2'):status=libc.renameat2(stagefd,b'tree',parentfd,os.fsencode(name),1)
+   elif hasattr(libc,'renameatx_np'):status=libc.renameatx_np(stagefd,b'tree',parentfd,os.fsencode(name),4)
+   else:fail('invalid')
+   if status!=0:
+    code=ctypes.get_errno()
+    if code not in (errno.EEXIST,errno.ENOTEMPTY):raise OSError(code,os.strerror(code))
+    if not stat.S_ISDIR(os.stat(name,dir_fd=parentfd,follow_symlinks=False).st_mode):fail('invalid')
+   print(json.dumps({'seeded':status==0}))
+ except FileNotFoundError:fail('not_found')
+ except (OSError,ValueError):fail('invalid')
+ finally:
+  if lockfd is not None:os.close(lockfd)
+  if stagefd is not None:os.close(stagefd)
+  if act=='seed-commit':
+   try:shutil.rmtree(staging,dir_fd=parentfd)
+   except FileNotFoundError:pass
+  os.close(parentfd)
+ sys.exit(0)
 try:rootfd=directory(root.split('/'))
 except FileNotFoundError:fail('not_found')
 except OSError:fail('invalid')

server/src/modules/computers/application/service.test.ts

@@ -545,6 +545,32 @@ with open(output,'wb') as out,open(error,'wb') as err:
       rmSync(temp, { recursive: true, force: true });
     }
   });
+  it("bounds initial seed requests and aborts a failed upload without publishing", async () => {
+    const f = fixture();
+    await f.attach();
+    const files = await f.service.files({ ...f.scope, agentId: "agent" });
+    vi.mocked(f.backend.remote).mockImplementation(async (_record, input) => {
+      if (input.action === "seed-begin") return { started: true };
+      if (input.action === "seed-commit") return { seeded: true };
+      return {};
+    });
+    await expect(files.seedBytes({ "large.bin": Buffer.alloc(2 * 1024 * 1024 + 1) })).resolves.toEqual({ seeded: true });
+    const chunks = vi.mocked(f.backend.remote).mock.calls.map(([, input]) => input).filter(input => input.action === "seed-chunk");
+    expect(chunks).toHaveLength(3);
+    expect(chunks.map(input => input.offset)).toEqual([0, 1024 * 1024, 2 * 1024 * 1024]);
+    expect(chunks.every(input => Buffer.from(input.base64 as string, "base64").length <= 1024 * 1024)).toBe(true);
+    vi.mocked(f.backend.remote).mockClear();
+    await expect(files.seedFiles((async function* () {
+      yield { path: "first", offset: 0, bytes: Buffer.from("first") };
+      throw new Error("source disappeared");
+    })())).rejects.toThrow("source disappeared");
+    const actions = vi.mocked(f.backend.remote).mock.calls.map(([, input]) => input.action);
+    expect(actions).toEqual(["seed-begin", "seed-chunk", "seed-abort"]);
+    vi.mocked(f.backend.remote).mockResolvedValue({ started: false });
+    let read = false;
+    await expect(files.seedFiles((async function* () { read = true; yield { path: "unused", offset: 0, bytes: Buffer.alloc(0) }; })())).resolves.toEqual({ seeded: false });
+    expect(read).toBe(false);
+  });
   it("realizes probe cwd before readiness without consuming the initial personal-home seed", async () => {
     const f = fixture();
     const temp = realpathSync(mkdtempSync(join(tmpdir(), "computer-probe-")));

server/src/modules/computers/application/service.ts

@@ -687,9 +687,33 @@ finally:
         base64: bytes.toString("base64"),
         expectedSha256,
       });
+    const seedFiles = async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>): Promise<{ seeded: boolean }> => {
+      const seedId = randomUUID();
+      const started = await remote({ action: "seed-begin", seedId });
+      if (!started.started) return { seeded: false };
+      try {
+        for await (const chunk of chunks) {
+          if (chunk.bytes.length > 1024 * 1024) throw new ComputerError("invalid", "Initial home chunk exceeds its size limit");
+          await remote({ action: "seed-chunk", seedId, path: chunk.path, offset: chunk.offset, base64: chunk.bytes.toString("base64") });
+        }
+        return await remote({ action: "seed-commit", seedId });
+      } catch (error) {
+        await remote({ action: "seed-abort", seedId }).catch(() => undefined);
+        throw error;
+      }
+    };
+    const seedBytes = async (files: Record<string, Buffer>): Promise<{ seeded: boolean }> => seedFiles((async function* () {
+      for (const [path, bytes] of Object.entries(files)) {
+        for (let offset = 0; offset < Math.max(1, bytes.length); offset += 1024 * 1024) {
+          yield { path, offset, bytes: bytes.subarray(offset, offset + 1024 * 1024) };
+        }
+      }
+    })());
     return {
       root,
       readBytes,
+      seedFiles,
+      seedBytes,
       writeBytes,
       async read(path: string) {
         const value = await readBytes(path);
@@ -729,29 +753,8 @@ finally:
       ): Promise<{ sha256: string }> {
         return remote({ action: "move", path: from, to, expectedSha256 });
       },
-      async seedBytes(
-        files: Record<string, Buffer>,
-      ): Promise<{ seeded: boolean }> {
-        return remote({
-          action: "seed",
-          files: Object.fromEntries(
-            Object.entries(files).map(([key, value]) => [
-              key,
-              value.toString("base64"),
-            ]),
-          ),
-        });
-      },
       async seed(files: Record<string, string>): Promise<{ seeded: boolean }> {
-        return remote({
-          action: "seed",
-          files: Object.fromEntries(
-            Object.entries(files).map(([key, value]) => [
-              key,
-              Buffer.from(value).toString("base64"),
-            ]),
-          ),
-        });
+        return seedBytes(Object.fromEntries(Object.entries(files).map(([path, content]) => [path, Buffer.from(content)])));
       },
     };
   }

server/src/services/agent-instructions.ts

@@ -513,6 +513,9 @@ export function agentInstructionsService(db?: Db) {
     const result: Array<{ path: string; size: number }> = [];
     for (const entry of await remote.list(relative || undefined)) {
       const name = relative ? `${relative}/${entry.name}` : entry.name;
+      // Only the home-level directory belongs to the runtime; nested names can
+      // be ordinary user content and retain their existing listing behavior.
+      if (!relative && entry.name === ".paperclip-runtime") continue;
       instructionPath(name);
       if (entry.kind === "directory") {
         if (!IGNORED_INSTRUCTIONS_DIRECTORY_NAMES.has(entry.name)) result.push(...await remotePaths(remote, name));

server/src/services/persistent-agent-files.ts

@@ -1,4 +1,5 @@
 import fs from "node:fs/promises";
+import { constants } from "node:fs";
 import path from "node:path";
 import { and, eq } from "drizzle-orm";
 import { agents, environments, instanceSettings, type Db } from "@paperclipai/db";
@@ -49,7 +50,7 @@ export async function hashPersistentAgentFile(files: NonNullable<Awaited<ReturnT
 export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnType<typeof persistentAgentFiles>>>, localRoot: string) {
   try { await files.listPage("", { limit: 1 }); return; }
   catch (error) { if (!isMissingRemoteFile(error)) throw error; }
-  const seed: Record<string, Buffer> = {};
+  const initialFiles: Array<{ name: string; size: number; dev: number; ino: number }> = [];
   let total = 0, count = 0;
   async function walk(relative: string) {
     const entries = await fs.readdir(path.join(localRoot, relative), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => {
@@ -58,6 +59,7 @@ export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnT
     });
     for (const entry of entries) {
       const name = relative ? `${relative}/${entry.name}` : entry.name;
+      if (!relative && entry.name === ".paperclip-runtime") continue;
       instructionPath(name);
       if (++count > 100_000) throw new Error("Initial agent folder exceeds its entry limit");
       const stat = await fs.lstat(path.join(localRoot, name));
@@ -65,12 +67,28 @@ export async function seedPersistentAgentHome(files: NonNullable<Awaited<ReturnT
       else if (stat.isFile() && stat.nlink === 1) {
         total += stat.size;
         if (stat.size > 256 * 1024 * 1024 || total > 2 * 1024 * 1024 * 1024) throw new Error("Initial agent folder exceeds its storage limit");
-        seed[name] = await fs.readFile(path.join(localRoot, name));
+        initialFiles.push({ name, size: stat.size, dev: stat.dev, ino: stat.ino });
       } else throw new Error("Agent folders cannot contain links or special files");
     }
   }
   await walk("");
-  await files.seedBytes(seed);
+  await files.seedFiles((async function* () {
+    const buffer = Buffer.allocUnsafe(1024 * 1024);
+    for (const file of initialFiles) {
+      const handle = await fs.open(path.join(localRoot, file.name), constants.O_RDONLY | constants.O_NOFOLLOW);
+      try {
+        const stat = await handle.stat();
+        if (!stat.isFile() || stat.nlink !== 1 || stat.dev !== file.dev || stat.ino !== file.ino || stat.size !== file.size) throw new Error("Initial agent file changed during setup");
+        let offset = 0;
+        do {
+          const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, file.size - offset), offset);
+          if (bytesRead === 0 && offset < file.size) throw new Error("Initial agent file changed during setup");
+          yield { path: file.name, offset, bytes: buffer.subarray(0, bytesRead) };
+          offset += bytesRead;
+        } while (offset < file.size);
+      } finally { await handle.close(); }
+    }
+  })());
 }
 
 /** Translate the domain's transport-neutral errors at the HTTP service edge. */