fix: batch sandbox file transfers and reconcile interrupted hydration

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/services/sandbox-work-folders.ts

@@ -1,3 +1,4 @@
+import { prefetchWorkFiles } from "./work-folder-transfer.js";
 import { createHash, randomUUID } from "node:crypto";
 import path from "node:path";
 import { managedAgentFiles } from "./work-folder-agent-import.js";
@@ -134,10 +135,31 @@ export async function prepareSandboxWorkFolders(input: {
     }
     await db.update(workFolders).set({ importedAt: new Date() }).where(eq(workFolders.id, folder.id));
   }
+  async function reconcileIncoming(scope: WorkFolderScope, current: WorkTreeEntry[]) {
+    const targets = baselines[`incoming:${scope}`];
+    const removed = baselines[`incomingRemoved:${scope}`];
+    if (!targets && !removed) return;
+    const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry]));
+    const observed = new Map(current.map((entry) => [entry.path, entry]));
+    // A failed command may have published some imports or removed some files.
+    // Only adopt effects actually observed on disk; different bytes remain
+    // genuine local edits and must still pass through outgoing synchronization.
+    for (const entry of removed ?? []) if (!observed.has(entry.path)) before.delete(entry.path);
+    for (const entry of targets ?? []) {
+      if (signature(observed.get(entry.path)) === signature(entry)) before.set(entry.path, entry);
+    }
+    baselines[scope] = [...before.values()];
+    delete baselines[`incoming:${scope}`];
+    delete baselines[`incomingRemoved:${scope}`];
+    // Persist the reconciled baseline and remove its provenance atomically,
+    // before any outgoing write can be accepted by the shared collection.
+    await saveState("saving");
+  }
   async function outgoing(scope: WorkFolderScope) {
     const folder = folders[scope];
     if (!folder) return;
     const current = await transport.scan(paths[scope]!);
+    await reconcileIncoming(scope, current);
     const before = new Map((baselines[scope] ?? []).map((entry) => [entry.path, entry]));
     const after = new Map(current.map((entry) => [entry.path, entry]));
     async function operation(filePath: string, nextSignature: string, apply: (id: string) => Promise<unknown>, accept: () => void) {
@@ -186,25 +208,48 @@ export async function prepareSandboxWorkFolders(input: {
       cursor = page.nextCursor ?? undefined;
     } while (cursor);
     const desired = new Map(saved.map((entry) => [entry.path, entry]));
-    // Remove stale children before replacing their parent directory with a file.
-    for (const entry of [...current.values()].sort((a, b) => b.path.length - a.path.length)) {
-      if (!desired.has(entry.path) || desired.get(entry.path)!.kind !== entry.kind) {
-        await transport.remove(paths[scope]!, entry.path);
-        current.delete(entry.path);
-      }
+    // Persist all deletion intents before the first removal, including children
+    // that can disappear when a directory is replaced. A lost response must not
+    // turn an imported deletion into a new delete against a newer shared file.
+    const removed = [...current.values()].filter((entry) => !desired.has(entry.path) || desired.get(entry.path)!.kind !== entry.kind)
+      .sort((a, b) => b.path.length - a.path.length);
+    if (removed.length) {
+      baselines[`incomingRemoved:${scope}`] = removed;
+      await saveState("starting");
     }
-    for (const entry of saved.sort((a, b) => a.path.length - b.path.length)) {
-      if (signature(current.get(entry.path)) === signature(entry)) continue;
-      if (entry.kind === "directory") await transport.mkdir(paths[scope]!, entry.path);
-      else {
-        const result = await svc.content(folder, entry.path);
-        // A shared file can change after listing. Validate and baseline the
-        // version opened by content(), whose metadata and stream belong together.
-        Object.assign(entry, { byteSize: result.file.byteSize, sha256: result.file.sha256, executable: result.file.executable });
-        try { await transport.write(paths[scope]!, staging, entry, result.stream); } finally { result.stream.destroy(); }
-      }
+    for (const entry of removed) {
+      await transport.remove(paths[scope]!, entry.path);
+      current.delete(entry.path);
     }
+    const changed = saved.sort((a, b) => a.path.length - b.path.length)
+      .filter((entry) => signature(current.get(entry.path)) !== signature(entry));
+    await transport.writeMany(paths[scope]!, staging, prefetchWorkFiles(changed, async (entry) => {
+      if (entry.kind === "directory") return { entry };
+      const result = await svc.content(folder, entry.path);
+      // A shared file can change after listing. Validate and baseline the
+      // version opened by content(), whose metadata and stream belong together.
+      Object.assign(entry, { byteSize: result.file.byteSize, sha256: result.file.sha256, executable: result.file.executable });
+      return { entry, body: result.stream };
+    }), async (entries) => {
+      const targets = new Map((baselines[`incoming:${scope}`] ?? []).map((entry) => [entry.path, entry]));
+      for (const entry of entries) {
+        targets.set(entry.path, entry);
+        // Publishing nested files can create parents absent from the listing.
+        // Record those directory imports too, so they cannot be mistaken for
+        // agent-created directories after an interrupted batch.
+        let parent = path.posix.dirname(entry.path);
+        while (parent !== ".") {
+          if (!targets.has(parent)) targets.set(parent, { path: parent, kind: "directory", byteSize: 0, sha256: null, executable: false });
+          parent = path.posix.dirname(parent);
+        }
+      }
+      baselines[`incoming:${scope}`] = [...targets.values()];
+      await saveState("starting");
+    });
     baselines[scope] = saved;
+    delete baselines[`incoming:${scope}`];
+    delete baselines[`incomingRemoved:${scope}`];
+    await saveState("starting");
   }
 
   const bindings: Array<{ binding: typeof taskRepositoryBindings.$inferSelect; root: string }> = [];
@@ -340,10 +385,15 @@ export async function prepareSandboxWorkFolders(input: {
         .where(eq(workFolderRuns.runId, input.runId));
       if (run?.refreshRequested) {
         // The successful final flush protects edits before incoming refresh.
-        await assertBindings();
-        for (const scope of WORK_FOLDER_SCOPES) await incoming(scope);
-        await db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() })
-          .where(eq(workFolderRuns.runId, input.runId));
+        try {
+          await assertBindings();
+          for (const scope of WORK_FOLDER_SCOPES) await incoming(scope);
+          await db.update(workFolderRuns).set({ refreshRequested: false, baselines, updatedAt: new Date() })
+            .where(eq(workFolderRuns.runId, input.runId));
+        } catch (error) {
+          await saveState("failed", "Work folder refresh failed; existing files were retained");
+          throw error;
+        }
       }
       // Publish resume identity after data is durable, before releasing a turn.
       await beforeCompletion?.();

server/src/services/scripts/work-folder-io.mjs

@@ -6,7 +6,7 @@ import os from "node:os";
 import { createHash } from "node:crypto";
 import { execFileSync } from "node:child_process";
 
-const input = JSON.parse(Buffer.from(process.argv[1], "base64").toString("utf8"));
+let input = JSON.parse(Buffer.from(process.argv[1], "base64").toString("utf8"));
 const MAX_CHUNK = 256 * 1024;
 const MAX_ENTRIES = 100_000;
 function safeRelative(value) {
@@ -129,6 +129,8 @@ function scan() {
   return results.sort((a, b) => a.path.localeCompare(b.path));
 }
 
+function execute(request) {
+input = request;
 let result;
 if (input.operation === "home") {
   result = { home: os.homedir() };
@@ -147,7 +149,9 @@ if (input.operation === "home") {
   else if (input.operation === "read") {
     const fd = checked(full(input.path));
     try {
-      const buffer = Buffer.alloc(MAX_CHUNK);
+      const length = input.length ?? MAX_CHUNK;
+      if (!Number.isSafeInteger(length) || length < 1 || length > 1024 * 1024) throw new Error("invalid_read_length");
+      const buffer = Buffer.alloc(length);
       const count = fs.readSync(fd, buffer, 0, buffer.length, input.offset);
       result = { data: buffer.subarray(0, count).toString("base64") };
     } finally { fs.closeSync(fd); }
@@ -200,4 +204,43 @@ if (input.operation === "home") {
     result = {};
   } else throw new Error("unknown_operation");
 }
-process.stdout.write(JSON.stringify(result));
+return result;
+}
+
+const request = input;
+if (request.operation === "read-batch") {
+  if (!Array.isArray(request.entries) || request.entries.length > 64) throw new Error("invalid_read_batch");
+  let bytes = 0;
+  for (const entry of request.entries) {
+    safeRelative(entry.path);
+    if (!Number.isSafeInteger(entry.byteSize) || entry.byteSize < 0) throw new Error("invalid_read_length");
+    bytes += entry.byteSize;
+    if (bytes > 1024 * 1024) throw new Error("read_batch_too_large");
+  }
+  const results = request.entries.map((entry) => execute({ operation: "read", root: request.root,
+    path: entry.path, offset: 0, length: Math.max(1, entry.byteSize) }));
+  process.stdout.write(JSON.stringify(results));
+} else if (request.operation === "batch") {
+  // The provider transports stdin as one bounded upload. The roots stay in the
+  // host-authored argv envelope, so batch contents cannot redirect operations.
+  const chunks = [];
+  let size = 0;
+  const buffer = Buffer.alloc(64 * 1024);
+  let count;
+  while ((count = fs.readSync(0, buffer, 0, buffer.length, null)) > 0) {
+    size += count;
+    if (size > 8 * 1024 * 1024) throw new Error("batch_too_large");
+    chunks.push(Buffer.from(buffer.subarray(0, count)));
+  }
+  const operations = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+  if (!Array.isArray(operations) || operations.length > 512) throw new Error("invalid_batch");
+  for (const operation of operations) {
+    if (!operation || !["write", "publish", "mkdir"].includes(operation.operation)) throw new Error("invalid_batch_operation");
+    // Never accept roots, source roots or filesystem commands from the body.
+    execute({ ...operation, root: operation.operation === "write" ? request.stagingRoot : request.root,
+      stagingRoot: request.stagingRoot });
+  }
+  process.stdout.write(JSON.stringify({ completed: operations.length }));
+} else {
+  process.stdout.write(JSON.stringify(execute(request)));
+}

server/src/services/work-folder-read-cache.ts

@@ -0,0 +1,78 @@
+import { Readable } from "node:stream";
+import type { WorkTreeEntry } from "./work-folder-transport.js";
+
+const MAX_BATCH_BYTES = 1024 * 1024;
+const MAX_BATCH_ENTRIES = 64;
+const MAX_CACHED_BATCHES = 4;
+
+/**
+ * A checkpoint-local cache, not a snapshot or a retry source. The caller limits
+ * concurrent readers to four. At most four 1 MiB batches stay cached; evicted
+ * batches held by those active readers can add at most another 4 MiB. Larger
+ * files use the separately bounded streaming fallback. Metadata is O(entries).
+ */
+export function createWorkFolderReadCache(
+  entries: WorkTreeEntry[],
+  load: (entries: WorkTreeEntry[]) => Promise<Buffer[]>,
+  fallback: (entry: WorkTreeEntry) => Readable,
+) {
+  const locations = new Map<string, { batch: WorkTreeEntry[]; index: number }>();
+  let batch: WorkTreeEntry[] = [];
+  let batchBytes = 0;
+  for (const entry of entries) {
+    if (entry.kind !== "file" || entry.linkTarget || entry.byteSize > MAX_BATCH_BYTES) continue;
+    if (batch.length >= MAX_BATCH_ENTRIES || batchBytes + entry.byteSize > MAX_BATCH_BYTES) {
+      batch = []; batchBytes = 0;
+    }
+    locations.set(entry.path, { batch, index: batch.length });
+    batch.push(entry); batchBytes += entry.byteSize;
+  }
+  const cached = new Map<WorkTreeEntry[], Promise<Buffer[]>>();
+  const readPaths = new Set<string>();
+  let cleared = false;
+
+  function getBatch(group: WorkTreeEntry[]) {
+    let result = cached.get(group);
+    if (result) {
+      cached.delete(group); cached.set(group, result);
+      return result;
+    }
+    result = Promise.resolve().then(() => load(group)).then((buffers) => {
+      if (buffers.length !== group.length || buffers.some((buffer, index) =>
+        !Buffer.isBuffer(buffer) || buffer.length !== group[index]!.byteSize)) {
+        throw new Error("Work folder batch content does not match its entries");
+      }
+      return buffers;
+    }).catch((error: unknown) => {
+      if (cached.get(group) === result) cached.delete(group);
+      throw error;
+    });
+    cached.set(group, result);
+    while (cached.size > MAX_CACHED_BATCHES) cached.delete(cached.keys().next().value!);
+    return result;
+  }
+
+  function read(entry: WorkTreeEntry): Readable {
+    const repeated = readPaths.has(entry.path);
+    readPaths.add(entry.path);
+    const location = locations.get(entry.path);
+    // Mark each source request, even if its stream is never consumed. A retry
+    // must reopen the physical file instead of replaying possibly stale bytes.
+    return Readable.from((async function* () {
+      if (cleared || repeated || !location) {
+        const source = fallback(entry);
+        try { yield* source; } finally { source.destroy(); }
+        return;
+      }
+      const buffers = await getBatch(location.batch);
+      yield buffers[location.index]!;
+    })());
+  }
+  function clear() {
+    cleared = true;
+    cached.clear();
+    locations.clear();
+    readPaths.clear();
+  }
+  return { read, clear };
+}

server/src/services/work-folder-repositories.ts

@@ -1,3 +1,5 @@
+import { createWorkFolderReadCache } from "./work-folder-read-cache.js";
+import { prefetchWorkFiles } from "./work-folder-transfer.js";
 import { createHash, randomUUID } from "node:crypto";
 import { and, eq, inArray, isNull } from "drizzle-orm";
 import { taskRepositoryBindings, workFolderObjects, type Db } from "@paperclipai/db";
@@ -35,17 +37,47 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
     const prefix = `${binding.companyId}/task-repositories/${binding.id}/`;
     if (!knownByBinding.has(binding.id) && binding.checkpointKey) await loadManifest(binding);
     const known = knownByBinding.get(binding.id) ?? new Set<string>();
-    const files: Array<WorkTreeEntry & { objectKey: string | null }> = [];
-    for (const entry of entries) {
-      const objectKey = entry.kind === "file" && !entry.linkTarget ? `${prefix}blobs/${entry.sha256}` : null;
-      if (objectKey && !known.has(objectKey)) await registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id });
-      if (objectKey && !known.has(objectKey) && !(await storage.headObject({ objectKey })).exists) {
-        await uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream",
-          contentLength: entry.byteSize, sha256: entry.sha256!,
-          createSource: () => transport.read(root, entry.path, entry.byteSize) });
+    // Keep manifest order independent of transfer completion, and send each
+    // content-addressed blob only once even when multiple paths share bytes.
+    const files = entries.map((entry) => ({ ...entry,
+      objectKey: entry.kind === "file" && !entry.linkTarget ? `${prefix}blobs/${entry.sha256}` : null,
+    }));
+    const unknown = new Map<string, WorkTreeEntry>();
+    for (const entry of files) {
+      if (entry.objectKey && !known.has(entry.objectKey) && !unknown.has(entry.objectKey)) {
+        unknown.set(entry.objectKey, entry);
       }
-      files.push({ ...entry, objectKey });
     }
+    const objects = [...unknown];
+    const readCache = transport.readBatch ? createWorkFolderReadCache([...unknown.values()],
+      (entries) => transport.readBatch!(root, entries),
+      (entry) => transport.read(root, entry.path, entry.byteSize)) : undefined;
+    let next = 0;
+    let failure: { error: unknown } | undefined;
+    try {
+      await Promise.all(Array.from({ length: Math.min(4, objects.length) }, async () => {
+        while (!failure) {
+          const object = objects[next++];
+          if (!object) return;
+          const [objectKey, entry] = object;
+          try {
+            await registerWorkFolderObject(db, storage, { objectKey, companyId: binding.companyId, repositoryBindingId: binding.id });
+            if (failure) return;
+            const { exists } = await storage.headObject({ objectKey });
+            if (!exists && !failure) {
+              await uploadWorkFolderObject(storage, { objectKey, contentType: "application/octet-stream",
+                contentLength: entry.byteSize, sha256: entry.sha256!,
+                createSource: () => readCache ? readCache.read(entry) : transport.read(root, entry.path, entry.byteSize) });
+            }
+          } catch (error) {
+            // Stop scheduling after the first error, but drain the other workers
+            // before returning. Their streaming PUTs must not outlive this save.
+            failure ??= { error };
+          }
+        }
+      }));
+    } finally { readCache?.clear(); }
+    if (failure) throw failure.error;
     // Never publish a torn Git index/worktree snapshot as a completed save.
     if (signature(await transport.scan(root, true)) !== digest) throw new Error("Repository changed during checkpoint; retry required");
     const checkpointKey = `${prefix}checkpoints/${randomUUID()}.json`;
@@ -109,14 +141,17 @@ export function workFolderRepositoryService(db: Db, storage: StorageProvider, tr
     if (!binding.checkpointKey) return false;
     const manifest = await loadManifest(binding);
     await transport.mkdirRoot(root);
-    for (const entry of manifest.files) {
-      if (entry.kind === "directory") await transport.mkdir(root, entry.path);
-      else if (entry.linkTarget) await transport.symlink(root, stagingRoot, entry);
-      else {
+    await transport.writeMany(root, stagingRoot, prefetchWorkFiles(
+      manifest.files.filter((entry) => !entry.linkTarget), async (entry) => {
+        if (entry.kind === "directory") return { entry };
         if (!entry.objectKey) throw new Error("Repository checkpoint file is missing");
         const result = await storage.getObject({ objectKey: entry.objectKey });
-        try { await transport.write(root, stagingRoot, entry, result.stream); } finally { result.stream.destroy(); }
-      }
+        return { entry, body: result.stream };
+      }));
+    // Restore links only after ordinary files. No transfer follows a link as
+    // a parent, and symlink() still confines its target to this repository.
+    for (const entry of manifest.files) {
+      if (entry.linkTarget) await transport.symlink(root, stagingRoot, entry);
     }
     return true;
   }

server/src/services/work-folder-transfer.ts

@@ -0,0 +1,36 @@
+import type { WorkFileTransfer } from "./work-folder-transport.js";
+
+// Open a few storage responses ahead without buffering their bodies. Drain all
+// pending opens on failure so abandoned HTTP response streams are closed too.
+export async function* prefetchWorkFiles<T>(
+  entries: Iterable<T>,
+  open: (entry: T) => Promise<WorkFileTransfer>,
+): AsyncGenerator<WorkFileTransfer> {
+  type Result = { value: WorkFileTransfer } | { error: unknown };
+  const iterator = entries[Symbol.iterator]();
+  const pending: Array<Promise<Result>> = [];
+  function enqueue() {
+    const next = iterator.next();
+    if (!next.done) pending.push(Promise.resolve().then(() => open(next.value))
+      .then((value): Result => {
+        // A response can fail while queued, before its async iterator exists.
+        // Keep that error handled; consuming the stream still throws it.
+        value.body?.on("error", () => {});
+        return { value };
+      }, (error): Result => ({ error })));
+  }
+  try {
+    for (let i = 0; i < 4; i++) enqueue();
+    while (pending.length) {
+      const result = await pending.shift()!;
+      if ("error" in result) throw result.error;
+      try { yield result.value; } finally { result.value.body?.destroy(); }
+      enqueue();
+    }
+  } finally {
+    for (const result of await Promise.all(pending)) {
+      if ("value" in result) result.value.body?.destroy();
+    }
+    iterator.return?.();
+  }
+}

server/src/services/work-folder-transport.ts

@@ -15,6 +15,9 @@ let source: Promise<string> | undefined;
 // Requests are base64 encoded twice (file bytes, then JSON). Stay below
 // Linux's 128 KiB single-argument limit, including a provider shell wrapper.
 const WRITE_CHUNK_BYTES = 48 * 1024;
+const BATCH_BYTES = 4 * 1024 * 1024;
+const BATCH_OPERATIONS = 256;
+export type WorkFileTransfer = { entry: WorkTreeEntry; body?: Readable };
 
 function transientReadFailure(error: unknown) {
   if (!(error instanceof Error)) return false;
@@ -24,15 +27,19 @@ function transientReadFailure(error: unknown) {
 }
 
 export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
-  async function command(input: Record<string, unknown>): Promise<unknown> {
+  // Native file-sync providers already support bounded stdin uploads. SSH
+  // runners explicitly advertise streaming stdin. Other providers retain the
+  // small-argv transport without assuming additional capabilities.
+  const bulkStdin = Boolean(runner.syncIn && runner.syncOut) || runner.supportsSingleStreamStdinProgress === true;
+  async function command(input: Record<string, unknown>, stdin?: string): Promise<unknown> {
     source ??= readFile(new URL("./scripts/work-folder-io.mjs", import.meta.url), "utf8");
     const args = ["--input-type=module", "-e", await source, Buffer.from(JSON.stringify(input)).toString("base64")];
-    const readOnly = ["home", "scan", "read"].includes(String(input.operation));
+    const readOnly = ["home", "scan", "read", "read-batch"].includes(String(input.operation));
     const deadline = Date.now() + 120_000;
     let result;
     for (let attempt = 0; ; attempt++) {
       try {
-        result = await runner.execute({ command: "node", args, bypassSession: true,
+        result = await runner.execute({ command: "node", args, ...(stdin === undefined ? {} : { stdin }), bypassSession: true,
           timeoutMs: Math.max(1, deadline - Date.now()) });
         break;
       } catch (error) {
@@ -57,7 +64,8 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
     validateWorkFilePath(filePath);
     return Readable.from((async function* () {
       for (let offset = 0; offset < byteSize;) {
-        const result = z.object({ data: z.string().max(350_000) }).parse(await command({ operation: "read", root, path: filePath, offset }));
+        const length = bulkStdin ? 1024 * 1024 : 256 * 1024;
+        const result = z.object({ data: z.string().max(Math.ceil(length / 3) * 4) }).parse(await command({ operation: "read", root, path: filePath, offset, length }));
         const bytes = Buffer.from(result.data, "base64");
         if (bytes.length === 0 || offset + bytes.length > byteSize) throw new Error("Work file changed during transfer");
         offset += bytes.length;
@@ -65,11 +73,30 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
       }
     })());
   }
-  async function write(root: string, stagingRoot: string, entry: WorkTreeEntry, body: Readable) {
+  async function readBatch(root: string, entries: WorkTreeEntry[]) {
+    if (entries.length > 64) throw new Error("Work folder read batch exceeds entry limit");
+    let bytes = 0;
+    for (const entry of entries) {
+      entrySchema.parse(entry);
+      if (entry.kind !== "file" || entry.linkTarget) throw new Error("Work folder read batch requires regular files");
+      bytes += entry.byteSize;
+    }
+    if (bytes > 1024 * 1024) throw new Error("Work folder read batch exceeds byte limit");
+    if (!entries.length) return [];
+    const result = z.array(z.object({ data: z.string().max(Math.ceil(1024 * 1024 / 3) * 4) })).max(64)
+      .parse(await command({ operation: "read-batch", root, entries: entries.map(({ path, byteSize }) => ({ path, byteSize })) }));
+    if (result.length !== entries.length) throw new Error("Work folder read batch did not complete");
+    return result.map(({ data }, index) => {
+      const buffer = Buffer.from(data, "base64");
+      if (buffer.length !== entries[index]!.byteSize) throw new Error("Work file changed during transfer");
+      return buffer;
+    });
+  }
+  async function writeChunks(root: string, stagingRoot: string, entry: WorkTreeEntry, body: Readable) {
     const stagingPath = randomUUID();
     let offset = 0;
     for await (const value of body) {
-      const chunk = Buffer.from(value);
+      const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
       for (let start = 0; start < chunk.length; start += WRITE_CHUNK_BYTES) {
         const bytes = chunk.subarray(start, start + WRITE_CHUNK_BYTES);
         await command({ operation: "write", root: stagingRoot, path: stagingPath, offset, data: bytes.toString("base64") });
@@ -81,7 +108,66 @@ export function workFolderTransport(runner: CommandManagedRuntimeRunner) {
     await command({ operation: "publish", root, stagingRoot, stagingPath, path: entry.path,
       sha256: entry.sha256, executable: entry.executable });
   }
-  return { home, scan, read, write,
+  async function writeMany(root: string, stagingRoot: string, transfers: AsyncIterable<WorkFileTransfer>,
+    beforePublish?: (entries: WorkTreeEntry[]) => Promise<void>) {
+    let publishedEntries: WorkTreeEntry[] = [];
+    let operations: Array<Record<string, unknown>> = [];
+    let bufferedBytes = 0;
+    async function flush() {
+      if (!operations.length) return;
+      const body = JSON.stringify(operations);
+      if (Buffer.byteLength(body) > 8 * 1024 * 1024) throw new Error("Work folder batch exceeds transfer limit");
+      if (publishedEntries.length) await beforePublish?.(publishedEntries);
+      const result = z.object({ completed: z.number().int() }).parse(await command({ operation: "batch", root, stagingRoot }, body));
+      if (result.completed !== operations.length) throw new Error("Work folder batch did not complete");
+      operations = []; bufferedBytes = 0; publishedEntries = [];
+    }
+    async function append(operation: Record<string, unknown>, bytes = 0, entry?: WorkTreeEntry) {
+      if (bufferedBytes + bytes > BATCH_BYTES || operations.length >= BATCH_OPERATIONS) await flush();
+      operations.push(operation); bufferedBytes += bytes;
+      if (entry) publishedEntries.push({ ...entry });
+    }
+    for await (const { entry, body } of transfers) {
+      try {
+        entrySchema.parse(entry);
+        if (entry.linkTarget) throw new Error("Work folder batch cannot materialize symlinks");
+        if (entry.kind === "directory") {
+          if (body) throw new Error("Directory transfer cannot have a body");
+          if (bulkStdin) await append({ operation: "mkdir", path: entry.path }, 0, entry);
+          else {
+            await beforePublish?.([entry]);
+            await command({ operation: "mkdir", root, path: entry.path });
+          }
+          continue;
+        }
+        if (!body) throw new Error("Work file transfer source is missing");
+        if (!bulkStdin) {
+          await beforePublish?.([entry]);
+          await writeChunks(root, stagingRoot, entry, body);
+          continue;
+        }
+        const stagingPath = randomUUID();
+        let offset = 0;
+        for await (const value of body) {
+          const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
+          for (let start = 0; start < chunk.length; start += 256 * 1024) {
+            const bytes = chunk.subarray(start, start + 256 * 1024);
+            if (offset + bytes.length > entry.byteSize) throw new Error("Work file size changed during transfer");
+            await append({ operation: "write", path: stagingPath, offset, data: bytes.toString("base64") }, bytes.length);
+            offset += bytes.length;
+          }
+        }
+        if (offset === 0) await append({ operation: "write", path: stagingPath, offset: 0, data: "" });
+        if (offset !== entry.byteSize) throw new Error("Work file size changed during transfer");
+        await append({ operation: "publish", stagingPath, path: entry.path, sha256: entry.sha256, executable: entry.executable }, 0, entry);
+      } finally { body?.destroy(); }
+    }
+    if (bulkStdin) await flush();
+  }
+  async function write(root: string, stagingRoot: string, entry: WorkTreeEntry, body: Readable) {
+    return writeMany(root, stagingRoot, (async function* () { yield { entry, body }; })());
+  }
+  return { home, scan, read, readBatch: bulkStdin ? readBatch : undefined, write, writeMany,
     moveRoot: async (source: string, root: string) => { await command({ operation: "move-root", source, root }); },
     symlink: async (root: string, stagingRoot: string, entry: WorkTreeEntry) => { await command({ operation: "symlink", root, stagingRoot, stagingPath: randomUUID(), path: entry.path, linkTarget: entry.linkTarget }); },
     mkdirRoot: async (root: string) => { await command({ operation: "mkdir-root", root }); },