From cebfa8b09f388fa6172b25e554ff7d600d1bc817 Mon Sep 17 00:00:00 2001 From: Dotta Date: Sat, 10 Oct 2026 10:59:25 -0500 Subject: [PATCH] Bound initial computer home uploads and exclude runtime state Stream initial personal files through bounded chunks into private staging and atomically publish without replacing an existing home. Keep the home-level runtime directory out of instruction enumeration and seeding while preserving nested user content. Co-Authored-By: Paperclip --- .../agent-directory-working-copies.test.ts | 14 ++-- .../agent-instructions-service.test.ts | 21 ++++++ .../modules/computers/adapters/boat.test.ts | 37 ++++++++++- .../computers/adapters/remote-program.ts | 64 +++++++++++++++++++ .../computers/application/service.test.ts | 26 ++++++++ .../modules/computers/application/service.ts | 47 +++++++------- server/src/services/agent-instructions.ts | 3 + server/src/services/persistent-agent-files.ts | 24 ++++++- 8 files changed, 204 insertions(+), 32 deletions(-) diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 0f86e1b18a..6f27e64403 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -78,11 +78,15 @@ describe("persistent agent directories", () => { it("seeds a persistent execution home before a run or instruction-copy receipt exists", async () => { const remoteRoot = "/home/user/paperclip/test/agents/target"; let exists = false; - const seedBytes = vi.fn(async () => { exists = true; }); + const received: Record = {}; + const seedFiles = vi.fn(async (chunks: AsyncIterable<{ path: string; bytes: Buffer }>) => { + for await (const chunk of chunks) received[chunk.path] = Buffer.concat([received[chunk.path] ?? Buffer.alloc(0), chunk.bytes]); + exists = true; + }); const remote = { root: remoteRoot, listPage: async () => { if (!exists) throw { code: "not_found" }; return { entries: [], truncated: false }; - }, seedBytes }; + }, seedFiles }; const lookup = vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue(remote as never); const environmentId = randomUUID(); const executionTarget = { kind: "remote", transport: "computer", environmentId, @@ -91,14 +95,14 @@ describe("persistent agent directories", () => { await fs.writeFile(path.join(root, "memory.bin"), Buffer.from([0, 255, 7])); expect(await preparePersistentAgentExecutionHome(db, { companyId, agentId, target: executionTarget })).toBe(remoteRoot); expect(lookup).toHaveBeenCalledWith(db, companyId, agentId, environmentId); - expect(seedBytes).toHaveBeenCalledOnce(); - expect(seedBytes).toHaveBeenCalledWith(expect.objectContaining({ + expect(seedFiles).toHaveBeenCalledOnce(); + expect(received).toEqual(expect.objectContaining({ [entryFile]: Buffer.from(initial), "memory.bin": Buffer.from([0, 255, 7]), })); // Subsequent setup adopts the remote tree, regardless of controller edits. await fs.writeFile(path.join(root, "memory.bin"), "stale controller bytes"); await preparePersistentAgentExecutionHome(db, { companyId, agentId, target: executionTarget }); - expect(seedBytes).toHaveBeenCalledOnce(); + expect(seedFiles).toHaveBeenCalledOnce(); expect(await db.select().from(agentInstructionWorkingCopies).where(eq(agentInstructionWorkingCopies.agentId, agentId))).toEqual([]); } finally { lookup.mockRestore(); } }); diff --git a/server/src/__tests__/agent-instructions-service.test.ts b/server/src/__tests__/agent-instructions-service.test.ts index 9c845c3926..e7727c8aa7 100644 --- a/server/src/__tests__/agent-instructions-service.test.ts +++ b/server/src/__tests__/agent-instructions-service.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as fileStore from "../services/agent-file-store.js"; import * as persistentFiles from "../services/persistent-agent-files.js"; import { agentInstructionsService } from "../services/agent-instructions.js"; @@ -82,6 +83,26 @@ describe("agent instructions service", () => { expect(db.transaction).not.toHaveBeenCalled(); }); + it("excludes only home-level runtime files from remote listing and export", async () => { + const agent = makeAgent({ instructionsBundleMode: "managed" }); + const list = vi.fn(async (relative = "") => { + if (relative === ".paperclip-runtime") throw new Error("Must not enumerate provider packs or session state"); + if (!relative) return [{ name: "AGENTS.md", kind: "file", size: 12 }, { name: ".paperclip-runtime", kind: "directory", size: 0 }, { name: "notes", kind: "directory", size: 0 }]; + if (relative === "notes") return [{ name: ".paperclip-runtime", kind: "directory", size: 0 }]; + return [{ name: "user.md", kind: "file", size: 12 }]; + }); + const readBytes = vi.fn(async () => ({ bytes: Buffer.from("instructions"), sha256: "hash" })); + vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue({ root: "/remote/home", list, readBytes } as never); + vi.spyOn(persistentFiles, "seedPersistentAgentHome").mockResolvedValue(undefined); + vi.spyOn(fileStore, "adoptAgentFiles").mockResolvedValue("/controller/home"); + const db = { transaction: async (fn: (tx: unknown) => Promise) => fn({ select: () => ({ from: () => ({ where: async () => [agent] }) }) }) }; + const svc = agentInstructionsService(db as never); + expect((await svc.getBundle(agent)).files.map(file => file.path)).toEqual(["AGENTS.md", "notes/.paperclip-runtime/user.md"]); + expect((await svc.exportFiles(agent)).files).toEqual({ "AGENTS.md": "instructions", "notes/.paperclip-runtime/user.md": "instructions" }); + expect(list).not.toHaveBeenCalledWith(".paperclip-runtime"); + expect(readBytes).not.toHaveBeenCalledWith(expect.stringMatching(/^\.paperclip-runtime\//)); + }); + it("previews an explicit external-to-managed migration before adopting remote personal files", async () => { const root = await makeTempDir("boat-instructions-migration-"); cleanupDirs.add(root); await fs.writeFile(path.join(root, "AGENTS.md"), "external migration source"); diff --git a/server/src/modules/computers/adapters/boat.test.ts b/server/src/modules/computers/adapters/boat.test.ts index 22593d9e0b..13e8529854 100644 --- a/server/src/modules/computers/adapters/boat.test.ts +++ b/server/src/modules/computers/adapters/boat.test.ts @@ -1,14 +1,16 @@ import { existsSync, - mkdtempSync, readFileSync, + readdirSync, + statSync, + mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync, mkdirSync, } from "node:fs"; -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { spawnSync } from "node:child_process"; @@ -172,6 +174,37 @@ describe("confined computer files", () => { error: "invalid", }); }); + it("publishes bounded seed chunks atomically and aborts partial homes", () => { + const f = fixture(); const seedId = randomUUID(); + expect(f.call({ action: "seed-begin", seedId })).toEqual({ started: true }); + const chunk = Buffer.alloc(1024 * 1024, 19); + for (const offset of [0, chunk.length]) expect(f.call({ action: "seed-chunk", seedId, path: "memory/data", offset, base64: chunk.toString("base64") })).toEqual({}); + expect(existsSync(f.root)).toBe(false); + const staging = readdirSync(f.temp).find(name => name.startsWith(".paperclip-seed-"))!; + expect(statSync(join(f.temp, staging)).mode & 0o777).toBe(0o700); + expect(statSync(join(f.temp, staging, "tree", "memory", "data")).mode & 0o777).toBe(0o600); + expect(f.call({ action: "seed-chunk", seedId, path: "memory/data", offset: 0, base64: "" })).toEqual({ error: "conflict" }); + expect(f.call({ action: "seed-commit", seedId })).toEqual({ seeded: true }); + expect(readFileSync(join(f.root, "memory", "data"))).toEqual(Buffer.concat([chunk, chunk])); + expect(readdirSync(f.temp)).toEqual(["home"]); + expect(f.call({ action: "seed-begin", seedId: randomUUID() })).toEqual({ started: false }); + const g = fixture(); const partial = randomUUID(); + g.call({ action: "seed-begin", seedId: partial }); + expect(g.call({ action: "seed-chunk", seedId: partial, path: "../escape", offset: 0, base64: "" })).toEqual({ error: "invalid" }); + expect(g.call({ action: "seed-chunk", seedId: partial, path: "large", offset: 0, base64: Buffer.alloc(1024 * 1024 + 1).toString("base64") })).toEqual({ error: "invalid" }); + expect(g.call({ action: "seed-abort", seedId: partial })).toEqual({}); + expect(existsSync(g.root)).toBe(false); + expect(readdirSync(g.temp)).toEqual([]); + }); + it("does not replace a home created while its initial upload was in progress", () => { + const f = fixture(); const seedId = randomUUID(); + f.call({ action: "seed-begin", seedId }); + f.call({ action: "seed-chunk", seedId, path: "initial", offset: 0, base64: Buffer.from("staged").toString("base64") }); + mkdirSync(f.root); + expect(f.call({ action: "seed-commit", seedId })).toEqual({ seeded: false }); + expect(readdirSync(f.root)).toEqual([]); + expect(readdirSync(f.temp)).toEqual(["home"]); + }); it("hashes and deletes files over the read limit while fencing changed bytes", () => { const f = fixture(); f.call({ action: "seed", files: {} }); diff --git a/server/src/modules/computers/adapters/remote-program.ts b/server/src/modules/computers/adapters/remote-program.ts index 709727c5c4..f5b37d0391 100644 --- a/server/src/modules/computers/adapters/remote-program.ts +++ b/server/src/modules/computers/adapters/remote-program.ts @@ -107,6 +107,70 @@ def directory(parts,create=False,start=None): os.close(fd);fd=nxt return fd except BaseException:os.close(fd);raise +# Initial uploads stay in a private sibling until a no-replace rename publishes them. +# Each request carries at most one bounded chunk, never the whole home. +if act in ('seed-begin','seed-chunk','seed-commit','seed-abort'): + import uuid,ctypes,errno + token=p.get('seedId','') + try: + if str(uuid.UUID(token))!=token:fail('invalid') + except (ValueError,TypeError,AttributeError):fail('invalid') + parentfd=directory(os.path.dirname(root).split('/'),create=True) + name=os.path.basename(root);staging='.paperclip-seed-'+name+'-'+token + stagefd=None;lockfd=None + try: + if act=='seed-abort': + try:shutil.rmtree(staging,dir_fd=parentfd) + except FileNotFoundError:pass + print('{}');sys.exit(0) + if act=='seed-begin': + try: + existing=os.stat(name,dir_fd=parentfd,follow_symlinks=False) + if not stat.S_ISDIR(existing.st_mode):fail('invalid') + print(json.dumps({'started':False}));sys.exit(0) + except FileNotFoundError:pass + os.mkdir(staging,0o700,dir_fd=parentfd) + stagefd=os.open(staging,os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW,dir_fd=parentfd) + lockfd=os.open('lock',os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600,dir_fd=stagefd) + fcntl.flock(lockfd,fcntl.LOCK_EX) + if act=='seed-begin': + os.mkdir('tree',0o700,dir_fd=stagefd) + print(json.dumps({'started':True})) + elif act=='seed-chunk': + relative=p.get('path','');safe(relative) + parts=[part for part in relative.split('/') if part not in ('','.')] + if not parts:fail('invalid') + data=base64.b64decode(p['base64'],validate=True);offset=p.get('offset') + if len(data)>1024*1024 or type(offset)!=int or offset<0 or offset+len(data)>256*1024*1024:fail('invalid') + destfd=directory(['tree']+parts[:-1],create=True,start=stagefd) + try:fd=os.open(parts[-1],os.O_WRONLY|os.O_CREAT|os.O_NOFOLLOW,0o600,dir_fd=destfd) + finally:os.close(destfd) + with os.fdopen(fd,'r+b') as f: + info=os.fstat(f.fileno()) + if not stat.S_ISREG(info.st_mode) or info.st_nlink!=1:fail('invalid') + if info.st_size!=offset:fail('conflict') + f.seek(offset);f.write(data) + print('{}') + else: + libc=ctypes.CDLL(None,use_errno=True) + if hasattr(libc,'renameat2'):status=libc.renameat2(stagefd,b'tree',parentfd,os.fsencode(name),1) + elif hasattr(libc,'renameatx_np'):status=libc.renameatx_np(stagefd,b'tree',parentfd,os.fsencode(name),4) + else:fail('invalid') + if status!=0: + code=ctypes.get_errno() + if code not in (errno.EEXIST,errno.ENOTEMPTY):raise OSError(code,os.strerror(code)) + if not stat.S_ISDIR(os.stat(name,dir_fd=parentfd,follow_symlinks=False).st_mode):fail('invalid') + print(json.dumps({'seeded':status==0})) + except FileNotFoundError:fail('not_found') + except (OSError,ValueError):fail('invalid') + finally: + if lockfd is not None:os.close(lockfd) + if stagefd is not None:os.close(stagefd) + if act=='seed-commit': + try:shutil.rmtree(staging,dir_fd=parentfd) + except FileNotFoundError:pass + os.close(parentfd) + sys.exit(0) try:rootfd=directory(root.split('/')) except FileNotFoundError:fail('not_found') except OSError:fail('invalid') diff --git a/server/src/modules/computers/application/service.test.ts b/server/src/modules/computers/application/service.test.ts index 3057464183..ac41da391d 100644 --- a/server/src/modules/computers/application/service.test.ts +++ b/server/src/modules/computers/application/service.test.ts @@ -545,6 +545,32 @@ with open(output,'wb') as out,open(error,'wb') as err: rmSync(temp, { recursive: true, force: true }); } }); + it("bounds initial seed requests and aborts a failed upload without publishing", async () => { + const f = fixture(); + await f.attach(); + const files = await f.service.files({ ...f.scope, agentId: "agent" }); + vi.mocked(f.backend.remote).mockImplementation(async (_record, input) => { + if (input.action === "seed-begin") return { started: true }; + if (input.action === "seed-commit") return { seeded: true }; + return {}; + }); + await expect(files.seedBytes({ "large.bin": Buffer.alloc(2 * 1024 * 1024 + 1) })).resolves.toEqual({ seeded: true }); + const chunks = vi.mocked(f.backend.remote).mock.calls.map(([, input]) => input).filter(input => input.action === "seed-chunk"); + expect(chunks).toHaveLength(3); + expect(chunks.map(input => input.offset)).toEqual([0, 1024 * 1024, 2 * 1024 * 1024]); + expect(chunks.every(input => Buffer.from(input.base64 as string, "base64").length <= 1024 * 1024)).toBe(true); + vi.mocked(f.backend.remote).mockClear(); + await expect(files.seedFiles((async function* () { + yield { path: "first", offset: 0, bytes: Buffer.from("first") }; + throw new Error("source disappeared"); + })())).rejects.toThrow("source disappeared"); + const actions = vi.mocked(f.backend.remote).mock.calls.map(([, input]) => input.action); + expect(actions).toEqual(["seed-begin", "seed-chunk", "seed-abort"]); + vi.mocked(f.backend.remote).mockResolvedValue({ started: false }); + let read = false; + await expect(files.seedFiles((async function* () { read = true; yield { path: "unused", offset: 0, bytes: Buffer.alloc(0) }; })())).resolves.toEqual({ seeded: false }); + expect(read).toBe(false); + }); it("realizes probe cwd before readiness without consuming the initial personal-home seed", async () => { const f = fixture(); const temp = realpathSync(mkdtempSync(join(tmpdir(), "computer-probe-"))); diff --git a/server/src/modules/computers/application/service.ts b/server/src/modules/computers/application/service.ts index 3e6a226420..6ef886c7f5 100644 --- a/server/src/modules/computers/application/service.ts +++ b/server/src/modules/computers/application/service.ts @@ -687,9 +687,33 @@ finally: base64: bytes.toString("base64"), expectedSha256, }); + const seedFiles = async (chunks: AsyncIterable<{ path: string; offset: number; bytes: Buffer }>): Promise<{ seeded: boolean }> => { + const seedId = randomUUID(); + const started = await remote({ action: "seed-begin", seedId }); + if (!started.started) return { seeded: false }; + try { + for await (const chunk of chunks) { + if (chunk.bytes.length > 1024 * 1024) throw new ComputerError("invalid", "Initial home chunk exceeds its size limit"); + await remote({ action: "seed-chunk", seedId, path: chunk.path, offset: chunk.offset, base64: chunk.bytes.toString("base64") }); + } + return await remote({ action: "seed-commit", seedId }); + } catch (error) { + await remote({ action: "seed-abort", seedId }).catch(() => undefined); + throw error; + } + }; + const seedBytes = async (files: Record): Promise<{ seeded: boolean }> => seedFiles((async function* () { + for (const [path, bytes] of Object.entries(files)) { + for (let offset = 0; offset < Math.max(1, bytes.length); offset += 1024 * 1024) { + yield { path, offset, bytes: bytes.subarray(offset, offset + 1024 * 1024) }; + } + } + })()); return { root, readBytes, + seedFiles, + seedBytes, writeBytes, async read(path: string) { const value = await readBytes(path); @@ -729,29 +753,8 @@ finally: ): Promise<{ sha256: string }> { return remote({ action: "move", path: from, to, expectedSha256 }); }, - async seedBytes( - files: Record, - ): Promise<{ seeded: boolean }> { - return remote({ - action: "seed", - files: Object.fromEntries( - Object.entries(files).map(([key, value]) => [ - key, - value.toString("base64"), - ]), - ), - }); - }, async seed(files: Record): Promise<{ seeded: boolean }> { - return remote({ - action: "seed", - files: Object.fromEntries( - Object.entries(files).map(([key, value]) => [ - key, - Buffer.from(value).toString("base64"), - ]), - ), - }); + return seedBytes(Object.fromEntries(Object.entries(files).map(([path, content]) => [path, Buffer.from(content)]))); }, }; } diff --git a/server/src/services/agent-instructions.ts b/server/src/services/agent-instructions.ts index 138d24a5e0..a571dd8aea 100644 --- a/server/src/services/agent-instructions.ts +++ b/server/src/services/agent-instructions.ts @@ -513,6 +513,9 @@ export function agentInstructionsService(db?: Db) { const result: Array<{ path: string; size: number }> = []; for (const entry of await remote.list(relative || undefined)) { const name = relative ? `${relative}/${entry.name}` : entry.name; + // Only the home-level directory belongs to the runtime; nested names can + // be ordinary user content and retain their existing listing behavior. + if (!relative && entry.name === ".paperclip-runtime") continue; instructionPath(name); if (entry.kind === "directory") { if (!IGNORED_INSTRUCTIONS_DIRECTORY_NAMES.has(entry.name)) result.push(...await remotePaths(remote, name)); diff --git a/server/src/services/persistent-agent-files.ts b/server/src/services/persistent-agent-files.ts index 0f5966dfe3..e47d095f2c 100644 --- a/server/src/services/persistent-agent-files.ts +++ b/server/src/services/persistent-agent-files.ts @@ -1,4 +1,5 @@ import fs from "node:fs/promises"; +import { constants } from "node:fs"; import path from "node:path"; import { and, eq } from "drizzle-orm"; import { agents, environments, instanceSettings, type Db } from "@paperclipai/db"; @@ -49,7 +50,7 @@ export async function hashPersistentAgentFile(files: NonNullable>>, localRoot: string) { try { await files.listPage("", { limit: 1 }); return; } catch (error) { if (!isMissingRemoteFile(error)) throw error; } - const seed: Record = {}; + const initialFiles: Array<{ name: string; size: number; dev: number; ino: number }> = []; let total = 0, count = 0; async function walk(relative: string) { const entries = await fs.readdir(path.join(localRoot, relative), { withFileTypes: true }).catch((error: NodeJS.ErrnoException) => { @@ -58,6 +59,7 @@ export async function seedPersistentAgentHome(files: NonNullable 100_000) throw new Error("Initial agent folder exceeds its entry limit"); const stat = await fs.lstat(path.join(localRoot, name)); @@ -65,12 +67,28 @@ export async function seedPersistentAgentHome(files: NonNullable 256 * 1024 * 1024 || total > 2 * 1024 * 1024 * 1024) throw new Error("Initial agent folder exceeds its storage limit"); - seed[name] = await fs.readFile(path.join(localRoot, name)); + initialFiles.push({ name, size: stat.size, dev: stat.dev, ino: stat.ino }); } else throw new Error("Agent folders cannot contain links or special files"); } } await walk(""); - await files.seedBytes(seed); + await files.seedFiles((async function* () { + const buffer = Buffer.allocUnsafe(1024 * 1024); + for (const file of initialFiles) { + const handle = await fs.open(path.join(localRoot, file.name), constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const stat = await handle.stat(); + if (!stat.isFile() || stat.nlink !== 1 || stat.dev !== file.dev || stat.ino !== file.ino || stat.size !== file.size) throw new Error("Initial agent file changed during setup"); + let offset = 0; + do { + const { bytesRead } = await handle.read(buffer, 0, Math.min(buffer.length, file.size - offset), offset); + if (bytesRead === 0 && offset < file.size) throw new Error("Initial agent file changed during setup"); + yield { path: file.name, offset, bytes: buffer.subarray(0, bytesRead) }; + offset += bytesRead; + } while (offset < file.size); + } finally { await handle.close(); } + } + })()); } /** Translate the domain's transport-neutral errors at the HTTP service edge. */