diff --git a/server/src/__tests__/agent-directory-working-copies.test.ts b/server/src/__tests__/agent-directory-working-copies.test.ts index 468f3f9ee5e..1c2f199795d 100644 --- a/server/src/__tests__/agent-directory-working-copies.test.ts +++ b/server/src/__tests__/agent-directory-working-copies.test.ts @@ -79,9 +79,9 @@ describe("persistent agent directories", () => { const remoteRoot = "/home/user/paperclip/test/agents/target"; let exists = false; const seedBytes = vi.fn(async () => { exists = true; }); - const remote = { root: remoteRoot, list: async () => { + const remote = { root: remoteRoot, listPage: async () => { if (!exists) throw { code: "not_found" }; - return []; + return { entries: [], truncated: false }; }, seedBytes }; const lookup = vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue(remote as never); const environmentId = randomUUID(); @@ -129,7 +129,7 @@ describe("persistent agent directories", () => { remoteFiles.set(relative, bytes); return { sha256: fileHash(bytes) }; }); const seedBytes = vi.fn(); - const remote = { root: remoteRoot, list: async () => [], readBytes, writeBytes, seedBytes }; + const remote = { root: remoteRoot, list: async () => [], listPage: async () => ({ entries: [], truncated: false }), readBytes, writeBytes, seedBytes }; const lookup = vi.spyOn(persistentFiles, "persistentAgentFiles").mockResolvedValue(remote as never); const shell = vi.spyOn(executionTargetTools, "runAdapterExecutionTargetShellCommand"); try { diff --git a/server/src/__tests__/file-resources.test.ts b/server/src/__tests__/file-resources.test.ts index 12c32f8e7fd..6d56bc0fed0 100644 --- a/server/src/__tests__/file-resources.test.ts +++ b/server/src/__tests__/file-resources.test.ts @@ -218,18 +218,24 @@ describeEmbeddedPostgres("workspace file resources", () => { .where(eq(executionWorkspaces.sourceIssueId, graph.issueId)); const bytes = Buffer.from("only on Boat"); const readBytes = vi.fn(async () => ({ bytes, sha256: "unused-by-preview" })); - const list = vi.fn(async () => [{ name: "README.md", kind: "file", size: bytes.length, mtimeMs: 1 }, - { name: ".env", kind: "file", size: 1, mtimeMs: 1 }, { name: ".paperclip-runtime", kind: "directory", size: 0, mtimeMs: 1 }]); - const workspaceFiles = vi.fn(async () => ({ root: "/home/user/paperclip/remote/project", list, readBytes })); + const listPage = vi.fn(async () => ({ entries: [{ name: "README.md", kind: "file", size: bytes.length, mtimeMs: 1 }, + { name: ".env", kind: "file", size: 1, mtimeMs: 1 }, { name: ".paperclip-runtime", kind: "directory", size: 0, mtimeMs: 1 }], truncated: true })); + const stat = vi.fn(async (name: string) => ({ name, kind: "file", size: bytes.length, mtimeMs: 1 })); + const workspaceFiles = vi.fn(async () => ({ root: "/home/user/paperclip/remote/project", listPage, stat, readBytes })); const factory = vi.spyOn(computerModule, "computerService").mockReturnValue({ workspaceFiles } as never); try { const service = workspaceFileResourceService(db); const listed = await service.list(graph.issueId, { workspace: "execution" }); expect(listed.items.map(item => item.title)).toEqual(["README.md"]); + expect(listed.truncated).toBe(true); + expect(listPage).toHaveBeenCalledWith(undefined, { limit: 1000 }); expect(workspaceFiles).toHaveBeenCalledWith({ companyId: graph.companyId, environmentId, placementId }); const content = await service.readContent(graph.issueId, { path: "README.md", workspace: "execution" }); expect(content.content).toEqual({ encoding: "utf8", data: "only on Boat" }); expect(readBytes).toHaveBeenCalledWith("README.md", WORKSPACE_FILE_TEXT_MAX_BYTES); + await service.readContent(graph.issueId, { path: "beyond-listing-cap.md", workspace: "execution" }); + expect(stat).toHaveBeenCalledWith("beyond-listing-cap.md"); + expect(listPage).toHaveBeenCalledOnce(); expect(await service.prepareDownload(graph.issueId, { path: "README.md", workspace: "execution" })).toMatchObject({ bytes }); readBytes.mockClear(); await expect(service.readContent(graph.issueId, { path: ".env", workspace: "execution" })).rejects.toMatchObject({ status: 403 }); diff --git a/server/src/modules/computers/adapters/boat.test.ts b/server/src/modules/computers/adapters/boat.test.ts index 5cec21e4066..d6f96f78bd3 100644 --- a/server/src/modules/computers/adapters/boat.test.ts +++ b/server/src/modules/computers/adapters/boat.test.ts @@ -199,13 +199,48 @@ describe("confined computer files", () => { expectedSha256: second.sha256, }), ).toEqual({ sha256: second.sha256 }); - expect(f.call({ action: "list" }).map((e: any) => e.name)).toEqual([ + expect(f.call({ action: "list" }).entries.map((e: any) => e.name)).toEqual([ "other", ]); expect( f.call({ action: "remove", path: "other", expectedSha256: first.sha256 }), ).toEqual({ error: "conflict" }); }); + it("bounds directory enumeration and stats files beyond the listing cap", () => { + const f = fixture(); + f.call({ action: "seed", files: {} }); + for (let i = 0; i < 1005; i++) writeFileSync(join(f.root, `file-${i}`), "content"); + const page = f.call({ action: "list" }); + expect(page.entries).toHaveLength(1000); + expect(page.truncated).toBe(true); + const omitted = Array.from({ length: 1005 }, (_, i) => `file-${i}`).find(name => !page.entries.some((entry: { name: string }) => entry.name === name))!; + expect(f.call({ action: "stat", path: omitted })).toMatchObject({ name: omitted, kind: "file", size: 7 }); + expect(f.call({ action: "list", limit: 2 }).entries).toHaveLength(2); + expect(f.call({ action: "list", limit: 1001 })).toEqual({ error: "invalid" }); + }); + it.each([false, true])("uses the requested worktree branch separately from its start ref (explicit base: %s)", explicitBase => { + const f = fixture(); + const checkout = join(f.root, "checkout"); + mkdirSync(checkout, { recursive: true }); + const git = (...args: string[]) => { + const result = spawnSync("git", args, { cwd: checkout, encoding: "utf8" }); + if (result.status !== 0) throw new Error(result.stderr); + return result.stdout.trim(); + }; + git("init"); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "--allow-empty", "-m", "base"); + const base = git("rev-parse", "HEAD"); + git("-c", "user.name=Test", "-c", "user.email=test@example.com", "commit", "--allow-empty", "-m", "latest"); + const expectedHead = explicitBase ? base : git("rev-parse", "HEAD"); + const input = { action: "workspace", mode: "worktree", taskId: "issue-1", branch: "paperclip/task-issue-1", ...(explicitBase ? { baseRef: base } : {}) }; + const result = f.call(input); + expect(result.remoteCwd).toBe(join(f.root, "tasks", "issue-1")); + expect(git("-C", result.remoteCwd, "symbolic-ref", "--short", "HEAD")).toBe(input.branch); + expect(git("-C", result.remoteCwd, "rev-parse", "HEAD")).toBe(expectedHead); + writeFileSync(join(result.remoteCwd, "preserved"), "user work"); + expect(f.call({ ...input, branch: "another-branch" })).toEqual({ error: "conflict" }); + expect(git("-C", result.remoteCwd, "status", "--short")).toContain("preserved"); + }); it("roundtrips binary bytes without interpreting text", () => { const f = fixture(); const bytes = Buffer.from([0, 255, 254, 13, 10, 128]); diff --git a/server/src/modules/computers/adapters/remote-program.ts b/server/src/modules/computers/adapters/remote-program.ts index 903e2159334..2f658d45fb1 100644 --- a/server/src/modules/computers/adapters/remote-program.ts +++ b/server/src/modules/computers/adapters/remote-program.ts @@ -76,8 +76,10 @@ if act=='workspace': if p.get('mode')=='worktree': if not task:fail('invalid') target=os.path.join(root,'tasks',task);os.makedirs(os.path.dirname(target),exist_ok=True) - if not os.path.exists(target):run(['git','worktree','add','-b','paperclip/'+task,target,base_ref or branch or 'HEAD'],checkout) + task_branch=branch or 'paperclip/task-'+task + if not os.path.exists(target):run(['git','worktree','add','-b',task_branch,target,base_ref or 'HEAD'],checkout) run(['git','rev-parse','--show-toplevel'],target) + if run(['git','symbolic-ref','--quiet','--short','HEAD'],target)!=task_branch:fail('conflict') else: target=checkout if branch:run(['git','checkout',branch],checkout) @@ -114,14 +116,27 @@ try: if act=='list': fd=directory(path.split('/'),start=rootfd) try: - out=[] - for entry in os.scandir(fd): - if entry.name=='.paperclip-editor.lock' or entry.is_symlink():continue - info=entry.stat(follow_symlinks=False) - if not stat.S_ISDIR(info.st_mode) and not stat.S_ISREG(info.st_mode):continue - out.append({'name':entry.name,'kind':'directory' if stat.S_ISDIR(info.st_mode) else 'file','size':info.st_size,'mtimeMs':info.st_mtime*1000}) - print(json.dumps(out)) + limit=p.get('limit',1000) + if type(limit)!=int or limit<1 or limit>1000:fail('invalid') + out=[];truncated=False;scanned=0 + with os.scandir(fd) as entries: + for entry in entries: + if entry.name=='.paperclip-editor.lock':continue + if scanned>=limit:truncated=True;break + scanned+=1 + if entry.is_symlink():continue + try:info=entry.stat(follow_symlinks=False) + except FileNotFoundError:continue + if not stat.S_ISDIR(info.st_mode) and not stat.S_ISREG(info.st_mode):continue + out.append({'name':entry.name,'kind':'directory' if stat.S_ISDIR(info.st_mode) else 'file','size':info.st_size,'mtimeMs':info.st_mtime*1000}) + print(json.dumps({'entries':out,'truncated':truncated})) finally:os.close(fd) + elif act=='stat': + parentfd,name=parent(path) + try:info=os.stat(name,dir_fd=parentfd,follow_symlinks=False) + finally:os.close(parentfd) + if not stat.S_ISDIR(info.st_mode) and not stat.S_ISREG(info.st_mode):fail('invalid') + print(json.dumps({'name':name,'kind':'directory' if stat.S_ISDIR(info.st_mode) else 'file','size':info.st_size,'mtimeMs':info.st_mtime*1000})) elif act=='read': parentfd,name=parent(path) try:fd=os.open(name,os.O_RDONLY|os.O_NOFOLLOW,dir_fd=parentfd) diff --git a/server/src/modules/computers/application/service.ts b/server/src/modules/computers/application/service.ts index 38cf67056a8..988798b6da9 100644 --- a/server/src/modules/computers/application/service.ts +++ b/server/src/modules/computers/application/service.ts @@ -681,15 +681,19 @@ finally: ) { return writeBytes(path, Buffer.from(content), expectedSha256); }, - async list(path = ""): Promise< - Array<{ - name: string; - kind: "file" | "directory"; - size: number; - mtimeMs: number; - }> - > { - return remote({ action: "list", path }); + async listPage(path = "", options: { limit?: number } = {}): Promise<{ + entries: Array<{ name: string; kind: "file" | "directory"; size: number; mtimeMs: number }>; + truncated: boolean; + }> { + return remote({ action: "list", path, limit: options.limit ?? 1000 }); + }, + async list(path = "") { + const page = await this.listPage(path); + if (page.truncated) throw new ComputerError("invalid", "Computer directory exceeds the listing limit"); + return page.entries; + }, + async stat(path: string): Promise<{ name: string; kind: "file" | "directory"; size: number; mtimeMs: number }> { + return remote({ action: "stat", path }); }, async remove(path: string, expectedSha256: string) { await remote({ action: "remove", path, expectedSha256 }); diff --git a/server/src/services/persistent-agent-files.test.ts b/server/src/services/persistent-agent-files.test.ts index 18c652ae343..58a48d76c39 100644 --- a/server/src/services/persistent-agent-files.test.ts +++ b/server/src/services/persistent-agent-files.test.ts @@ -14,26 +14,26 @@ describe("persistent home initialization", () => { await fs.writeFile(path.join(root, "memory", "binary.bin"), Buffer.from([0, 255, 1])); return root; } - it("adopts an existing empty remote folder without reading or uploading the controller tree", async () => { + it.each([false, true])("adopts existing remote storage without reading the controller tree (truncated: %s)", async truncated => { const seedBytes = vi.fn(); - await seedPersistentAgentHome({ list: async () => [], seedBytes } as never, "/missing-controller-folder"); + await seedPersistentAgentHome({ listPage: async () => ({ entries: [], truncated }), seedBytes } as never, "/missing-controller-folder"); expect(seedBytes).not.toHaveBeenCalled(); }); it("seeds absent personal storage once with exact binary bytes", async () => { const seedBytes = vi.fn(async () => ({ seeded: true })); - await seedPersistentAgentHome({ list: async () => { throw { code: "not_found" }; }, seedBytes } as never, await local()); + await seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedBytes } as never, await local()); expect(seedBytes).toHaveBeenCalledWith({ "AGENTS.md": Buffer.from("instructions"), "memory/binary.bin": Buffer.from([0, 255, 1]) }); }); it("does not interpret provider outages as an absent directory", async () => { const seedBytes = vi.fn(); - await expect(seedPersistentAgentHome({ list: async () => { throw new Error("offline"); }, seedBytes } as never, await local())).rejects.toThrow("offline"); + await expect(seedPersistentAgentHome({ listPage: async () => { throw new Error("offline"); }, seedBytes } as never, await local())).rejects.toThrow("offline"); expect(seedBytes).not.toHaveBeenCalled(); }); it("rejects symlinked initial files before any upload", async () => { const root = await local(); await fs.symlink(path.join(root, "AGENTS.md"), path.join(root, "alias.md")); const seedBytes = vi.fn(); - await expect(seedPersistentAgentHome({ list: async () => { throw { code: "not_found" }; }, seedBytes } as never, root)).rejects.toThrow("links or special files"); + await expect(seedPersistentAgentHome({ listPage: async () => { throw { code: "not_found" }; }, seedBytes } as never, root)).rejects.toThrow("links or special files"); expect(seedBytes).not.toHaveBeenCalled(); }); }); diff --git a/server/src/services/persistent-agent-files.ts b/server/src/services/persistent-agent-files.ts index 92ed772d1a9..8b39c0c7f60 100644 --- a/server/src/services/persistent-agent-files.ts +++ b/server/src/services/persistent-agent-files.ts @@ -41,7 +41,7 @@ export async function readPersistentAgentFile(files: NonNullable>>, localRoot: string) { - try { await files.list(); return; } + try { await files.listPage("", { limit: 1 }); return; } catch (error) { if (!isMissingRemoteFile(error)) throw error; } const seed: Record = {}; let total = 0, count = 0; diff --git a/server/src/services/workspace-file-resources.ts b/server/src/services/workspace-file-resources.ts index f7443619580..82b3486f10e 100644 --- a/server/src/services/workspace-file-resources.ts +++ b/server/src/services/workspace-file-resources.ts @@ -1061,12 +1061,15 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor, throwIfDenied(normalized.segments); const files = await placementFiles(candidate); if (directory) { - await files.list(normalized.relativePath); + if (normalized.relativePath) { + if ((await files.stat(normalized.relativePath)).kind !== "directory") throw notFound("Workspace directory not found"); + } else { + await files.listPage("", { limit: 1 }); + } return directoryResource({ candidate, relativePath: normalized.relativePath }); } - const parent = path.posix.dirname(normalized.relativePath); - const entry = (await files.list(parent === "." ? undefined : parent)).find(item => item.name === path.posix.basename(normalized.relativePath)); - if (!entry || entry.kind !== "file") throw notFound("Workspace file not found"); + const entry = await files.stat(normalized.relativePath); + if (entry.kind !== "file") throw notFound("Workspace file not found"); const item = listItemFromStat({ candidate, relativePath: normalized.relativePath, stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!; const { relativePath: _relative, modifiedAt: _modified, ...resource } = item; @@ -1092,7 +1095,11 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor, const recursive = mode === "recent" || Boolean(normalizedQuery); async function walk(relative: string, depth: number) { if (depth > MAX_LIST_DEPTH) { truncated = true; return; } - for (const entry of await files.list(relative || undefined)) { + const remaining = WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES - scannedCount; + if (remaining <= 0) { truncated = true; return; } + const page = await files.listPage(relative || undefined, { limit: Math.min(1000, remaining) }); + truncated ||= page.truncated; + for (const entry of page.entries) { if (++scannedCount > WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES) { truncated = true; return; } const name = relative ? `${relative}/${entry.name}` : entry.name; const normalized = normalizeWorkspaceRelativePath(name); @@ -1103,7 +1110,7 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor, } else if (matchesSearch(name, normalizedQuery)) { items.push(listItemFromStat({ candidate, relativePath: name, stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!); } - if (truncated) return; + if (scannedCount >= WORKSPACE_FILE_LIST_MAX_SCANNED_ENTRIES) { truncated = true; return; } } } await walk(normalizedPath?.relativePath ?? "", 0);