diff --git a/server/src/__tests__/file-resources.test.ts b/server/src/__tests__/file-resources.test.ts index 10d11dffc40..ba09fc93740 100644 --- a/server/src/__tests__/file-resources.test.ts +++ b/server/src/__tests__/file-resources.test.ts @@ -288,6 +288,41 @@ describeEmbeddedPostgres("workspace file resources", () => { } finally { factory.mockRestore(); await fs.rm(workspace.root, { recursive: true, force: true }); } }); + it.each([["not_found", 404], ["forbidden", 403], ["provider_error", 502]] as const)("keeps recursive persistent searches scoped when a child returns %s", async (code, status) => { + const workspace = await makeWorkspace(); + const graph = await seedGraph(db, { projectRoot: workspace.projectRoot, executionRoot: workspace.executionRoot }); + await db.update(executionWorkspaces).set({ providerType: "computer", cwd: workspace.executionRoot, + metadata: { fileAuthority: { kind: "remote-persistent", environmentId: crypto.randomUUID(), placementId: crypto.randomUUID() } } }) + .where(eq(executionWorkspaces.sourceIssueId, graph.issueId)); + await fs.mkdir(path.join(workspace.projectRoot, "docs")); + await fs.writeFile(path.join(workspace.projectRoot, "docs", "guide-local.md"), "must not replace remote matches"); + const listPage = vi.fn(async (relative: string) => { + if (relative === "docs/vanished") throw Object.assign(new Error("Child unavailable"), { code }); + expect(relative).toBe("docs"); + return { entries: [ + { name: "guide-before.md", kind: "file", size: 12, mtimeMs: 1 }, + { name: "vanished", kind: "directory", size: 0, mtimeMs: 1 }, + { name: "guide-after.md", kind: "file", size: 12, mtimeMs: 1 }, + ], truncated: false }; + }); + const workspaceFiles = vi.fn(async () => ({ root: workspace.executionRoot, listPage })); + const factory = vi.spyOn(computerModule, "computerService").mockReturnValue({ workspaceFiles } as never); + try { + const service = workspaceFileResourceService(db); + const result = service.list(graph.issueId, { path: "docs/", q: "guide" }); + if (code === "not_found") { + const listed = await result; + expect(listed.items.map(item => item.title)).toEqual(["guide-after.md", "guide-before.md"]); + expect(listed.items.every(item => item.workspaceKind === "execution_workspace")).toBe(true); + expect(listed.scannedCount).toBe(3); + expect(listed.truncated).toBe(false); + } else { + await expect(result).rejects.toMatchObject({ status }); + } + expect(listPage.mock.calls.map(([relative]) => relative)).toEqual(["docs", "docs/vanished"]); + } finally { factory.mockRestore(); await fs.rm(workspace.root, { recursive: true, force: true }); } + }); + it.each([["forbidden", 403], ["provider_error", 502]] as const)("does not fall back from a persistent %s error", async (code, status) => { const workspace = await makeWorkspace(); const graph = await seedGraph(db, { projectRoot: workspace.projectRoot, executionRoot: workspace.executionRoot }); diff --git a/server/src/modules/computers/adapters/boat.test.ts b/server/src/modules/computers/adapters/boat.test.ts index 1c1cd506c4a..e1ec76f173d 100644 --- a/server/src/modules/computers/adapters/boat.test.ts +++ b/server/src/modules/computers/adapters/boat.test.ts @@ -9,6 +9,8 @@ import { symlinkSync, writeFileSync, mkdirSync, + renameSync, + chmodSync, } from "node:fs"; import { createHash, randomUUID } from "node:crypto"; import { tmpdir } from "node:os"; @@ -237,6 +239,98 @@ afterEach(() => { rmSync(root, { recursive: true, force: true }); }); describe("confined computer files", () => { + function interruptedWrite() { + const f = fixture(); + f.call({ action: "seed", files: { "nested/notes": Buffer.from("original").toString("base64") } }); + const result = spawnSync("python3", ["-c", remoteProgram + .replaceAll("/home/user/paperclip/", `${f.temp}/`) + .replace("f.write(data);f.flush();os.fsync(f.fileno())", "f.write(data);f.flush();os.kill(os.getpid(),9)")], { + input: JSON.stringify({ root: f.root, action: "write", path: "nested/notes", expectedSha256: createHash("sha256").update("original").digest("hex"), base64: Buffer.from("interrupted content").toString("base64") }), + encoding: "utf8", + }); + expect(result.signal).toBe("SIGKILL"); + const receipt = JSON.parse(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")); + expect(readFileSync(join(f.root, receipt.path), "utf8")).toBe("interrupted content"); + return { ...f, receipt }; + } + it("reclaims only the recorded interrupted write before the next file operation", () => { + const f = interruptedWrite(); + expect(f.call({ action: "list", path: "nested" }).entries.map((entry: { name: string }) => entry.name)).toEqual(["notes"]); + expect(readFileSync(join(f.root, "nested/notes"), "utf8")).toBe("original"); + expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe(""); + }); + it("clears the write receipt after a successful atomic save", () => { + const f = fixture(); f.call({ action: "seed", files: {} }); + expect(f.call({ action: "write", path: "notes", expectedSha256: null, base64: Buffer.from("saved").toString("base64") })).toEqual({ sha256: createHash("sha256").update("saved").digest("hex") }); + expect(readFileSync(join(f.root, "notes"), "utf8")).toBe("saved"); + expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe(""); + expect(readdirSync(f.root).sort()).toEqual([".paperclip-editor.lock", "notes"]); + }); + it("preserves a replacement inode and unrelated temp-looking user files", () => { + const f = interruptedWrite(); + const temp = join(f.root, f.receipt.path); + renameSync(temp, join(f.root, "user-kept-content")); + writeFileSync(temp, "replacement user file"); + const unrelated = join(f.root, ".paperclip-write-" + "a".repeat(32)); + writeFileSync(unrelated, "unrelated user file"); + f.call({ action: "list", path: "" }); + expect(readFileSync(temp, "utf8")).toBe("replacement user file"); + expect(readFileSync(unrelated, "utf8")).toBe("unrelated user file"); + expect(readFileSync(join(f.root, "user-kept-content"), "utf8")).toBe("interrupted content"); + expect(readFileSync(join(f.root, "nested/notes"), "utf8")).toBe("original"); + }); + it("preserves a symlink replacing the recorded temp without following it", () => { + const f = interruptedWrite(); + const temp = join(f.root, f.receipt.path); + renameSync(temp, join(f.root, "user-kept-content")); + const outside = join(f.temp, "outside"); writeFileSync(outside, "preserve"); + symlinkSync(outside, temp); + f.call({ action: "list", path: "nested" }); + expect(readFileSync(outside, "utf8")).toBe("preserve"); + expect(existsSync(temp)).toBe(true); + }); + it.each(["missing", "file", "symlink"])("discards an unusable receipt when its parent is %s", (replacement) => { + const f = interruptedWrite(); + renameSync(join(f.root, "nested"), join(f.root, "moved-by-user")); + const outside = join(f.temp, "outside"); mkdirSync(outside); + writeFileSync(join(outside, "notes"), "outside user content"); + if (replacement === "file") writeFileSync(join(f.root, "nested"), "replacement parent"); + if (replacement === "symlink") symlinkSync(outside, join(f.root, "nested")); + writeFileSync(join(f.root, "unrelated"), "still readable"); + expect(f.call({ action: "list", path: "" }).entries).toEqual(expect.any(Array)); + expect(Buffer.from(f.call({ action: "read", path: "unrelated" }).base64, "base64").toString()).toBe("still readable"); + expect(f.call({ action: "write", path: "new-file", expectedSha256: null, base64: Buffer.from("new content").toString("base64") }).sha256).toBeTruthy(); + expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe(""); + expect(readFileSync(join(f.root, "moved-by-user/notes"), "utf8")).toBe("original"); + expect(readFileSync(join(f.root, "moved-by-user", f.receipt.path.split("/").at(-1)), "utf8")).toBe("interrupted content"); + expect(readFileSync(join(outside, "notes"), "utf8")).toBe("outside user content"); + if (replacement === "file") expect(readFileSync(join(f.root, "nested"), "utf8")).toBe("replacement parent"); + if (replacement === "symlink") expect(existsSync(join(f.root, "nested/notes"))).toBe(true); + }); + it.skipIf(process.getuid?.() === 0)("preserves an inaccessible temp parent without blocking unrelated reads", () => { + const f = interruptedWrite(); + writeFileSync(join(f.root, "unrelated"), "still readable"); + chmodSync(join(f.root, "nested"), 0); + try { + expect(Buffer.from(f.call({ action: "read", path: "unrelated" }).base64, "base64").toString()).toBe("still readable"); + expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe(""); + } finally { chmodSync(join(f.root, "nested"), 0o700); } + expect(readFileSync(join(f.root, f.receipt.path), "utf8")).toBe("interrupted content"); + expect(readFileSync(join(f.root, "nested/notes"), "utf8")).toBe("original"); + }); + it("leaves only an empty temp when killed before its ownership receipt", () => { + const f = fixture(); f.call({ action: "seed", files: {} }); + const result = spawnSync("python3", ["-c", remoteProgram + .replaceAll("/home/user/paperclip/", `${f.temp}/`) + .replace("record_write(relative,f.fileno())", "os.kill(os.getpid(),9)")], { + input: JSON.stringify({ root: f.root, action: "write", path: "notes", expectedSha256: null, base64: Buffer.from("must not reach disk").toString("base64") }), encoding: "utf8", + }); + expect(result.signal).toBe("SIGKILL"); + const temp = readdirSync(f.root).find(name => name.startsWith(".paperclip-write-"))!; + f.call({ action: "list", path: "" }); + expect(statSync(join(f.root, temp)).size).toBe(0); + expect(existsSync(join(f.root, "notes"))).toBe(false); + }); it("seeds absent home only and preserves existing content", () => { const f = fixture(); expect( diff --git a/server/src/modules/computers/adapters/remote-program.ts b/server/src/modules/computers/adapters/remote-program.ts index f6d0e7afd0e..1d555baab02 100644 --- a/server/src/modules/computers/adapters/remote-program.ts +++ b/server/src/modules/computers/adapters/remote-program.ts @@ -1,6 +1,6 @@ /** Fixed remote program. Inputs travel as JSON, never interpolated into Python or shell. */ export const remoteProgram = String.raw` -import os,sys,json,hashlib,base64,tempfile,subprocess,fcntl,stat,shutil +import os,sys,json,hashlib,base64,tempfile,subprocess,fcntl,stat,shutil,errno p=json.load(sys.stdin) def fail(code): print(json.dumps({'error':code}));sys.exit(0) @@ -249,8 +249,47 @@ def parent(relative,create=False): return directory(parts[:-1],create,rootfd),parts[-1] path=p.get('path','');safe(path) lock=os.open('.paperclip-editor.lock',os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600,dir_fd=rootfd) +lock_info=os.fstat(lock) +if not stat.S_ISREG(lock_info.st_mode) or lock_info.st_nlink!=1:fail('invalid') fcntl.flock(lock,fcntl.LOCK_EX) +def clear_write_receipt(): + os.ftruncate(lock,0);os.fsync(lock) +def recover_write(): + os.lseek(lock,0,os.SEEK_SET);raw=os.read(lock,4097) + if not raw:return + if len(raw)>4096:fail('invalid') + try:receipt=json.loads(raw) + except (ValueError,UnicodeDecodeError): + # Content is written only after the complete receipt has been fsynced. A + # torn receipt can leave an empty temp, but gives no authority to delete it. + clear_write_receipt();return + if not isinstance(receipt,dict) or receipt.get('version')!=1:fail('invalid') + relative=receipt.get('path');name=relative.rsplit('/',1)[-1] if isinstance(relative,str) else '' + if not name.startswith('.paperclip-write-') or len(name)!=49 or any(c not in '0123456789abcdef' for c in name[17:]):fail('invalid') + if relative.startswith('/') or '\x00' in relative or '..' in relative.split('/'):fail('invalid') + parentfd=None + try: + parentfd=directory(relative.split('/')[:-1],start=rootfd) + info=os.stat(name,dir_fd=parentfd,follow_symlinks=False) + if stat.S_ISREG(info.st_mode) and info.st_nlink==1 and info.st_dev==receipt.get('device') and info.st_ino==receipt.get('inode'): + os.unlink(name,dir_fd=parentfd) + except OSError as error: + # A removed/replaced/inaccessible ancestor makes this receipt unusable. Preserve + # whatever occupies that path and allow unrelated file operations to proceed. + if error.errno not in (errno.ENOENT,errno.ENOTDIR,errno.ELOOP,errno.EACCES,errno.EPERM):raise + finally: + if parentfd is not None:os.close(parentfd) + clear_write_receipt() +def record_write(relative,fd): + info=os.fstat(fd) + receipt=json.dumps({'version':1,'path':relative,'device':info.st_dev,'inode':info.st_ino}).encode() + if len(receipt)>4096:fail('invalid') + os.lseek(lock,0,os.SEEK_SET);os.ftruncate(lock,0) + while receipt: + written=os.write(lock,receipt);receipt=receipt[written:] + os.fsync(lock) try: + recover_write() if act=='list': fd=directory(path.split('/'),start=rootfd) try: @@ -320,11 +359,13 @@ try: temp='.paperclip-write-'+secrets.token_hex(16) fd=os.open(temp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=parentfd) try: - with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno()) + with os.fdopen(fd,'wb') as f: + relative='/'.join([part for part in path.split('/') if part not in ('','.')][:-1]+[temp]) + record_write(relative,f.fileno()) + f.write(data);f.flush();os.fsync(f.fileno()) os.replace(temp,name,src_dir_fd=parentfd,dst_dir_fd=parentfd) finally: - try:os.unlink(temp,dir_fd=parentfd) - except FileNotFoundError:pass + recover_write() print(json.dumps({'sha256':digest(data)})) elif act=='remove': if old is None:fail('not_found') diff --git a/server/src/services/workspace-file-resources.ts b/server/src/services/workspace-file-resources.ts index 250083698ba..a4e633d15ae 100644 --- a/server/src/services/workspace-file-resources.ts +++ b/server/src/services/workspace-file-resources.ts @@ -1105,8 +1105,14 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor, const normalized = normalizeWorkspaceRelativePath(name); if (shouldPruneSegments(normalized.segments)) continue; if (entry.kind === "directory") { - if (recursive) await walk(name, depth + 1); - else items.push(listItemFromDirectory({ candidate, relativePath: name, stat: { mtime: new Date(entry.mtimeMs) } })); + if (recursive) { + try { await walk(name, depth + 1); } + catch (error) { + // A child may disappear during the scan. Only a missing starting + // directory should make automatic resolution try another workspace. + if (!isHttpStatus(error, 404)) throw error; + } + } else items.push(listItemFromDirectory({ candidate, relativePath: name, stat: { mtime: new Date(entry.mtimeMs) } })); } else if (matchesSearch(name, normalizedQuery)) { items.push(listItemFromStat({ candidate, relativePath: name, stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!); }