Merge final Boat runtime candidate into staging compatibility branch

Co-Authored-By: Paperclip <noreply@paperclip.ing>

* commit 'ac09d6ecce2809ed1ad6541d2908e713c543b39f':
  fix: discard inaccessible computer write receipts safely
  fix: retain Boat search results when child folders disappear
  fix: reclaim recorded interrupted computer saves

server/src/__tests__/file-resources.test.ts

@@ -288,6 +288,41 @@ describeEmbeddedPostgres("workspace file resources", () => {
     } finally { factory.mockRestore(); await fs.rm(workspace.root, { recursive: true, force: true }); }
   });
 
+  it.each([["not_found", 404], ["forbidden", 403], ["provider_error", 502]] as const)("keeps recursive persistent searches scoped when a child returns %s", async (code, status) => {
+    const workspace = await makeWorkspace();
+    const graph = await seedGraph(db, { projectRoot: workspace.projectRoot, executionRoot: workspace.executionRoot });
+    await db.update(executionWorkspaces).set({ providerType: "computer", cwd: workspace.executionRoot,
+      metadata: { fileAuthority: { kind: "remote-persistent", environmentId: crypto.randomUUID(), placementId: crypto.randomUUID() } } })
+      .where(eq(executionWorkspaces.sourceIssueId, graph.issueId));
+    await fs.mkdir(path.join(workspace.projectRoot, "docs"));
+    await fs.writeFile(path.join(workspace.projectRoot, "docs", "guide-local.md"), "must not replace remote matches");
+    const listPage = vi.fn(async (relative: string) => {
+      if (relative === "docs/vanished") throw Object.assign(new Error("Child unavailable"), { code });
+      expect(relative).toBe("docs");
+      return { entries: [
+        { name: "guide-before.md", kind: "file", size: 12, mtimeMs: 1 },
+        { name: "vanished", kind: "directory", size: 0, mtimeMs: 1 },
+        { name: "guide-after.md", kind: "file", size: 12, mtimeMs: 1 },
+      ], truncated: false };
+    });
+    const workspaceFiles = vi.fn(async () => ({ root: workspace.executionRoot, listPage }));
+    const factory = vi.spyOn(computerModule, "computerService").mockReturnValue({ workspaceFiles } as never);
+    try {
+      const service = workspaceFileResourceService(db);
+      const result = service.list(graph.issueId, { path: "docs/", q: "guide" });
+      if (code === "not_found") {
+        const listed = await result;
+        expect(listed.items.map(item => item.title)).toEqual(["guide-after.md", "guide-before.md"]);
+        expect(listed.items.every(item => item.workspaceKind === "execution_workspace")).toBe(true);
+        expect(listed.scannedCount).toBe(3);
+        expect(listed.truncated).toBe(false);
+      } else {
+        await expect(result).rejects.toMatchObject({ status });
+      }
+      expect(listPage.mock.calls.map(([relative]) => relative)).toEqual(["docs", "docs/vanished"]);
+    } finally { factory.mockRestore(); await fs.rm(workspace.root, { recursive: true, force: true }); }
+  });
+
   it.each([["forbidden", 403], ["provider_error", 502]] as const)("does not fall back from a persistent %s error", async (code, status) => {
     const workspace = await makeWorkspace();
     const graph = await seedGraph(db, { projectRoot: workspace.projectRoot, executionRoot: workspace.executionRoot });

server/src/modules/computers/adapters/boat.test.ts

@@ -9,6 +9,8 @@ import {
   symlinkSync,
   writeFileSync,
   mkdirSync,
+  renameSync,
+  chmodSync,
 } from "node:fs";
 import { createHash, randomUUID } from "node:crypto";
 import { tmpdir } from "node:os";
@@ -237,6 +239,98 @@ afterEach(() => {
     rmSync(root, { recursive: true, force: true });
 });
 describe("confined computer files", () => {
+  function interruptedWrite() {
+    const f = fixture();
+    f.call({ action: "seed", files: { "nested/notes": Buffer.from("original").toString("base64") } });
+    const result = spawnSync("python3", ["-c", remoteProgram
+      .replaceAll("/home/user/paperclip/", `${f.temp}/`)
+      .replace("f.write(data);f.flush();os.fsync(f.fileno())", "f.write(data);f.flush();os.kill(os.getpid(),9)")], {
+      input: JSON.stringify({ root: f.root, action: "write", path: "nested/notes", expectedSha256: createHash("sha256").update("original").digest("hex"), base64: Buffer.from("interrupted content").toString("base64") }),
+      encoding: "utf8",
+    });
+    expect(result.signal).toBe("SIGKILL");
+    const receipt = JSON.parse(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8"));
+    expect(readFileSync(join(f.root, receipt.path), "utf8")).toBe("interrupted content");
+    return { ...f, receipt };
+  }
+  it("reclaims only the recorded interrupted write before the next file operation", () => {
+    const f = interruptedWrite();
+    expect(f.call({ action: "list", path: "nested" }).entries.map((entry: { name: string }) => entry.name)).toEqual(["notes"]);
+    expect(readFileSync(join(f.root, "nested/notes"), "utf8")).toBe("original");
+    expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe("");
+  });
+  it("clears the write receipt after a successful atomic save", () => {
+    const f = fixture(); f.call({ action: "seed", files: {} });
+    expect(f.call({ action: "write", path: "notes", expectedSha256: null, base64: Buffer.from("saved").toString("base64") })).toEqual({ sha256: createHash("sha256").update("saved").digest("hex") });
+    expect(readFileSync(join(f.root, "notes"), "utf8")).toBe("saved");
+    expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe("");
+    expect(readdirSync(f.root).sort()).toEqual([".paperclip-editor.lock", "notes"]);
+  });
+  it("preserves a replacement inode and unrelated temp-looking user files", () => {
+    const f = interruptedWrite();
+    const temp = join(f.root, f.receipt.path);
+    renameSync(temp, join(f.root, "user-kept-content"));
+    writeFileSync(temp, "replacement user file");
+    const unrelated = join(f.root, ".paperclip-write-" + "a".repeat(32));
+    writeFileSync(unrelated, "unrelated user file");
+    f.call({ action: "list", path: "" });
+    expect(readFileSync(temp, "utf8")).toBe("replacement user file");
+    expect(readFileSync(unrelated, "utf8")).toBe("unrelated user file");
+    expect(readFileSync(join(f.root, "user-kept-content"), "utf8")).toBe("interrupted content");
+    expect(readFileSync(join(f.root, "nested/notes"), "utf8")).toBe("original");
+  });
+  it("preserves a symlink replacing the recorded temp without following it", () => {
+    const f = interruptedWrite();
+    const temp = join(f.root, f.receipt.path);
+    renameSync(temp, join(f.root, "user-kept-content"));
+    const outside = join(f.temp, "outside"); writeFileSync(outside, "preserve");
+    symlinkSync(outside, temp);
+    f.call({ action: "list", path: "nested" });
+    expect(readFileSync(outside, "utf8")).toBe("preserve");
+    expect(existsSync(temp)).toBe(true);
+  });
+  it.each(["missing", "file", "symlink"])("discards an unusable receipt when its parent is %s", (replacement) => {
+    const f = interruptedWrite();
+    renameSync(join(f.root, "nested"), join(f.root, "moved-by-user"));
+    const outside = join(f.temp, "outside"); mkdirSync(outside);
+    writeFileSync(join(outside, "notes"), "outside user content");
+    if (replacement === "file") writeFileSync(join(f.root, "nested"), "replacement parent");
+    if (replacement === "symlink") symlinkSync(outside, join(f.root, "nested"));
+    writeFileSync(join(f.root, "unrelated"), "still readable");
+    expect(f.call({ action: "list", path: "" }).entries).toEqual(expect.any(Array));
+    expect(Buffer.from(f.call({ action: "read", path: "unrelated" }).base64, "base64").toString()).toBe("still readable");
+    expect(f.call({ action: "write", path: "new-file", expectedSha256: null, base64: Buffer.from("new content").toString("base64") }).sha256).toBeTruthy();
+    expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe("");
+    expect(readFileSync(join(f.root, "moved-by-user/notes"), "utf8")).toBe("original");
+    expect(readFileSync(join(f.root, "moved-by-user", f.receipt.path.split("/").at(-1)), "utf8")).toBe("interrupted content");
+    expect(readFileSync(join(outside, "notes"), "utf8")).toBe("outside user content");
+    if (replacement === "file") expect(readFileSync(join(f.root, "nested"), "utf8")).toBe("replacement parent");
+    if (replacement === "symlink") expect(existsSync(join(f.root, "nested/notes"))).toBe(true);
+  });
+  it.skipIf(process.getuid?.() === 0)("preserves an inaccessible temp parent without blocking unrelated reads", () => {
+    const f = interruptedWrite();
+    writeFileSync(join(f.root, "unrelated"), "still readable");
+    chmodSync(join(f.root, "nested"), 0);
+    try {
+      expect(Buffer.from(f.call({ action: "read", path: "unrelated" }).base64, "base64").toString()).toBe("still readable");
+      expect(readFileSync(join(f.root, ".paperclip-editor.lock"), "utf8")).toBe("");
+    } finally { chmodSync(join(f.root, "nested"), 0o700); }
+    expect(readFileSync(join(f.root, f.receipt.path), "utf8")).toBe("interrupted content");
+    expect(readFileSync(join(f.root, "nested/notes"), "utf8")).toBe("original");
+  });
+  it("leaves only an empty temp when killed before its ownership receipt", () => {
+    const f = fixture(); f.call({ action: "seed", files: {} });
+    const result = spawnSync("python3", ["-c", remoteProgram
+      .replaceAll("/home/user/paperclip/", `${f.temp}/`)
+      .replace("record_write(relative,f.fileno())", "os.kill(os.getpid(),9)")], {
+      input: JSON.stringify({ root: f.root, action: "write", path: "notes", expectedSha256: null, base64: Buffer.from("must not reach disk").toString("base64") }), encoding: "utf8",
+    });
+    expect(result.signal).toBe("SIGKILL");
+    const temp = readdirSync(f.root).find(name => name.startsWith(".paperclip-write-"))!;
+    f.call({ action: "list", path: "" });
+    expect(statSync(join(f.root, temp)).size).toBe(0);
+    expect(existsSync(join(f.root, "notes"))).toBe(false);
+  });
   it("seeds absent home only and preserves existing content", () => {
     const f = fixture();
     expect(

server/src/modules/computers/adapters/remote-program.ts

@@ -1,6 +1,6 @@
 /** Fixed remote program. Inputs travel as JSON, never interpolated into Python or shell. */
 export const remoteProgram = String.raw`
-import os,sys,json,hashlib,base64,tempfile,subprocess,fcntl,stat,shutil
+import os,sys,json,hashlib,base64,tempfile,subprocess,fcntl,stat,shutil,errno
 p=json.load(sys.stdin)
 def fail(code):
  print(json.dumps({'error':code}));sys.exit(0)
@@ -249,8 +249,47 @@ def parent(relative,create=False):
  return directory(parts[:-1],create,rootfd),parts[-1]
 path=p.get('path','');safe(path)
 lock=os.open('.paperclip-editor.lock',os.O_CREAT|os.O_RDWR|os.O_NOFOLLOW,0o600,dir_fd=rootfd)
+lock_info=os.fstat(lock)
+if not stat.S_ISREG(lock_info.st_mode) or lock_info.st_nlink!=1:fail('invalid')
 fcntl.flock(lock,fcntl.LOCK_EX)
+def clear_write_receipt():
+ os.ftruncate(lock,0);os.fsync(lock)
+def recover_write():
+ os.lseek(lock,0,os.SEEK_SET);raw=os.read(lock,4097)
+ if not raw:return
+ if len(raw)>4096:fail('invalid')
+ try:receipt=json.loads(raw)
+ except (ValueError,UnicodeDecodeError):
+  # Content is written only after the complete receipt has been fsynced. A
+  # torn receipt can leave an empty temp, but gives no authority to delete it.
+  clear_write_receipt();return
+ if not isinstance(receipt,dict) or receipt.get('version')!=1:fail('invalid')
+ relative=receipt.get('path');name=relative.rsplit('/',1)[-1] if isinstance(relative,str) else ''
+ if not name.startswith('.paperclip-write-') or len(name)!=49 or any(c not in '0123456789abcdef' for c in name[17:]):fail('invalid')
+ if relative.startswith('/') or '\x00' in relative or '..' in relative.split('/'):fail('invalid')
+ parentfd=None
+ try:
+  parentfd=directory(relative.split('/')[:-1],start=rootfd)
+  info=os.stat(name,dir_fd=parentfd,follow_symlinks=False)
+  if stat.S_ISREG(info.st_mode) and info.st_nlink==1 and info.st_dev==receipt.get('device') and info.st_ino==receipt.get('inode'):
+   os.unlink(name,dir_fd=parentfd)
+ except OSError as error:
+  # A removed/replaced/inaccessible ancestor makes this receipt unusable. Preserve
+  # whatever occupies that path and allow unrelated file operations to proceed.
+  if error.errno not in (errno.ENOENT,errno.ENOTDIR,errno.ELOOP,errno.EACCES,errno.EPERM):raise
+ finally:
+  if parentfd is not None:os.close(parentfd)
+ clear_write_receipt()
+def record_write(relative,fd):
+ info=os.fstat(fd)
+ receipt=json.dumps({'version':1,'path':relative,'device':info.st_dev,'inode':info.st_ino}).encode()
+ if len(receipt)>4096:fail('invalid')
+ os.lseek(lock,0,os.SEEK_SET);os.ftruncate(lock,0)
+ while receipt:
+  written=os.write(lock,receipt);receipt=receipt[written:]
+ os.fsync(lock)
 try:
+ recover_write()
  if act=='list':
   fd=directory(path.split('/'),start=rootfd)
   try:
@@ -320,11 +359,13 @@ try:
     temp='.paperclip-write-'+secrets.token_hex(16)
     fd=os.open(temp,os.O_WRONLY|os.O_CREAT|os.O_EXCL|os.O_NOFOLLOW,0o600,dir_fd=parentfd)
     try:
-     with os.fdopen(fd,'wb') as f:f.write(data);f.flush();os.fsync(f.fileno())
+     with os.fdopen(fd,'wb') as f:
+      relative='/'.join([part for part in path.split('/') if part not in ('','.')][:-1]+[temp])
+      record_write(relative,f.fileno())
+      f.write(data);f.flush();os.fsync(f.fileno())
      os.replace(temp,name,src_dir_fd=parentfd,dst_dir_fd=parentfd)
     finally:
-     try:os.unlink(temp,dir_fd=parentfd)
-     except FileNotFoundError:pass
+     recover_write()
     print(json.dumps({'sha256':digest(data)}))
    elif act=='remove':
     if old is None:fail('not_found')

server/src/services/workspace-file-resources.ts

@@ -1105,8 +1105,14 @@ export function workspaceFileResourceService(db: Db, actor?: AuthorizationActor,
         const normalized = normalizeWorkspaceRelativePath(name);
         if (shouldPruneSegments(normalized.segments)) continue;
         if (entry.kind === "directory") {
-          if (recursive) await walk(name, depth + 1);
-          else items.push(listItemFromDirectory({ candidate, relativePath: name, stat: { mtime: new Date(entry.mtimeMs) } }));
+          if (recursive) {
+            try { await walk(name, depth + 1); }
+            catch (error) {
+              // A child may disappear during the scan. Only a missing starting
+              // directory should make automatic resolution try another workspace.
+              if (!isHttpStatus(error, 404)) throw error;
+            }
+          } else items.push(listItemFromDirectory({ candidate, relativePath: name, stat: { mtime: new Date(entry.mtimeMs) } }));
         } else if (matchesSearch(name, normalizedQuery)) {
           items.push(listItemFromStat({ candidate, relativePath: name, stat: { size: entry.size, mtime: new Date(entry.mtimeMs) } })!);
         }