feat(runner): qualify native Codex platforms and cache hosted builds

Extract the tested runtime and installed-package prerequisite without changing agent creation defaults or experimental gates.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

server/src/__tests__/dev-runner-native-binary.test.ts

@@ -1,16 +1,22 @@
 import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 
 import {
   paperclipRunnerBinaryNeedsBuild,
+  resolveNativeRunnerDevBinary,
   resolveNativeRunnerRequirement,
 } from "../../../scripts/dev-runner-native-binary.mjs";
 
 const tempRoots = new Set<string>();
+const initialPlatform = Object.getOwnPropertyDescriptor(process, "platform")!;
+const initialArchitecture = Object.getOwnPropertyDescriptor(process, "arch")!;
 
 afterEach(() => {
+  vi.restoreAllMocks();
+  Object.defineProperty(process, "platform", initialPlatform);
+  Object.defineProperty(process, "arch", initialArchitecture);
   for (const root of tempRoots) {
     fs.rmSync(root, { recursive: true, force: true });
   }
@@ -36,10 +42,24 @@ function createRunnerCheckout(): { root: string; source: string; binary: string
   fs.writeFileSync(path.join(runnerRoot, "Cargo.toml"), "[workspace]\n", "utf8");
   fs.writeFileSync(path.join(runnerRoot, "Cargo.lock"), "", "utf8");
   fs.writeFileSync(source, "fn main() {}\n", "utf8");
-  fs.writeFileSync(binary, "runnerd", "utf8");
+  fs.writeFileSync(binary, nativeHeader(process.platform, process.arch));
   return { root, source, binary };
 }
 
+function nativeHeader(platform: string, architecture: string): Buffer {
+  const bytes = Buffer.alloc(128);
+  if (platform === "darwin") {
+    bytes.writeUInt32LE(0xfeedfacf);
+    bytes.writeUInt32LE(architecture === "x64" ? 0x01000007 : 0x0100000c, 4);
+  } else if (platform === "win32") {
+    bytes.set([0x4d, 0x5a]); bytes.writeUInt32LE(64, 60);
+    bytes.writeUInt32LE(0x00004550, 64); bytes.writeUInt16LE(architecture === "x64" ? 0x8664 : 0xaa64, 68);
+  } else {
+    bytes.set([0x7f, 0x45, 0x4c, 0x46, 2, 1]); bytes.writeUInt16LE(architecture === "x64" ? 62 : 183, 18);
+  }
+  return bytes;
+}
+
 describe("paperclip runner native dev prerequisite", () => {
   it("uses an explicit status response and fails safe when status is unknown", () => {
     expect(
@@ -110,6 +130,50 @@ describe("paperclip runner native dev prerequisite", () => {
     ).toBe(false);
   });
 
+  it.each(["arm64", "x64"])("uses the staged Mac %s target before the flat build output", architecture => {
+    const checkout = createRunnerCheckout();
+    const target = path.join(path.dirname(checkout.binary), `darwin-${architecture}`, "paperclip-runnerd");
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.writeFileSync(target, nativeHeader("darwin", architecture));
+    const old = new Date(Date.now() - 2000), current = new Date(Date.now() + 2000);
+    fs.utimesSync(checkout.binary, old, old);
+    fs.utimesSync(target, current, current);
+    const input = { repoRoot: checkout.root, nativeRunnerRequired: true, platform: "darwin", architecture };
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false);
+    // A newer Linux build-host output cannot mask the remote Mac artifact.
+    fs.writeFileSync(checkout.binary, nativeHeader("linux", "x64"));
+    const wrongHostNewer = new Date(Date.now() + 4000);
+    fs.utimesSync(checkout.binary, wrongHostNewer, wrongHostNewer);
+    expect(resolveNativeRunnerDevBinary(input)).toEqual({ binary: target, sourceBuild: false });
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false);
+    fs.rmSync(checkout.binary);
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false);
+    fs.utimesSync(target, old, old);
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true);
+    fs.rmSync(target);
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true);
+  });
+
+  it.each(["arm64", "x64"])("uses only a newer matching Mac %s source build without changing release selection", architecture => {
+    const checkout = createRunnerCheckout();
+    const target = path.join(path.dirname(checkout.binary), `darwin-${architecture}`, "paperclip-runnerd");
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.writeFileSync(target, nativeHeader("darwin", architecture));
+    fs.writeFileSync(checkout.binary, nativeHeader("darwin", architecture));
+    const old = new Date(Date.now() - 2000), fresh = new Date(Date.now() + 2000), changed = new Date(Date.now() + 4000);
+    fs.utimesSync(target, old, old); fs.utimesSync(checkout.binary, fresh, fresh);
+    const input = { repoRoot: checkout.root, nativeRunnerRequired: true, platform: "darwin", architecture };
+    expect(resolveNativeRunnerDevBinary(input)).toEqual({ binary: checkout.binary, sourceBuild: true });
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(false);
+    fs.utimesSync(checkout.source, changed, changed);
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true);
+    fs.writeFileSync(checkout.binary, nativeHeader("darwin", architecture === "x64" ? "arm64" : "x64"));
+    expect(resolveNativeRunnerDevBinary(input)).toEqual({ binary: target, sourceBuild: false });
+    fs.rmSync(target);
+    expect(resolveNativeRunnerDevBinary(input)).toBeNull();
+    expect(paperclipRunnerBinaryNeedsBuild(input)).toBe(true);
+  });
+
   it("does not build a workspace binary when an explicit binary is configured", () => {
     expect(
       paperclipRunnerBinaryNeedsBuild({

server/src/__tests__/remove-test-home.test.ts

@@ -2,7 +2,7 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 
-import { afterEach, describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 
 import { removeTestHome } from "./remove-test-home.js";
 
@@ -49,7 +49,9 @@ describe("removeTestHome", () => {
 
     // The plain removal the setup file used to run is what fails in CI.
     if (process.platform !== "win32" && process.getuid?.() !== 0) {
-      expect(() => fs.rmSync(home, { recursive: true, force: true })).toThrow(/EACCES|EPERM/);
+      expect(() => fs.rmSync(home, { recursive: true, force: true })).toThrow(
+        process.platform === "darwin" ? /EACCES|EPERM|ENOTEMPTY/ : /EACCES|EPERM/,
+      );
       expect(fs.existsSync(bundle)).toBe(true);
     }
 
@@ -61,6 +63,18 @@ describe("removeTestHome", () => {
     const home = makeHome();
     fs.mkdirSync(path.join(home, "instances", "default"), { recursive: true });
     fs.writeFileSync(path.join(home, "instances", "default", "paperclip.json"), "{}\n");
+    // An unrelated ENOTEMPTY must not cause permission changes or be swallowed.
+    const error = Object.assign(new Error("unrelated nonempty directory"), { code: "ENOTEMPTY" });
+    const removal = vi.spyOn(fs, "rmSync").mockImplementation(() => { throw error; });
+    const permissionChange = vi.spyOn(fs, "chmodSync");
+    try {
+      expect(() => removeTestHome(home)).toThrow(error);
+      expect(removal).toHaveBeenCalledTimes(1);
+      expect(permissionChange).not.toHaveBeenCalled();
+    } finally {
+      removal.mockRestore();
+      permissionChange.mockRestore();
+    }
     removeTestHome(home);
     expect(fs.existsSync(home)).toBe(false);
     expect(() => removeTestHome(home)).not.toThrow();

server/src/__tests__/remove-test-home.ts

@@ -9,7 +9,8 @@ import path from "node:path";
  * in them read-only (`makeDirectoriesReadOnly` in
  * services/native-runtime/runtime-context.ts), and a run that is still
  * finishing when the worker exits can leave such a bundle under the worker's
- * home. Removing a file inside a read-only directory fails with `EACCES`,
+ * home. Removing a file inside a read-only directory fails with `EACCES`
+ * or, on macOS, the recursive removal can report `ENOTEMPTY` instead.
  * Vitest reports it as an unhandled error, and the shard fails after every
  * test passed. So on a permission error every directory in the tree is made
  * writable first and the removal is tried once more.
@@ -19,8 +20,12 @@ export function removeTestHome(home: string): void {
   try {
     fs.rmSync(home, options);
   } catch (error) {
-    if (!isPermissionError(error)) throw error;
-    makeDirectoriesWritable(home);
+    const permissionError = isPermissionError(error);
+    if (!permissionError && (error as NodeJS.ErrnoException | null)?.code !== "ENOTEMPTY") throw error;
+    const restored = makeDirectoriesWritable(home);
+    // ENOTEMPTY also has unrelated causes; retry only after restoring an
+    // actual read-only directory, and still surface a failed second removal.
+    if (!permissionError && !restored) throw error;
     fs.rmSync(home, options);
   }
 }
@@ -30,28 +35,33 @@ function isPermissionError(error: unknown): boolean {
   return code === "EACCES" || code === "EPERM";
 }
 
-function makeDirectoriesWritable(directory: string): void {
+function makeDirectoriesWritable(directory: string): boolean {
   let stat: fs.Stats;
   try {
     stat = fs.lstatSync(directory);
   } catch {
-    return;
+    return false;
   }
-  if (!stat.isDirectory()) return;
+  if (!stat.isDirectory()) return false;
   // Owner read, write and search: enough to list and unlink the children.
   // A symlink is not followed, so nothing outside the home is touched.
-  try {
-    fs.chmodSync(directory, stat.mode | 0o700);
-  } catch {
-    // A directory that cannot be changed still fails the removal below, loudly.
+  let restored = false;
+  if ((stat.mode & 0o700) !== 0o700) {
+    try {
+      fs.chmodSync(directory, stat.mode | 0o700);
+      restored = true;
+    } catch {
+      // A directory that cannot be changed still fails the removal below, loudly.
+    }
   }
   let entries: fs.Dirent[];
   try {
     entries = fs.readdirSync(directory, { withFileTypes: true });
   } catch {
-    return;
+    return restored;
   }
   for (const entry of entries) {
-    if (entry.isDirectory()) makeDirectoriesWritable(path.join(directory, entry.name));
+    if (entry.isDirectory() && makeDirectoriesWritable(path.join(directory, entry.name))) restored = true;
   }
+  return restored;
 }