From 24508a8ba58cf69908360e1457c2e493ba90e921 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:01:54 -0600 Subject: [PATCH] perf(dev): add agent preflight with a shared content-keyed build cache (#19555) perf(dev): add agent:preflight with a shared content-keyed build cache Agent worktrees each built the Runner from nothing: a cold cargo debug and release build, the Runner TypeScript build and its checks, about 8 minutes per fresh worktree on a loaded machine. `pnpm agent:preflight` replaces the manual install + Runner build sequence. It skips steps whose inputs are unchanged and reuses other worktrees' results from ~/.cache/paperclip/agent-build: - Runner and eval-kernel dist, keyed by the Runner build-input digest and the toolchain - Runner debug binaries, keyed by every file the Rust build reads - a cargo seed with third-party crates only; the Runner crate is scrubbed because cargo records its out-of-package includes by absolute path Entries are written once with an atomic rename and restored as APFS clones. Also exclude provider-assets from the Runner build inputs. The build writes that directory itself, so the first build in every fresh checkout saw its inputs change and never recorded a build stamp. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 7 + doc/DEVELOPING.md | 40 ++- .../paperclip-runner/scripts/build-inputs.mjs | 11 +- .../scripts/build-inputs.test.mjs | 1 + scripts/__tests__/agent-build-cache.test.mjs | 114 +++++++ scripts/agent-build-cache.mjs | 264 +++++++++++++++ scripts/agent-preflight.mjs | 317 ++++++++++++++++++ 7 files changed, 751 insertions(+), 3 deletions(-) create mode 100644 scripts/__tests__/agent-build-cache.test.mjs create mode 100644 scripts/agent-build-cache.mjs create mode 100644 scripts/agent-preflight.mjs diff --git a/AGENTS.md b/AGENTS.md index 7119e1fc26a..5918eb8d2ad 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -198,6 +198,13 @@ ignored; never commit aggregate catalogs, refresh review fingerprints, or append a shared review journal. Use Git diffs and keep review evidence in the PR or issue. Structural checks are not production acceptance. +Prepare a new or updated worktree with `node scripts/agent-preflight.mjs` instead of running +`pnpm install`, the Runner build and `build:rust` by hand. It skips unchanged +steps and reuses other worktrees' Runner builds, debug binaries and third-party +crates from a content-keyed cache in `~/.cache/paperclip/agent-build`. Do not +share `CARGO_TARGET_DIR` between worktrees. See "Agent worktree preflight" in +`doc/DEVELOPING.md`. + Default local/agent verification path: ```sh diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 29108db1513..fd93ba96369 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -1069,6 +1069,44 @@ defines synchronization, permissions and evidence. After map edits, run `node scripts/check-maps.mjs` for structural/reference checks; it does not start an instance or qualify access as an agent. +### Agent worktree preflight + +Prepare a new or updated agent worktree with one command: + +```sh +node scripts/agent-preflight.mjs +``` + +It does what `pnpm install --frozen-lockfile`, the Runner's `ensure:eval-build-deps`, +`build:rust`, `build` and `build:runner-binaries` did, and skips work whose inputs +are unchanged: + +- **install** is skipped when the lockfile, every `package.json`, `patches/`, Node + and pnpm match this worktree's last install. +- **Runner build** is skipped when the Runner's build stamp is current. When another + worktree built the same inputs (the build-input digest described under + "Reused Runner build in the server build", plus the compiler), the Runner and eval + kernel `dist` are cloned from the shared cache. A restore skips the build's lint + checks; CI runs them. +- **Runner debug binaries** (`runner/target/debug`) are reused the same way, keyed + by every file the Rust build reads. +- A worktree without `runner/target` starts from a clone of a shared cargo seed + that holds only third-party crates, so cargo compiles only the Runner crate. + +The cache lives in `~/.cache/paperclip/agent-build` (`PAPERCLIP_AGENT_CACHE_DIR` +moves it, `PAPERCLIP_AGENT_CACHE=0` or `--no-cache` turns it off). Entries are +keyed by content and toolchain, never by path or mtime, are written once with an +atomic rename, and are restored as APFS clones, so concurrent worktrees can share +them. The seed never holds the Runner crate: cargo records its out-of-package +includes by absolute path, so units from another checkout cannot be trusted. Keep +the cache on the worktrees' volume so clones stay cheap. `--explain` prints each +step's decision; `--skip-rust-debug` skips the debug binaries. + +Do not share `CARGO_TARGET_DIR` between worktrees: cargo serializes builds on its +lock, and the Runner crate's absolute-path dependency records would make one +worktree's freshness check read another's files. The preflight disables the cache +when `CARGO_TARGET_DIR` is set. + ### Incremental local verification Use the local verification entry point for ordinary issue work: @@ -1410,7 +1448,7 @@ build. It used to run the Runner's full `build` every time. Now the Runner's `build` brackets itself with `scripts/record-build-inputs.mjs`: `--begin`, its first step, forgets the previous stamp and records a digest of the build's inputs (every file under `packages/paperclip-runner` except `dist`, -`dist-browser`, `runner/target`, `node_modules` and dot-directories; every +`dist-browser`, `runner/target`, `provider-assets` (written by the build), `node_modules` and dot-directories; every file under `packages/paperclip-eval-kernel` except `dist` and `node_modules`; every file under the root `patches` directory, which the acpx profile check hashes and the provider entrypoint bundler copies into `dist`; and the root diff --git a/packages/paperclip-runner/scripts/build-inputs.mjs b/packages/paperclip-runner/scripts/build-inputs.mjs index 746ea66c920..8d1874d21cf 100644 --- a/packages/paperclip-runner/scripts/build-inputs.mjs +++ b/packages/paperclip-runner/scripts/build-inputs.mjs @@ -32,7 +32,7 @@ // build's `--finish` drops the stamp if its inputs did. // // Inputs: every file under packages/paperclip-runner except its build outputs -// (dist, dist-browser, runner/target), node_modules and dot-directories (tool +// (dist, dist-browser, runner/target, provider-assets), node_modules and dot-directories (tool // caches); every file under packages/paperclip-eval-kernel except dist and // node_modules, because build:typescript builds that package first; every // file under the root patches directory, because check:acpx-profiles hashes @@ -76,7 +76,14 @@ const ROOT_INPUT_FILES = [ ]; const OUTPUT_DIRECTORY_RELATIVE = "dist"; const SKIPPED_DIRECTORY_NAMES = new Set(["node_modules", "dist", "dist-browser"]); -const RUNNER_SKIPPED_DIRECTORIES = new Set([`${RUNNER_PACKAGE_RELATIVE}/runner/target`]); +// provider-assets is written by the build itself (install-copilot-assets.mjs, +// the last step of build:typescript) from a lockfile-pinned package. Counting +// it as an input made the first build in every fresh checkout see its inputs +// change while it ran, so that build never recorded a stamp. +const RUNNER_SKIPPED_DIRECTORIES = new Set([ + `${RUNNER_PACKAGE_RELATIVE}/runner/target`, + `${RUNNER_PACKAGE_RELATIVE}/provider-assets`, +]); export function runnerPackageRootFromScripts() { return path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); diff --git a/packages/paperclip-runner/scripts/build-inputs.test.mjs b/packages/paperclip-runner/scripts/build-inputs.test.mjs index 7822de2cbb3..27402c6cdb3 100644 --- a/packages/paperclip-runner/scripts/build-inputs.test.mjs +++ b/packages/paperclip-runner/scripts/build-inputs.test.mjs @@ -160,6 +160,7 @@ test("the digest is stable, ignores outputs and caches, and tracks every kind of "packages/paperclip-runner/node_modules/dep/index.js": "changed\n", "packages/paperclip-runner/.vite/cache.json": '{"changed":true}\n', "packages/paperclip-runner/runner/target/release/paperclip-runnerd": "rebuilt\n", + "packages/paperclip-runner/provider-assets/copilot/copilot": "installed by the build\n", "packages/paperclip-eval-kernel/dist/index.js": "changed\n", }); assert.equal(computeBuildInputsDigest(repoRoot).digest, before.digest, "an output changed the digest"); diff --git a/scripts/__tests__/agent-build-cache.test.mjs b/scripts/__tests__/agent-build-cache.test.mjs new file mode 100644 index 00000000000..88e90b71aef --- /dev/null +++ b/scripts/__tests__/agent-build-cache.test.mjs @@ -0,0 +1,114 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { cacheRoot, cloneInto, hashFiles, pruneKind, publishEntry, readEntry, scrubSeed, seedIsScrubbed } from "../agent-build-cache.mjs"; + +function scratch(t) { + const root = mkdtempSync(path.join(os.tmpdir(), "agent-build-cache-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + return root; +} + +function write(file, content = "x") { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, content); +} + +test("a seed keeps third-party crates and loses every unit of the workspace crate", t => { + const target = path.join(scratch(t), "target"); + const bins = ["paperclip-runnerd", "fake-codex-app-server"]; + for (const profile of ["debug", "release"]) { + const base = path.join(target, profile); + write(path.join(base, ".fingerprint/serde-0123/lib-serde")); + write(path.join(base, ".fingerprint/paperclip-runner-core-0123/lib-paperclip_runner_core")); + write(path.join(base, ".fingerprint/paperclip-runner-core-4567/bin-paperclip-runnerd")); + write(path.join(base, "deps/libserde-0123.rlib")); + write(path.join(base, "deps/libpaperclip_runner_core-0123.rlib")); + write(path.join(base, "deps/paperclip_runnerd-0123")); + write(path.join(base, "deps/fake_codex_app_server-0123.d")); + write(path.join(base, "build/ring-0123/out/lib.a")); + write(path.join(base, "incremental/paperclip_runner_core-0123/s")); + write(path.join(base, "paperclip-runnerd")); + write(path.join(base, "paperclip-runnerd.d")); + write(path.join(base, ".cargo-lock")); + } + write(path.join(target, "tmp/scratch")); + write(path.join(target, "CACHEDIR.TAG")); + assert.equal(seedIsScrubbed(target), false); + scrubSeed(target, bins); + assert.equal(seedIsScrubbed(target), true); + for (const profile of ["debug", "release"]) { + const base = path.join(target, profile); + assert.deepEqual(readdirSync(path.join(base, ".fingerprint")), ["serde-0123"]); + assert.deepEqual(readdirSync(path.join(base, "deps")), ["libserde-0123.rlib"]); + assert.ok(existsSync(path.join(base, "build/ring-0123/out/lib.a"))); + assert.deepEqual(readdirSync(path.join(base, "incremental")), []); + for (const gone of ["paperclip-runnerd", "paperclip-runnerd.d", ".cargo-lock"]) assert.equal(existsSync(path.join(base, gone)), false); + } + assert.equal(existsSync(path.join(target, "tmp")), false); + assert.ok(existsSync(path.join(target, "CACHEDIR.TAG"))); +}); + +test("entries are published whole, once, and read back only when complete", t => { + const cache = scratch(t); + assert.equal(readEntry(cache, "runner-dist", "k1"), null); + assert.throws(() => publishEntry(cache, "runner-dist", "k1", () => { throw new Error("build output vanished"); })); + assert.equal(readEntry(cache, "runner-dist", "k1"), null); + assert.deepEqual(readdirSync(path.join(cache, "runner-dist")), []); + assert.equal(publishEntry(cache, "runner-dist", "k1", directory => write(path.join(directory, "a"), "first"), + { meta: { profiles: ["debug"] } }), true); + assert.equal(publishEntry(cache, "runner-dist", "k1", directory => write(path.join(directory, "a"), "second")), false); + const entry = readEntry(cache, "runner-dist", "k1"); + assert.equal(readFileSync(path.join(entry.directory, "a"), "utf8"), "first"); + assert.deepEqual(entry.meta.profiles, ["debug"]); + assert.equal(publishEntry(cache, "runner-dist", "k1", directory => write(path.join(directory, "a"), "third"), + { replace: meta => !meta.profiles.includes("release") }), true); + assert.equal(readFileSync(path.join(readEntry(cache, "runner-dist", "k1").directory, "a"), "utf8"), "third"); + assert.equal(readEntry(cache, "other-kind", "k1"), null); + write(path.join(cache, "runner-dist", "k2", "entry.json"), "{\"schema\":\"old\"}"); + assert.equal(readEntry(cache, "runner-dist", "k2"), null); +}); + +test("a clone lands whole under the destination name", t => { + const root = scratch(t); + write(path.join(root, "source/nested/file"), "content"); + cloneInto(path.join(root, "source"), path.join(root, "copy")); + assert.equal(readFileSync(path.join(root, "copy/nested/file"), "utf8"), "content"); + assert.deepEqual(readdirSync(root).sort(), ["copy", "source"]); +}); + +test("keys depend on content and repo-relative paths, not the checkout location", t => { + const first = scratch(t), second = scratch(t); + for (const root of [first, second]) write(path.join(root, "runner/src/lib.rs"), "fn main() {}"); + const digest = root => hashFiles(root, ["runner/src/lib.rs", "runner/missing.rs"]).digest("hex"); + assert.equal(digest(first), digest(second)); + write(path.join(second, "runner/src/lib.rs"), "fn main() { }"); + assert.notEqual(digest(first), digest(second)); + write(path.join(second, "runner/missing.rs"), ""); + write(path.join(first, "runner/src/lib.rs"), "fn main() { }"); + assert.notEqual(digest(first), digest(second)); +}); + +test("the cache directory follows the environment and can be turned off", () => { + assert.equal(cacheRoot({ PAPERCLIP_AGENT_CACHE: "0" }), null); + assert.equal(cacheRoot({ PAPERCLIP_AGENT_CACHE_DIR: "/tmp/agent-cache" }), "/tmp/agent-cache"); + assert.equal(cacheRoot({ XDG_CACHE_HOME: "/tmp/xdg" }), "/tmp/xdg/paperclip/agent-build"); +}); + +test("pruning keeps the most recently used entries and anything in use", t => { + const cache = scratch(t); + const now = Date.now(); + const hour = 60 * 60 * 1000; + const names = ["s1", "s2", "s3", "s4", ".tmp-old", ".tmp-new"]; + for (const name of names) mkdirSync(path.join(cache, "cargo-seed", name), { recursive: true }); + const age = { s1: 5 * hour, s2: 1 * hour, s3: 3 * hour, s4: 60 * 1000, ".tmp-old": 30 * hour, ".tmp-new": hour }; + for (const name of names) { + const when = new Date(now - age[name]); + utimesSync(path.join(cache, "cargo-seed", name), when, when); + } + pruneKind(cache, "cargo-seed", "s1", now); + // Limit 2: s4 and s2 are newest; s1 is the entry just published; s3 goes. + assert.deepEqual(readdirSync(path.join(cache, "cargo-seed")).sort(), [".tmp-new", "s1", "s2", "s4"]); +}); diff --git a/scripts/agent-build-cache.mjs b/scripts/agent-build-cache.mjs new file mode 100644 index 00000000000..2fcebd29fbd --- /dev/null +++ b/scripts/agent-build-cache.mjs @@ -0,0 +1,264 @@ +// Content-addressed build cache shared by the agent worktrees on one machine. +// +// Every worktree used to build the Runner from nothing: its own cargo target +// directory, its own Runner `dist`, its own debug binaries. This module keeps +// three kinds of entries under one cache directory, each written once and +// never modified afterwards: +// +// cargo-seed/ a cargo target directory with only third-party +// crates in it, keyed by the lockfile, manifests and +// compiler. A fresh worktree starts from a clone. +// runner-dist/ the Runner's and the eval kernel's `dist`, keyed by +// the Runner build-input digest and the toolchain +// runner-debug-bins/ the Runner's debug executables, keyed by every +// file the Rust build reads and the toolchain +// +// Why each entry is safe to reuse: +// +// - Seeds hold no workspace crate. The Runner crate includes files outside +// its package root (protocol schemas, fixtures), and cargo records those as +// absolute paths, so a unit compiled in another checkout would be judged by +// that checkout's files. `scrubSeed` deletes every unit of the workspace +// crate; with no fingerprint cargo always recompiles it. Registry crates are +// immutable sources under CARGO_HOME, and cargo records paths inside the +// target directory relative to it, so it checks them as usual. +// - Dist and binary entries are keyed by the content of their inputs, not +// paths or mtimes. Any changed byte selects a different key. The Runner's +// executables use CARGO_MANIFEST_DIR only in tests, so a binary built in +// another checkout behaves the same. +// - Entries are published with one rename from a temporary sibling, and +// restored by cloning into a temporary sibling and renaming it into place. +// A reader never sees a partial entry or leaves a partial tree. +// +// On APFS `cp -c` clones files, so a restore costs metadata, not bytes, and +// the clones share disk blocks with the entry. Elsewhere the copy falls back +// to `cp --reflink=auto` or a plain copy. +// +// PAPERCLIP_AGENT_CACHE_DIR moves the cache; PAPERCLIP_AGENT_CACHE=0 turns it +// off. Keep the cache on the same volume as the worktrees so clones work. + +import { execFileSync } from "node:child_process"; +import { createHash, randomUUID } from "node:crypto"; +import { existsSync, globSync, lstatSync, mkdirSync, readdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, + utimesSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +export const CACHE_SCHEMA = "paperclip.agent-build-cache.v1"; +export const RUNNER_PACKAGE = "packages/paperclip-runner"; +export const EVAL_KERNEL_PACKAGE = "packages/paperclip-eval-kernel"; +// Entries are touched on every hit, so these keep the most recently used ones. +// A seed is about 2 GB of mostly shared clone blocks; the others are 30-100 MB. +const KEEP_ENTRIES = { "cargo-seed": 2, "runner-dist": 12, "runner-debug-bins": 12 }; +// Never delete an entry used this recently: another worktree may be cloning it. +const IN_USE_MS = 30 * 60 * 1000; + +export function cacheRoot(env = process.env) { + if (env.PAPERCLIP_AGENT_CACHE === "0") return null; + if (env.PAPERCLIP_AGENT_CACHE_DIR) return path.resolve(env.PAPERCLIP_AGENT_CACHE_DIR); + return path.join(env.XDG_CACHE_HOME || path.join(os.homedir(), ".cache"), "paperclip", "agent-build"); +} + +function run(command, args, options = {}) { + try { + return execFileSync(command, args, { encoding: "utf8", stdio: ["ignore", "pipe", "pipe"], ...options }).trim(); + } catch { + return "unavailable"; + } +} + +/** Compiler, platform and the environment variables that change what rustc emits. */ +export function toolchainIdentity(root, env = process.env) { + const cwd = path.join(root, RUNNER_PACKAGE); + const buildEnv = Object.entries(env).filter(([name]) => + /^(?:RUSTFLAGS|RUSTDOCFLAGS|RUSTC|RUSTC_WRAPPER|RUSTC_WORKSPACE_WRAPPER|RUSTC_BOOTSTRAP|RUSTUP_TOOLCHAIN|RUST_TARGET_PATH|CC|CXX|CFLAGS|CXXFLAGS|LDFLAGS|MACOSX_DEPLOYMENT_TARGET)$/.test(name) + || /^CARGO_(?:BUILD|PROFILE|TARGET|ENCODED|INCREMENTAL)/.test(name)).sort(); + const cargoHome = env.CARGO_HOME || path.join(os.homedir(), ".cargo"); + const cargoConfig = ["config", "config.toml"].map(name => path.join(cargoHome, name)) + .map(file => existsSync(file) ? readFileSync(file, "utf8") : "missing"); + return { + platform: process.platform, arch: process.arch, node: process.version, + rustc: run("rustc", ["--version", "--verbose"], { cwd, env }), + cargo: run("cargo", ["--version"], { cwd, env }), + buildEnv, cargoConfig, + }; +} + +/** Hashes repo-relative files by path and content, so the key is the same in every checkout. */ +export function hashFiles(root, files, hash = createHash("sha256")) { + for (const file of [...new Set(files)].sort()) { + const absolute = path.join(root, file); + hash.update(file).update("\0"); + if (!existsSync(absolute)) hash.update("missing"); + else if (lstatSync(absolute).isSymbolicLink()) hash.update("link\0").update(readlinkSync(absolute)); + else hash.update(readFileSync(absolute)); + hash.update("\0"); + } + return hash; +} + +export function globFiles(root, patterns) { + return [...globSync(patterns, { cwd: root })] + .filter(file => !file.split(path.sep).includes("target") && statSync(path.join(root, file)).isFile()) + .map(file => file.split(path.sep).join("/")); +} + +export function makeKey(kind, parts) { + return createHash("sha256").update(JSON.stringify({ schema: CACHE_SCHEMA, kind, parts })).digest("hex").slice(0, 40); +} + +/** Copy-on-write clone of a file or directory tree, through a temporary sibling. */ +export function cloneInto(source, destination) { + const temporary = `${destination}.tmp-${randomUUID()}`; + mkdirSync(path.dirname(destination), { recursive: true }); + const attempts = process.platform === "darwin" ? [["-cRp"], ["-Rp"]] : [["-a", "--reflink=auto"], ["-a"]]; + let lastError; + for (const flags of attempts) { + try { + execFileSync("cp", [...flags, source, temporary], { stdio: ["ignore", "ignore", "pipe"] }); + renameSync(temporary, destination); + return; + } catch (error) { + lastError = error; + rmSync(temporary, { recursive: true, force: true }); + } + } + throw lastError; +} + +/** Moves an existing path out of the way and deletes it, so the name can be reused at once. */ +export function removeTree(target) { + if (!existsSync(target) && !isLink(target)) return; + // A hidden name, so pruning treats a leftover as a stale temporary. + const doomed = path.join(path.dirname(target), `.old-${path.basename(target)}-${randomUUID()}`); + renameSync(target, doomed); + rmSync(doomed, { recursive: true, force: true }); +} + +function isLink(target) { + try { return lstatSync(target).isSymbolicLink(); } catch { return false; } +} + +export function entryPath(root, kind, key) { + return path.join(root, kind, key); +} + +/** Reads a complete entry; touches it so pruning keeps entries in use. */ +export function readEntry(cache, kind, key) { + if (!cache) return null; + const directory = entryPath(cache, kind, key); + try { + const meta = JSON.parse(readFileSync(path.join(directory, "entry.json"), "utf8")); + if (meta.schema !== CACHE_SCHEMA || meta.kind !== kind || meta.key !== key) return null; + const now = new Date(); + try { utimesSync(directory, now, now); } catch {} + return { directory, meta }; + } catch { + return null; + } +} + +/** + * Publishes an entry: `populate(directory)` fills a private temporary + * directory, which is then renamed into place. When another process has + * published the same key first, its entry wins unless `replace` says the new + * one is better. Returns whether this call published. + */ +export function publishEntry(cache, kind, key, populate, { replace = () => false, meta = {} } = {}) { + if (!cache) return false; + // Check before copying anything, so an unchanged run does not clone a tree to discard it. + const current = readEntry(cache, kind, key); + if (current && !replace(current.meta)) return false; + const kindDirectory = path.join(cache, kind); + mkdirSync(kindDirectory, { recursive: true }); + const temporary = path.join(kindDirectory, `.tmp-${randomUUID()}`); + mkdirSync(temporary); + try { + populate(temporary); + writeFileSync(path.join(temporary, "entry.json"), `${JSON.stringify({ schema: CACHE_SCHEMA, kind, key, + createdAt: new Date().toISOString(), ...meta }, null, 2)}\n`); + const destination = entryPath(cache, kind, key); + const existing = readEntry(cache, kind, key); + if (existing && !replace(existing.meta)) return false; + if (existing) removeTree(destination); + try { + renameSync(temporary, destination); + } catch (error) { + if (["EEXIST", "ENOTEMPTY", "EPERM"].includes(error.code)) return false; + throw error; + } + pruneKind(cache, kind, key); + return true; + } finally { + rmSync(temporary, { recursive: true, force: true }); + } +} + +/** Keeps the most recently used entries of a kind and anything used in the last half hour. */ +export function pruneKind(cache, kind, keep, now = Date.now()) { + const kindDirectory = path.join(cache, kind); + let names; + try { names = readdirSync(kindDirectory); } catch { return; } + const entries = names.map(name => { + const full = path.join(kindDirectory, name); + try { return { name, full, mtime: statSync(full).mtimeMs }; } catch { return null; } + }).filter(Boolean).sort((a, b) => b.mtime - a.mtime); + const limit = KEEP_ENTRIES[kind] ?? 4; + entries.filter(entry => !entry.name.startsWith(".")).forEach((entry, index) => { + if (index >= limit && entry.mtime < now - IN_USE_MS && entry.name !== keep) { + try { removeTree(entry.full); } catch {} + } + }); + // Temporary directories left by an interrupted publish. + entries.filter(entry => entry.name.startsWith(".")).forEach(entry => { + const stale = entry.mtime < now - 24 * 60 * 60 * 1000; + if (stale) { + try { removeTree(entry.full); } catch {} + } + }); +} + +/** + * Deletes every unit of the workspace crate from a cargo target tree, so a + * seed made from it holds third-party crates only. `bins` are the crate's + * executable target names. + */ +export function scrubSeed(target, bins) { + const names = ["paperclip-runner-core", "paperclip_runner_core", ...bins.flatMap(bin => [bin, bin.replaceAll("-", "_")])]; + const owned = name => names.some(prefix => name === prefix || name.startsWith(`${prefix}-`) || name.startsWith(`${prefix}.`) + || name.startsWith(`lib${prefix}-`)); + let removed = 0; + for (const profile of readdirSync(target)) { + const profileDirectory = path.join(target, profile); + if (!statSync(profileDirectory).isDirectory()) continue; + if (!["debug", "release"].includes(profile)) { + rmSync(profileDirectory, { recursive: true, force: true }); + continue; + } + for (const name of readdirSync(profileDirectory)) { + const full = path.join(profileDirectory, name); + if ([".fingerprint", "deps", "incremental", "build", "examples"].includes(name)) { + for (const unit of readdirSync(full)) { + if (name === "incremental" || name === "examples" || owned(unit)) { + rmSync(path.join(full, unit), { recursive: true, force: true }); + removed += 1; + } + } + } else if (owned(name) || name.startsWith(".")) { + rmSync(full, { recursive: true, force: true }); + removed += 1; + } + } + } + return removed; +} + +/** True when no fingerprint of the workspace crate is left, so cargo must compile it. */ +export function seedIsScrubbed(target) { + for (const profile of ["debug", "release"]) { + const fingerprints = path.join(target, profile, ".fingerprint"); + if (!existsSync(fingerprints)) continue; + if (readdirSync(fingerprints).some(name => name.startsWith("paperclip-runner-core-"))) return false; + } + return true; +} diff --git a/scripts/agent-preflight.mjs b/scripts/agent-preflight.mjs new file mode 100644 index 00000000000..850a2109e88 --- /dev/null +++ b/scripts/agent-preflight.mjs @@ -0,0 +1,317 @@ +#!/usr/bin/env node +// One-command preflight for an agent worktree: install, the Runner build and +// the Runner's debug binaries, skipping work whose inputs are unchanged and +// reusing other worktrees' results through the content-addressed cache in +// scripts/agent-build-cache.mjs. +// +// It replaces this sequence, with the same results: +// pnpm install --frozen-lockfile +// pnpm --filter @paperclipai/paperclip-runner run ensure:eval-build-deps +// pnpm --filter @paperclipai/paperclip-runner run build:rust +// pnpm --filter @paperclipai/paperclip-runner build +// pnpm --filter @paperclipai/paperclip-runner run build:runner-binaries +// +// Steps: +// install skipped when the lockfile, manifests, patches and toolchain +// match the last successful install in this worktree +// runner build skipped when the Runner's own build stamp is current; +// restored from the cache when another worktree built the same +// inputs; otherwise built, then published +// debug bins the same, keyed by every file the Rust build reads +// A fresh worktree's cargo target directory starts as a clone of the shared +// third-party seed, so cargo compiles only the Runner crate. +// +// A restored Runner build skips the build's lint checks (protocol coverage, +// traceability and similar). They depend only on the same inputs, and CI runs +// them; run `pnpm --filter @paperclipai/paperclip-runner build` for them locally. +// +// Usage: node scripts/agent-preflight.mjs [--no-cache] [--force-install] [--skip-rust-debug] [--explain] + +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { + cacheRoot, cloneInto, EVAL_KERNEL_PACKAGE, globFiles, hashFiles, makeKey, publishEntry, readEntry, removeTree, + RUNNER_PACKAGE, scrubSeed, seedIsScrubbed, toolchainIdentity, +} from "./agent-build-cache.mjs"; +import { DEBUG_INPUTS, debugContract } from "./runner-debug-crate-cache.mjs"; +import { + assessRunnerBuild, buildInputsStampPath, collectBuildOutputFiles, computeBuildInputsDigest, expectedBuildOutputs, + findChangedBuildOutput, writeBuildInputsStamp, +} from "../packages/paperclip-runner/scripts/build-inputs.mjs"; + +const PREFLIGHT_SCRIPTS = ["scripts/agent-preflight.mjs", "scripts/agent-build-cache.mjs"]; +const SEED_INPUTS = [`${RUNNER_PACKAGE}/runner/Cargo.lock`, `${RUNNER_PACKAGE}/runner/Cargo.toml`, + `${RUNNER_PACKAGE}/runner/crates/runner-core/Cargo.toml`, `${RUNNER_PACKAGE}/rust-toolchain.toml`]; + +function log(message) { + console.log(`[agent:preflight] ${message}`); +} + +function exec(command, args, cwd, env = process.env) { + const started = Date.now(); + const result = spawnSync(command, args, { cwd, env, stdio: "inherit" }); + if (result.error) throw result.error; + if (result.status !== 0) throw new Error(`${command} ${args.join(" ")} failed with exit ${result.status ?? result.signal}`); + return (Date.now() - started) / 1000; +} + +function readJson(file) { + try { return JSON.parse(readFileSync(file, "utf8")); } catch { return null; } +} + +function writeJson(file, value) { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, `${JSON.stringify(value, null, 2)}\n`); +} + +function stateDirectory(root) { + return path.join(root, "node_modules", ".cache", "paperclip-agent-preflight"); +} + +// --- install --------------------------------------------------------------- + +export function installKey(root, toolchain) { + // Untracked, non-ignored manifests count too: a new workspace package needs an install. + const tracked = spawnSync("git", ["ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", "pnpm-lock.yaml", "pnpm-workspace.yaml", ".npmrc", + ":(glob)**/package.json", "patches"], { cwd: root, encoding: "utf8" }); + if (tracked.status !== 0) return null; + const files = tracked.stdout.split("\0").filter(Boolean); + const pnpm = spawnSync("pnpm", ["--version"], { cwd: root, encoding: "utf8" }).stdout?.trim(); + return hashFiles(root, files).update(JSON.stringify({ node: toolchain.node, platform: toolchain.platform, + arch: toolchain.arch, pnpm })).digest("hex"); +} + +function installStep(root, options, toolchain) { + const stampFile = path.join(stateDirectory(root), "install.json"); + const key = installKey(root, toolchain); + const modules = path.join(root, "node_modules", ".modules.yaml"); + const stamp = readJson(stampFile); + if (!options.forceInstall && key && existsSync(modules) && stamp?.key === key + && stamp.modulesMtimeMs === statSync(modules).mtimeMs) { + return { step: "install", action: "skipped", reason: "lockfile, manifests and toolchain unchanged" }; + } + const seconds = exec("pnpm", ["install", "--frozen-lockfile", "--prefer-offline"], root); + if (key) writeJson(stampFile, { key, modulesMtimeMs: statSync(modules).mtimeMs }); + return { step: "install", action: "ran", seconds }; +} + +// --- cargo seed ------------------------------------------------------------ + +function targetDirectory(root) { + return path.join(root, RUNNER_PACKAGE, "runner", "target"); +} + +function seedKey(root, toolchain) { + return makeKey("cargo-seed", { inputs: hashFiles(root, SEED_INPUTS).digest("hex"), toolchain }); +} + +function restoreSeed(root, cache, toolchain) { + const target = targetDirectory(root); + if (existsSync(target)) return { step: "cargo-seed", action: "kept", reason: "worktree already has a target directory" }; + const entry = readEntry(cache, "cargo-seed", seedKey(root, toolchain)); + if (!entry) return { step: "cargo-seed", action: "miss" }; + const started = Date.now(); + try { + cloneInto(path.join(entry.directory, "target"), target); + } catch (error) { + return { step: "cargo-seed", action: "failed", reason: error.message.split("\n")[0] }; + } + if (!seedIsScrubbed(target)) { + removeTree(target); + return { step: "cargo-seed", action: "discarded", reason: "seed held a workspace crate unit" }; + } + return { step: "cargo-seed", action: "restored", seconds: (Date.now() - started) / 1000, + profiles: entry.meta.profiles }; +} + +function publishSeed(root, cache, toolchain, bins) { + const target = targetDirectory(root); + const profiles = ["debug", "release"].filter(profile => existsSync(path.join(target, profile, "deps"))); + if (!cache || profiles.length === 0) return null; + const published = publishEntry(cache, "cargo-seed", seedKey(root, toolchain), directory => { + const copy = path.join(directory, "target"); + cloneInto(target, copy); + scrubSeed(copy, bins); + if (!seedIsScrubbed(copy)) throw new Error("seed still holds a workspace crate unit"); + }, { meta: { profiles }, replace: existing => profiles.some(profile => !existing.profiles?.includes(profile)) }); + return published ? { step: "cargo-seed", action: "published", profiles } : null; +} + +// --- Runner build (dist) --------------------------------------------------- + +const DIST_TREES = [`${RUNNER_PACKAGE}/dist`, `${EVAL_KERNEL_PACKAGE}/dist`]; + +function runnerDistKey(digest, toolchain) { + return makeKey("runner-dist", { digest, toolchain }); +} + +// Copilot assets live outside dist, and the build installs them after compiling. +// The installer is idempotent, so a skipped or restored build runs it too. +function installCopilotAssets(packageRoot) { + exec(process.execPath, ["scripts/install-copilot-assets.mjs"], packageRoot); +} + +function stampRestoredBuild(root, packageRoot, inputs, entryKey) { + const outputs = collectBuildOutputFiles(packageRoot); + const recordedAt = new Date().toISOString(); + writeBuildInputsStamp(buildInputsStampPath(packageRoot), { ...inputs, platform: process.platform, arch: process.arch, + startedAt: recordedAt, recordedAt, restoredFrom: entryKey, outputCount: outputs.length, outputs }); + return recordedAt; +} + +function runnerBuildStep(root, cache, toolchain) { + const packageRoot = path.join(root, RUNNER_PACKAGE); + const assessment = assessRunnerBuild({ repoRoot: root, packageRoot }); + const inputs = assessment.inputs ?? computeBuildInputsDigest(root); + const key = runnerDistKey(inputs.digest, toolchain); + // The Runner's stamp does not record the compiler or flags, so the preflight + // keeps its own record of the full key the current build was made with, + // tied to that build's stamp time. Any other Runner build rewrites the stamp + // and so breaks the tie: the next preflight rebuilds or restores. + const keyFile = path.join(stateDirectory(root), "runner-build.json"); + const record = readJson(keyFile); + if (assessment.current && record?.key === key && record.recordedAt === assessment.stamp.recordedAt) { + installCopilotAssets(packageRoot); + return { step: "runner-build", action: "skipped", reason: assessment.detail }; + } + rmSync(keyFile, { force: true }); + const entry = readEntry(cache, "runner-dist", key); + if (entry) { + const started = Date.now(); + let cloned = true; + try { + for (const tree of DIST_TREES) { + removeTree(path.join(root, tree)); + cloneInto(path.join(entry.directory, tree), path.join(root, tree)); + } + } catch (error) { + cloned = false; + log(`could not restore the cached Runner build: ${error.message.split("\n")[0]}`); + } + const changed = cloned && findChangedBuildOutput(packageRoot, entry.meta.outputs ?? []); + const missing = expectedBuildOutputs(packageRoot).filter(file => !existsSync(file)); + if (cloned && !changed && missing.length === 0 && computeBuildInputsDigest(root).digest === inputs.digest) { + installCopilotAssets(packageRoot); + writeJson(keyFile, { key, recordedAt: stampRestoredBuild(root, packageRoot, inputs, key) }); + return { step: "runner-build", action: "restored", seconds: (Date.now() - started) / 1000 }; + } + for (const tree of DIST_TREES) removeTree(path.join(root, tree)); + log("cached Runner build did not match; building"); + } + const seconds = exec("pnpm", ["--filter", "@paperclipai/paperclip-runner", "build"], root); + const after = assessRunnerBuild({ repoRoot: root, packageRoot }); + if (!after.current || after.stamp.digest !== inputs.digest) return { step: "runner-build", action: "built", seconds }; + writeJson(keyFile, { key, recordedAt: after.stamp.recordedAt }); + const published = publishEntry(cache, "runner-dist", key, directory => { + for (const tree of DIST_TREES) cloneInto(path.join(root, tree), path.join(directory, tree)); + }, { meta: { outputs: after.stamp.outputs } }); + return { step: "runner-build", action: "built", seconds, published }; +} + +// --- Runner debug binaries ------------------------------------------------- + +function debugBinsKey(root, toolchain) { + const files = [...globFiles(root, DEBUG_INPUTS), ...PREFLIGHT_SCRIPTS]; + return makeKey("runner-debug-bins", { inputs: hashFiles(root, files).digest("hex"), toolchain }); +} + +function binaryState(file) { + try { + const stat = statSync(file); + return { size: stat.size, mtimeMs: stat.mtimeMs }; + } catch { return null; } +} + +function debugBinsStep(root, cache, toolchain, bins) { + const debug = path.join(targetDirectory(root), "debug"); + const stampFile = path.join(debug, ".paperclip-agent-debug-bins.json"); + const key = debugBinsKey(root, toolchain); + const stamp = readJson(stampFile); + const states = () => Object.fromEntries(bins.map(bin => [bin, binaryState(path.join(debug, bin))])); + if (stamp?.key === key && bins.every(bin => stamp.bins?.[bin] && JSON.stringify(stamp.bins[bin]) + === JSON.stringify(binaryState(path.join(debug, bin))))) { + return { step: "debug-bins", action: "skipped", reason: "Rust inputs unchanged since this worktree's build" }; + } + const entry = readEntry(cache, "runner-debug-bins", key); + if (entry && bins.every(bin => existsSync(path.join(entry.directory, bin)))) { + const started = Date.now(); + try { + mkdirSync(debug, { recursive: true }); + for (const bin of bins) { + removeTree(path.join(debug, bin)); + cloneInto(path.join(entry.directory, bin), path.join(debug, bin)); + } + writeJson(stampFile, { key, bins: states() }); + return { step: "debug-bins", action: "restored", seconds: (Date.now() - started) / 1000 }; + } catch (error) { + // A concurrent prune can remove the entry mid-restore; build instead. + rmSync(stampFile, { force: true }); + log(`could not restore cached debug binaries: ${error.message.split("\n")[0]}`); + } + } + const seconds = exec("cargo", ["build", "--manifest-path", "runner/Cargo.toml", "--locked", "--workspace", "--bins"], + path.join(root, RUNNER_PACKAGE)); + if (debugBinsKey(root, toolchain) !== key) return { step: "debug-bins", action: "built", seconds, published: false }; + writeJson(stampFile, { key, bins: states() }); + const published = publishEntry(cache, "runner-debug-bins", key, directory => { + for (const bin of bins) cloneInto(path.join(debug, bin), path.join(directory, bin)); + }); + return { step: "debug-bins", action: "built", seconds, published }; +} + +// --- main ------------------------------------------------------------------ + +export function parseArguments(argv) { + const options = { cache: true, forceInstall: false, rustDebug: true, explain: false }; + for (const argument of argv) { + if (argument === "--no-cache") options.cache = false; + else if (argument === "--force-install") options.forceInstall = true; + else if (argument === "--skip-rust-debug") options.rustDebug = false; + else if (argument === "--explain") options.explain = true; + else throw new Error(`unknown argument ${argument}`); + } + return options; +} + +export function preflight({ root, options, env = process.env }) { + const started = Date.now(); + const results = []; + const record = result => { + if (result) { + results.push(result); + log(`${result.step}: ${result.action}${result.seconds === undefined ? "" : ` in ${result.seconds.toFixed(1)}s`}` + + `${result.reason ? ` (${result.reason})` : ""}${result.published ? "; published to the shared cache" : ""}`); + } + }; + // A custom target directory breaks the per-worktree layout the seed and + // binary steps assume; fall back to the plain build without the cache. + const cache = options.cache && !env.CARGO_TARGET_DIR && !env.CARGO_BUILD_TARGET_DIR ? cacheRoot(env) : null; + if (cache) mkdirSync(cache, { recursive: true }); + log(cache ? `shared cache ${cache}` : "shared cache disabled"); + + const toolchain = toolchainIdentity(root, env); + record(installStep(root, options, toolchain)); + const bins = debugContract(root).bins; + record(restoreSeed(root, cache, toolchain)); + record(runnerBuildStep(root, cache, toolchain)); + if (options.rustDebug) record(debugBinsStep(root, cache, toolchain, bins)); + record(publishSeed(root, cache, toolchain, bins)); + const seconds = (Date.now() - started) / 1000; + log(`done in ${seconds.toFixed(1)}s`); + if (options.explain) console.log(JSON.stringify({ cache, toolchain: createHash("sha256") + .update(JSON.stringify(toolchain)).digest("hex").slice(0, 12), results, seconds }, null, 2)); + return { results, seconds }; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + try { + preflight({ root, options: parseArguments(process.argv.slice(2)) }); + } catch (error) { + console.error(`[agent:preflight] ${error.message}`); + process.exitCode = 1; + } +}