import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { chmod, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, unlink, writeFile } from "node:fs/promises"; import { stripTypeScriptTypes } from "node:module"; import { dirname, isAbsolute, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { gunzipSync } from "node:zlib"; import { PI_DISTRIBUTION_CLOSURE_SHA256 } from "../src/drivers/acpx/pi-closure-pins.ts"; import { PI_NODE_DISTRIBUTIONS, PI_NODE_VERSION } from "../src/drivers/acpx/pi-node-pins.ts"; import { buildNodeStartupTimeout } from "./build-node-startup-timeout.mjs"; import { QUALIFIED_ACPX_PROFILES } from "../src/drivers/acpx/qualified-profiles.ts"; import { inventoryPiRuntimeFiles, verifyPiRuntimeManifest } from "../src/drivers/acpx/pi-verified-runtime.ts"; const run = promisify(execFile); const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const workspaceRoot = resolve(packageRoot, "../.."); const lockDirectory = join(packageRoot, "scripts/pi-distribution"); const patchPath = join(workspaceRoot, "patches/pi-acp@0.0.33.patch"); const supportedTargets = new Set(["darwin-arm64", "darwin-x64", "linux-x64"]); export const PI_DISTRIBUTION_PINS = Object.freeze({ wrapper: "0.0.33", runtime: "0.84.2", sdk: "0.26.0", zod: "3.25.76", nodeVersion: PI_NODE_VERSION, undici: "8.10.2", nodeBundledUndici: "7.29.1", wrapperSha256: "9fd9a685fb79b4f73a8dac08cbe360a6de87ffcb596cdaff1191e3ac57c24f56", helperSha256: "a19087c0af8fb3896fe3290280695f8f936574013bc4acde650f11f023f0f03b", }); const hash = (bytes) => createHash("sha256").update(bytes).digest("hex"); /** The sole reviewed departure from Pi 0.84.2's published shrinkwrap. * npm ci reifies that upstream shrinkwrap even when the outer lock and override * request the fixed version. Keep the published metadata intact, replace this * exact package explicitly, and bind the resulting files in all closure pins. * https://github.com/advisories/GHSA-3wwx-pv8p-q78v */ export const PI_UNDICI_SECURITY_OVERRIDE = Object.freeze({ path: "node_modules/@earendil-works/pi-coding-agent/node_modules/undici", upstreamPath: "node_modules/undici", previous: Object.freeze({ version: "8.9.0", resolved: "https://registry.npmjs.org/undici/-/undici-8.9.0.tgz", integrity: "sha512-aWZpUj7XoGonMClx4gdDRfgBjqeA+F473aDmROQQbM9n6PRfK/u1q/a0X4wMTgcHfT8H6fpbt98PFuDUwFg2YA==" }), fixed: Object.freeze({ version: "8.10.2", resolved: "https://registry.npmjs.org/undici/-/undici-8.10.2.tgz", integrity: "sha512-/y4/bH9YNU5hi9NIrpOuvGXFcxrj3CMrV+/AYpowAYTpHn8gX/XPFjNy766FPoYY0miQhdW977JFWKGNhBdwyQ==" }), }); const samePackagePin = (entry, pin) => entry?.version === pin.version && entry?.resolved === pin.resolved && entry?.integrity === pin.integrity; export function assertPiUndiciSecurityOverride(lock, shrinkwrap) { const pin = PI_UNDICI_SECURITY_OVERRIDE; if (shrinkwrap.version !== PI_DISTRIBUTION_PINS.runtime || shrinkwrap.lockfileVersion !== 3 || !samePackagePin(shrinkwrap.packages?.[pin.upstreamPath], pin.previous) || !samePackagePin(lock.packages?.[pin.path], pin.fixed)) throw new Error("Pi Undici security override differs from its exact reviewed old/new pins"); } export function assertPiUndiciArchive(bytes) { if (bytes.length > 4 * 1024 * 1024 || `sha512-${createHash("sha512").update(bytes).digest("base64")}` !== PI_UNDICI_SECURITY_OVERRIDE.fixed.integrity) throw new Error("Pi fixed Undici archive differs from its integrity pin"); // Bound the complete tar stream (headers and file payloads), not just gzip. gunzipSync(bytes, { maxOutputLength: 4 * 1024 * 1024 }); } export function assertPiUndiciArchiveEntries(paths, listing) { const entries = paths.trim().split("\n"); if (entries.length < 2 || entries.length > 512 || entries.some(path => !path.startsWith("package/") || path.includes("\\") || path.split("/").some(part => part === ".." || part === ".") || /[\x00-\x1f\x7f]/.test(path)) || listing.trim().split("\n").length !== entries.length || listing.trim().split("\n").some(line => !/^[d-]/.test(line))) throw new Error("Pi fixed Undici archive has an unsafe entry"); } export async function applyPiUndiciSecurityOverride(root, lock) { root = await realpath(root); const pin = PI_UNDICI_SECURITY_OVERRIDE; const piRoot = join(root, "node_modules/@earendil-works/pi-coding-agent"); const shrinkwrap = JSON.parse(await readFile(join(piRoot, "npm-shrinkwrap.json"), "utf8")); assertPiUndiciSecurityOverride(lock, shrinkwrap); const installed = join(root, pin.path); const metadata = JSON.parse(await readFile(join(installed, "package.json"), "utf8")); if ((await lstat(installed)).isSymbolicLink() || await realpath(installed) !== installed || metadata.name !== "undici" || metadata.version !== pin.previous.version) throw new Error("Pi npm install did not produce the expected upstream Undici package"); const temporary = await mkdtemp(join(dirname(installed), ".paperclip-undici-fix-")); try { const response = await fetch(pin.fixed.resolved, { redirect: "error", signal: AbortSignal.timeout(60_000) }); if (!response.ok || !response.body) throw new Error("Pi fixed Undici download failed"); const chunks = []; let length = 0; for await (const chunk of response.body) { length += chunk.length; if (length > 4 * 1024 * 1024) throw new Error("Pi fixed Undici archive exceeds its bound"); chunks.push(Buffer.from(chunk)); } const bytes = Buffer.concat(chunks); assertPiUndiciArchive(bytes); const archive = join(temporary, "fixed.tgz"); await writeFile(archive, bytes); const options = { env: { PATH: "/usr/bin:/bin" }, timeout: 30_000, maxBuffer: 1024 * 1024 }; const [paths, listing] = await Promise.all([run("tar", ["-tzf", archive], options), run("tar", ["-tvzf", archive], options)]); assertPiUndiciArchiveEntries(paths.stdout, listing.stdout); const extracted = join(temporary, "package"); await mkdir(extracted); await run("tar", ["-xzf", archive, "-C", extracted, "--strip-components=1", "--no-same-owner"], options); const fixed = JSON.parse(await readFile(join(extracted, "package.json"), "utf8")); if (fixed.name !== "undici" || fixed.version !== pin.fixed.version) throw new Error("Pi fixed Undici package identity is invalid"); const files = await inventoryPiRuntimeFiles(extracted); if (files.length !== 214 || files.some(file => file.kind !== "file")) throw new Error("Pi fixed Undici package payload is invalid"); await rm(installed, { recursive: true }); await rename(extracted, installed); } finally { await rm(temporary, { recursive: true, force: true }); } } /** npm ci honors Pi's published nested shrinkwrap; the exact security fix follows. */ export function piDistributionInstallCommand() { return ["ci", "--ignore-scripts", "--include=optional", "--omit=dev", "--no-audit", "--no-fund", "--registry=https://registry.npmjs.org", "--userconfig=.npmrc", "--globalconfig=.npmrc-global"]; } function supports(list, current) { return !Array.isArray(list) || (list.includes(current) || !list.some((value) => !value.startsWith("!"))) && !list.includes(`!${current}`); } /** Prove installed package versions and the complete published Pi graph. */ export async function verifyLockedPiPackageGraph(root, lock, target = { platform: process.platform, architecture: process.arch }) { if (lock.lockfileVersion !== 3 || !lock.packages || typeof lock.packages !== "object") throw new Error("Pi distribution lock is invalid"); let verified = 0; for (const [path, entry] of Object.entries(lock.packages)) { if (!path) continue; if (!path.startsWith("node_modules/") || path.split("/").some((part) => part === ".." || part === "." || !part) || path.includes("\\") || entry.link || typeof entry.version !== "string" || !/^https:\/\/registry\.npmjs\.org\//.test(entry.resolved ?? "") || !/^sha512-[A-Za-z0-9+/]+=*$/.test(entry.integrity ?? "")) throw new Error("Pi distribution lock contains an unpinned package"); const compatible = supports(entry.os, target.platform) && supports(entry.cpu, target.architecture); let metadata; try { metadata = JSON.parse(await readFile(join(root, path, "package.json"), "utf8")); } catch (error) { if (error.code === "ENOENT" && entry.optional === true && !compatible) continue; throw new Error(`Pi distribution is missing locked package ${path}`); } if (metadata.version !== entry.version) throw new Error(`Pi distribution package differs from its lock: ${path}`); verified++; } const piPath = "node_modules/@earendil-works/pi-coding-agent"; const shrinkwrap = JSON.parse(await readFile(join(root, piPath, "npm-shrinkwrap.json"), "utf8")); if (shrinkwrap.version !== PI_DISTRIBUTION_PINS.runtime || shrinkwrap.lockfileVersion !== 3) throw new Error("Pi upstream shrinkwrap is missing or changed"); for (const [path, entry] of Object.entries(shrinkwrap.packages)) { if (!path) continue; const pinned = lock.packages[`${piPath}/${path}`]; if (path === PI_UNDICI_SECURITY_OVERRIDE.upstreamPath) { assertPiUndiciSecurityOverride(lock, shrinkwrap); continue; } if (!pinned || pinned.version !== entry.version || (entry.integrity !== undefined && pinned.integrity !== entry.integrity) || pinned.resolved !== entry.resolved) throw new Error(`Pi distribution dropped or changed shrinkwrapped package ${path}`); } return verified; } /** Only OS libraries may remain outside the inventoried Node executable. */ export function assertPiNodeSystemDependencies(listing, platform) { const lines = listing.split("\n").map((line) => line.trim()).filter(Boolean); if (platform === "darwin") { for (const line of lines.slice(1)) { const path = line.split(" (", 1)[0]; if (!path.startsWith("/usr/lib/") && !path.startsWith("/System/Library/")) throw new Error("Pi Node requires an unbundled non-system library"); } if (lines.length < 2) throw new Error("Pi Node dependency inspection is empty"); } else { if (listing.includes("not found")) throw new Error("Pi Node requires a missing system library"); for (const line of lines) { if (line.startsWith("linux-vdso.") || line === "statically linked") continue; const path = line.includes(" => ") ? line.split(" => ")[1].split(" ")[0] : line.split(" ")[0]; const name = path.slice(path.lastIndexOf("/") + 1); if (!/^\/(?:usr\/)?lib(?:64)?\//.test(path) || !/^(?:lib(?:c|m|dl|pthread|gcc_s|stdc\+\+|rt|atomic)\.so(?:\.[0-9]+)*|ld-linux[^/]*\.so(?:\.[0-9]+)*)$/.test(name)) throw new Error("Pi Node requires an unbundled non-system library"); } if (!lines.length) throw new Error("Pi Node dependency inspection is empty"); } } export async function writePiDistributionManifest(runtimeRoot) { const manifest = { schema: "paperclip.pi-runtime-files.v1", node: "node/bin/node", piEntrypoint: "node_modules/@earendil-works/pi-coding-agent/dist/cli.js", extension: "extensions/paperclip.js", wrapperEntrypoint: "node_modules/pi-acp/dist/index.js", files: await inventoryPiRuntimeFiles(runtimeRoot), }; const verified = await verifyPiRuntimeManifest(runtimeRoot, manifest); return { manifest, ...verified }; } export function piDistributionBootstrapSource() { return [ 'const path=require("node:path");', 'process.env.PAPERCLIP_PI_NODE_EXECUTABLE=path.join(__dirname,"node/bin/node");', 'process.env.PAPERCLIP_PI_ENTRYPOINT=path.join(__dirname,"node_modules/@earendil-works/pi-coding-agent/dist/cli.js");', 'process.env.PAPERCLIP_PI_EXTENSION_PATH=path.join(__dirname,"extensions/paperclip.js");', 'process.env.PAPERCLIP_PI_MODULE_GUARD_PATH=path.join(__dirname,".paperclip-native-module-guard.cjs");', 'const protectedRoots=JSON.parse(process.env.PAPERCLIP_PI_PROTECTED_ROOTS??"[]");if(!Array.isArray(protectedRoots))throw new Error("Invalid Pi protected roots");', 'process.env.PAPERCLIP_PI_PROTECTED_ROOTS=JSON.stringify([...protectedRoots,__dirname]);', 'import(require("node:url").pathToFileURL(path.join(__dirname,"node_modules/pi-acp/dist/index.js")).href).catch(()=>{process.exitCode=1;});', '', ].join("\n"); } /** Build on the target platform. No lifecycle scripts, model requests, or auth. */ export async function materializePiDistribution({ outputRoot, nodeExecutable, npmExecutable = "npm" }) { if (typeof outputRoot !== "string" || !outputRoot || !isAbsolute(outputRoot)) throw new Error("Pi distribution output must be absolute"); const output = resolve(outputRoot); if (["/", workspaceRoot, packageRoot].includes(output)) throw new Error("Refusing unsafe Pi distribution output"); if (!supportedTargets.has(`${process.platform}-${process.arch}`)) throw new Error("Pi distribution target is unsupported"); try { await lstat(output); throw new Error("Pi distribution output already exists"); } catch (error) { if (error.code !== "ENOENT") throw error; } await mkdir(dirname(output), { recursive: true }); const staging = await mkdtemp(join(await realpath(dirname(output)), ".paperclip-pi-distribution-")); const runtimeRoot = join(staging, "runtime"); try { const target = `${process.platform}-${process.arch}`; const nodePin = PI_NODE_DISTRIBUTIONS[target]; let node; if (nodeExecutable) node = await realpath(nodeExecutable); else { const archiveName = `node-v${PI_NODE_VERSION}-${target}.tar.gz`; const response = await fetch(`https://nodejs.org/dist/v${PI_NODE_VERSION}/${archiveName}`, { redirect: "error", signal: AbortSignal.timeout(60_000) }); if (!response.ok || !response.body) throw new Error("Pinned Pi Node download failed"); const chunks = []; let length = 0; for await (const chunk of response.body) { length += chunk.length; if (length > 128 * 1024 * 1024) throw new Error("Pi Node archive exceeds its bound"); chunks.push(Buffer.from(chunk)); } const bytes = Buffer.concat(chunks); if (hash(bytes) !== nodePin.archiveSha256) throw new Error("Pi Node archive does not match its release pin"); const archivePath = join(staging, archiveName); await writeFile(archivePath, bytes); const nodeRoot = join(staging, "node-extract"); await mkdir(nodeRoot); await run("tar", ["-xzf", archivePath, "-C", nodeRoot, "--strip-components=2", `node-v${PI_NODE_VERSION}-${target}/bin/node`], { timeout: 30_000 }); node = join(nodeRoot, "node"); } if (hash(await readFile(node)) !== nodePin.executableSha256 || (await lstat(node)).size !== nodePin.executableSize) throw new Error("Pi Node executable does not match its target release pin"); const version = (await run(node, ["--version"], { env: {}, timeout: buildNodeStartupTimeout() })).stdout.trim(); if (version !== `v${PI_NODE_VERSION}`) throw new Error("Pi distribution requires exact pinned Node version"); if ((await run(node, ["-p", "process.versions.undici"], { env: {}, timeout: buildNodeStartupTimeout() })).stdout.trim() !== PI_DISTRIBUTION_PINS.nodeBundledUndici) throw new Error("Pi Node bundled Undici differs from its reviewed security pin"); await mkdir(runtimeRoot); await Promise.all(["package.json", "package-lock.json"].map((name) => copyFile(join(lockDirectory, name), join(runtimeRoot, name)))); await writeFile(join(runtimeRoot, ".npmrc"), "registry=https://registry.npmjs.org/\nignore-scripts=true\naudit=false\nfund=false\n"); await writeFile(join(runtimeRoot, ".npmrc-global"), ""); const buildHome = join(staging, "build-home"); await mkdir(buildHome); // Do not inherit NPM_TOKEN, npm_config_*, NODE_OPTIONS, provider keys or user // .npmrc. Public registry downloads need no private application credential. const environment = Object.fromEntries(["PATH", "LANG", "LC_ALL", "HTTPS_PROXY", "HTTP_PROXY", "NO_PROXY", "SSL_CERT_FILE", "SSL_CERT_DIR", "NODE_EXTRA_CA_CERTS"].flatMap((key) => typeof process.env[key] === "string" ? [[key, process.env[key]]] : [])); environment.HOME = buildHome; await run(npmExecutable, piDistributionInstallCommand(), { cwd: runtimeRoot, env: environment, timeout: 300_000, maxBuffer: 4 * 1024 * 1024 }); const installedLockBytes = await readFile(join(runtimeRoot, "package-lock.json")); if (!installedLockBytes.equals(await readFile(join(lockDirectory, "package-lock.json")))) throw new Error("Pi installation changed its committed lock"); const lock = JSON.parse(installedLockBytes.toString("utf8")); await applyPiUndiciSecurityOverride(runtimeRoot, lock); const packageCount = await verifyLockedPiPackageGraph(runtimeRoot, lock); const wrapper = join(runtimeRoot, "node_modules/pi-acp"); await run("git", ["apply", "--check", patchPath], { cwd: wrapper, env: environment, timeout: 10_000 }); await run("git", ["apply", patchPath], { cwd: wrapper, env: environment, timeout: 10_000 }); const [wrapperBytes, helperBytes, helperSource] = await Promise.all([ readFile(join(wrapper, "dist/index.js")), readFile(join(wrapper, "dist/paperclip-runtime.js")), readFile(join(packageRoot, "src/drivers/acpx/pi-acp-runtime.ts"), "utf8"), ]); const stripped = stripTypeScriptTypes(helperSource).split("\n").map((line) => line.trimEnd()).join("\n"); if (hash(wrapperBytes) !== PI_DISTRIBUTION_PINS.wrapperSha256 || hash(helperBytes) !== PI_DISTRIBUTION_PINS.helperSha256 || helperBytes.toString("utf8") !== stripped) throw new Error("Pi wrapper patch does not match its qualified source"); await mkdir(join(runtimeRoot, "extensions")); await writeFile(join(runtimeRoot, "extensions/paperclip.js"), stripTypeScriptTypes(await readFile(join(packageRoot, "src/drivers/acpx/pi-runtime-extension.ts"), "utf8")).replace('from "./pi-acp-runtime.js"', 'from "../node_modules/pi-acp/dist/paperclip-runtime.js"')); await mkdir(join(runtimeRoot, "node/bin"), { recursive: true }); const copiedNode = join(runtimeRoot, "node/bin/node"); await copyFile(node, copiedNode); await chmod(copiedNode, 0o755); const dependencyListing = process.platform === "darwin" ? (await run("/usr/bin/otool", ["-L", copiedNode], { env: {}, timeout: 10_000 })).stdout : (await run("ldd", [copiedNode], { env: { PATH: "/usr/bin:/bin" }, timeout: 10_000 })).stdout; assertPiNodeSystemDependencies(dependencyListing, process.platform); if ((await run(copiedNode, ["--version"], { env: {}, timeout: buildNodeStartupTimeout() })).stdout.trim() !== version) throw new Error("Copied Pi Node cannot execute after relocation"); await rm(buildHome, { recursive: true }); // npm-generated .bin links and hidden lock metadata are not package payload // files. No launch uses PATH; excluding them makes the closure regular-only. for (const file of await inventoryPiRuntimeFiles(runtimeRoot)) { if (file.kind === "symlink") { if (!/(?:^|\/)node_modules\/\.bin\/[^/]+$/.test(file.path)) throw new Error("Pi package payload contains an unsupported symbolic link"); await unlink(join(runtimeRoot, file.path)); } } await rm(join(runtimeRoot, "node_modules/.package-lock.json"), { force: true }); await writeFile(join(runtimeRoot, "pi-entry.cjs"), piDistributionBootstrapSource()); for (const item of await readdir(staging)) if (item !== "runtime") await rm(join(staging, item), { recursive: true, force: true }); const { manifest, manifestDigest } = await writePiDistributionManifest(runtimeRoot); const entries = await Promise.all(manifest.files.map(async (file) => { if (file.kind !== "file") throw new Error("Pi native closure must contain regular files only"); const stat = await lstat(join(runtimeRoot, file.path)); return { path: file.path, sha256: file.sha256.slice("sha256:".length), size: stat.size, executable: Boolean(stat.mode & 0o111) }; })); entries.sort((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0); const nativeClosureSha256 = hash(JSON.stringify(entries)); if (nativeClosureSha256 !== PI_DISTRIBUTION_CLOSURE_SHA256[target]) throw new Error("Pi native closure differs from its trusted target pin"); await writeFile(join(staging, "native-closure.json"), `${JSON.stringify({ entries })}\n`); const metadata = { schema: "paperclip.pi-distribution.v1", pins: PI_DISTRIBUTION_PINS, target: { platform: process.platform, architecture: process.arch, nodeVersion: version.slice(1) }, packageCount, lockSha256: hash(await readFile(join(runtimeRoot, "package-lock.json"))), manifestDigest, runtimeRoot: "runtime", nativeClosureSha256, manifest, }; await writeFile(join(staging, "pi-distribution.json"), `${JSON.stringify(metadata, null, 2)}\n`); await rename(staging, output); const finalRoot = join(output, "runtime"); const binding = await verifyPiRuntimeManifest(finalRoot, manifest); return { version: PI_DISTRIBUTION_PINS.runtime, profileDigest: QUALIFIED_ACPX_PROFILES.pi.commandDigest, closureDigest: `sha256:${nativeClosureSha256}`, outputRoot: output, runtimeRoot: finalRoot, manifestPath: join(output, "pi-distribution.json"), metadata, ...binding, }; } catch (error) { await rm(staging, { recursive: true, force: true }); throw error; } } if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { const args = process.argv.slice(2).filter((arg) => arg !== "--"); const outputArgs = args.filter((arg) => !arg.startsWith("--node=")); const nodeArgs = args.filter((arg) => arg.startsWith("--node=")); if (outputArgs.length !== 1 || nodeArgs.length > 1) throw new Error("Usage: node scripts/materialize-pi-distribution.mjs /absolute/output-directory [--node=/absolute/portable-node]"); const result = await materializePiDistribution({ outputRoot: outputArgs[0], ...(nodeArgs.length ? { nodeExecutable: nodeArgs[0].slice("--node=".length) } : {}) }); process.stdout.write(`${JSON.stringify({ outputRoot: result.outputRoot, manifestPath: result.manifestPath, manifestDigest: result.manifestDigest, packageCount: result.metadata.packageCount })}\n`); }