diff --git a/packages/paperclip-runner/scripts/build-provider-pack.mjs b/packages/paperclip-runner/scripts/build-provider-pack.mjs index 657e890aaa6..585950b6a46 100644 --- a/packages/paperclip-runner/scripts/build-provider-pack.mjs +++ b/packages/paperclip-runner/scripts/build-provider-pack.mjs @@ -22,7 +22,7 @@ import { fileURLToPath } from "node:url"; import { parseProviderPackArguments, materializeCandidateProviderPack, providerPackProviders, providerPackManifestFields } from "./candidate-provider-pack.mjs"; import { buildNodeStartupTimeout } from "./build-node-startup-timeout.mjs"; import { writePortableExecutableShim, writePortableCopilotShims } from "./provider-pack-executable-shims.mjs"; -import { removeControllerArtifactsFromProviderPack } from "./provider-pack-runtime-tree.mjs"; +import { providerPackLibc, removeControllerArtifactsFromProviderPack, removeForeignLibcPackagesFromProviderPack } from "./provider-pack-runtime-tree.mjs"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const workspaceRoot = resolve(packageRoot, "../.."); @@ -119,6 +119,10 @@ try { throw new Error(`pnpm deploy failed with exit code ${deployed.status}`); } removeControllerArtifactsFromProviderPack(temporaryRoot); + const foreignLibcPackages = removeForeignLibcPackagesFromProviderPack(temporaryRoot, providerPackLibc()); + if (foreignLibcPackages.length) { + process.stderr.write(`Removed ${foreignLibcPackages.length} provider-pack packages built for another libc: ${foreignLibcPackages.join(", ")}\n`); + } // Fail the image build if a bridge silently brings back an older/private // provider CLI. A direct dependency alone does not deduplicate pnpm's graph. diff --git a/packages/paperclip-runner/scripts/provider-pack-executable-shims.mjs b/packages/paperclip-runner/scripts/provider-pack-executable-shims.mjs index 49e5ffbe19c..bb31b862292 100644 --- a/packages/paperclip-runner/scripts/provider-pack-executable-shims.mjs +++ b/packages/paperclip-runner/scripts/provider-pack-executable-shims.mjs @@ -1,6 +1,8 @@ -import { chmodSync, existsSync, realpathSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import { join, relative } from "node:path"; +import { pnpmStoreEntryOf, removePnpmStoreEntries } from "./provider-pack-runtime-tree.mjs"; export function writePortableExecutableShim(packRoot, name, executable) { const shimPath = join(packRoot, "node_modules", ".bin", name); @@ -17,18 +19,27 @@ export function writePortableExecutableShim(packRoot, name, executable) { chmodSync(shimPath, 0o755); } +const sha256 = path => createHash("sha256").update(readFileSync(path)).digest("hex"); + export function writePortableCopilotShims(packRoot) { const packRequire = createRequire(join(packRoot, "package.json")); + const nodeModules = realpathSync(join(packRoot, "node_modules")); // Optional platform packages expose native .bin entries. Rewrite only the // installed entries, and resolve their pinned executable without running it. for (const platform of ["darwin-arm64", "darwin-x64", "linux-x64"]) { const name = `copilot-${platform}`; if (!existsSync(join(packRoot, "node_modules", ".bin", name))) continue; - const executable = packRequire.resolve(`@github/${name}`); - writePortableExecutableShim( - packRoot, - name, - relative(realpathSync(join(packRoot, "node_modules")), realpathSync(executable)), - ); + const executable = realpathSync(packRequire.resolve(`@github/${name}`)); + // The pack's verified Copilot asset is built from the same pinned npm + // archive. When its executable is byte-identical, keep the command name but + // point it at that asset and drop the second ~160 MB copy from node_modules. + const asset = join(packRoot, "provider-assets", "copilot", platform, "copilot"); + if (existsSync(asset) && sha256(asset) === sha256(executable)) { + const entry = pnpmStoreEntryOf(packRoot, executable); + writePortableExecutableShim(packRoot, name, relative(nodeModules, realpathSync(asset))); + removePnpmStoreEntries(packRoot, [entry]); + continue; + } + writePortableExecutableShim(packRoot, name, relative(nodeModules, executable)); } } diff --git a/packages/paperclip-runner/scripts/provider-pack-executable-shims.test.mjs b/packages/paperclip-runner/scripts/provider-pack-executable-shims.test.mjs index df8aa94f0d3..4e7049bd861 100644 --- a/packages/paperclip-runner/scripts/provider-pack-executable-shims.test.mjs +++ b/packages/paperclip-runner/scripts/provider-pack-executable-shims.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -57,3 +57,53 @@ test("fails when an installed command has no matching package", (t) => { writeFileSync(join(pack, "node_modules", ".bin", "copilot-linux-x64"), "invalid old shim"); assert.throws(() => writePortableCopilotShims(pack), { code: "MODULE_NOT_FOUND" }); }); + +function pnpmCopilot(pack, platform, executable) { + const name = `copilot-${platform}`; + const entry = `@github+${name}@1.0.88`; + const installed = join(pack, "node_modules", ".pnpm", entry, "node_modules", "@github", name); + mkdirSync(installed, { recursive: true }); + writeFileSync(join(installed, "package.json"), JSON.stringify({ + name: `@github/${name}`, exports: { ".": "./copilot" }, bin: { [name]: "copilot" }, + })); + writeFileSync(join(installed, "copilot"), executable, { mode: 0o755 }); + mkdirSync(join(pack, "node_modules", "@github"), { recursive: true }); + symlinkSync(`../.pnpm/${entry}/node_modules/@github/${name}`, join(pack, "node_modules", "@github", name)); + writeFileSync(join(pack, "node_modules", ".bin", name), "#!/bin/sh\nexit 99\n"); + return { name, entry: join(pack, "node_modules", ".pnpm", entry) }; +} + +function providerAsset(pack, platform, executable) { + const directory = join(pack, "provider-assets", "copilot", platform); + mkdirSync(directory, { recursive: true }); + writeFileSync(join(directory, "copilot"), executable, { mode: 0o755 }); +} + +test("points the Copilot command at the identical provider asset and drops the npm copy", (t) => { + const { root, pack } = fixture(t); + const executable = '#!/bin/sh\nprintf "asset %s\\n" "$@"\nexit 17\n'; + const { name, entry } = pnpmCopilot(pack, "linux-x64", executable); + providerAsset(pack, "linux-x64", executable); + + writePortableCopilotShims(pack); + assert.equal(existsSync(entry), false); + assert.equal(existsSync(join(pack, "node_modules", "@github")), false); + const relocated = join(root, "relocated pack"); + renameSync(pack, relocated); + const result = spawnSync(join(relocated, "node_modules", ".bin", name), ["one argument"], { + cwd: root, env: { PATH: "/usr/bin:/bin" }, encoding: "utf8", timeout: 5_000, + }); + assert.equal(result.status, 17); + assert.equal(result.stdout, "asset one argument\n"); +}); + +test("keeps the npm Copilot copy when the provider asset differs", (t) => { + const { pack } = fixture(t); + const { name, entry } = pnpmCopilot(pack, "linux-x64", "#!/bin/sh\necho npm\n"); + providerAsset(pack, "linux-x64", "#!/bin/sh\necho asset\n"); + + writePortableCopilotShims(pack); + assert.equal(existsSync(entry), true); + const result = spawnSync(join(pack, "node_modules", ".bin", name), [], { env: { PATH: "/usr/bin:/bin" }, encoding: "utf8", timeout: 5_000 }); + assert.equal(result.stdout, "npm\n"); +}); diff --git a/packages/paperclip-runner/scripts/provider-pack-runtime-tree.mjs b/packages/paperclip-runner/scripts/provider-pack-runtime-tree.mjs index 5080335ef8f..39248ce8e2a 100644 --- a/packages/paperclip-runner/scripts/provider-pack-runtime-tree.mjs +++ b/packages/paperclip-runner/scripts/provider-pack-runtime-tree.mjs @@ -1,8 +1,90 @@ -import { rmSync } from "node:fs"; -import { join } from "node:path"; +import { existsSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmdirSync, rmSync, unlinkSync } from "node:fs"; +import { basename, dirname, join, relative, resolve, sep } from "node:path"; export function removeControllerArtifactsFromProviderPack(packRoot) { for (const directory of ["bin", "remote-provider-packs"]) { rmSync(join(packRoot, "dist", directory), { recursive: true, force: true }); } } + +/** libc family of the Node copied into the pack; undefined where npm has no libc field. */ +export function providerPackLibc(platform = process.platform, report = () => process.report.getReport()) { + if (platform !== "linux") return undefined; + return report().header.glibcVersionRuntime ? "glibc" : "musl"; +} + +function ownPackageManifests(entryRoot) { + // In pnpm's virtual store an entry's own package is the only real directory + // under its node_modules; every dependency beside it is a symlink. + const modules = join(entryRoot, "node_modules"); + if (!existsSync(modules)) return []; + const manifests = []; + for (const child of readdirSync(modules, { withFileTypes: true })) { + const children = child.isDirectory() && child.name.startsWith("@") + ? readdirSync(join(modules, child.name), { withFileTypes: true }).map(scoped => ({ entry: scoped, path: join(modules, child.name, scoped.name) })) + : [{ entry: child, path: join(modules, child.name) }]; + for (const { entry, path } of children) { + if (entry.isDirectory() && !entry.name.startsWith(".") && existsSync(join(path, "package.json"))) { + manifests.push(JSON.parse(readFileSync(join(path, "package.json"), "utf8"))); + } + } + } + return manifests; +} + +/** + * Remove whole pnpm virtual-store entries and every link that resolves into + * them, so no dangling dependency slot survives into the immutable pack. + */ +export function removePnpmStoreEntries(packRoot, entryNames) { + const store = join(packRoot, "node_modules", ".pnpm"); + const removed = entryNames.map(name => { + const path = resolve(store, name); + if (dirname(path) !== store || name === "node_modules") throw new Error(`Refusing to remove pnpm store path ${name}`); + return path; + }); + if (!removed.length) return; + for (const path of removed) rmSync(path, { recursive: true, force: true }); + const inside = target => removed.some(path => target === path || target.startsWith(`${path}${sep}`)); + const visit = directory => { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isSymbolicLink()) { + if (inside(resolve(directory, readlinkSync(path)))) unlinkSync(path); + } else if (entry.isDirectory()) { + visit(path); + if (entry.name.startsWith("@") && readdirSync(path).length === 0) rmdirSync(path); + } + } + }; + visit(join(packRoot, "node_modules")); +} + +/** + * pnpm keeps optional platform packages for every libc on the target OS/CPU. + * The pack only runs its own copied Node, so packages built for the other libc + * can never load; the Claude and OpenCode musl builds add 400-600 MB per pack. + */ +export function removeForeignLibcPackagesFromProviderPack(packRoot, libc) { + if (!libc) return []; + const store = join(packRoot, "node_modules", ".pnpm"); + const foreign = readdirSync(store, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && entry.name !== "node_modules") + .map(entry => entry.name) + .filter(name => { + const manifests = ownPackageManifests(join(store, name)); + return manifests.length > 0 && manifests.every(manifest => Array.isArray(manifest.libc) && !manifest.libc.includes(libc)); + }) + .sort(); + removePnpmStoreEntries(packRoot, foreign); + return foreign; +} + +/** The pnpm virtual-store entry that owns a resolved package path inside the pack. */ +export function pnpmStoreEntryOf(packRoot, path) { + const parts = relative(realpathSync(join(packRoot, "node_modules", ".pnpm")), realpathSync(path)).split(sep); + if (parts.length < 2 || parts[0] === ".." || parts[0] === "node_modules" || basename(parts[0]) !== parts[0]) { + throw new Error(`${path} is not inside the provider pack's pnpm store`); + } + return parts[0]; +} diff --git a/packages/paperclip-runner/scripts/provider-pack-runtime-tree.test.mjs b/packages/paperclip-runner/scripts/provider-pack-runtime-tree.test.mjs index 83c64819f68..600163d7a8d 100644 --- a/packages/paperclip-runner/scripts/provider-pack-runtime-tree.test.mjs +++ b/packages/paperclip-runner/scripts/provider-pack-runtime-tree.test.mjs @@ -1,9 +1,14 @@ import assert from "node:assert/strict"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { dirname, join, relative } from "node:path"; import test from "node:test"; -import { removeControllerArtifactsFromProviderPack } from "./provider-pack-runtime-tree.mjs"; +import { + providerPackLibc, + removeControllerArtifactsFromProviderPack, + removeForeignLibcPackagesFromProviderPack, + removePnpmStoreEntries, +} from "./provider-pack-runtime-tree.mjs"; test("image and native producer packs retain identical provider outputs despite different controller artifacts", () => { const root = mkdtempSync(join(tmpdir(), "provider-pack-producers-")); @@ -37,3 +42,85 @@ test("removing an owned controller-artifact link preserves its external target", assert.equal(readFileSync(join(outside, "preserved"), "utf8"), "original"); } finally { rmSync(root, { recursive: true, force: true }); } }); + +function pnpmPackage(pack, entry, name, manifest = {}) { + const path = join(pack, "node_modules", ".pnpm", entry, "node_modules", name); + mkdirSync(path, { recursive: true }); + writeFileSync(join(path, "package.json"), JSON.stringify({ name, version: "1.0.0", ...manifest })); + writeFileSync(join(path, "binary"), `${name}\n`); + return path; +} + +function link(pack, from, entry, name) { + const path = join(pack, "node_modules", from, name); + mkdirSync(dirname(path), { recursive: true }); + symlinkSync(relative(dirname(path), join(pack, "node_modules", ".pnpm", entry, "node_modules", name)), path); + return path; +} + +test("detects the libc of the Node copied into Linux packs only", () => { + assert.equal(providerPackLibc("linux", () => ({ header: { glibcVersionRuntime: "2.39" } })), "glibc"); + assert.equal(providerPackLibc("linux", () => ({ header: {} })), "musl"); + assert.equal(providerPackLibc("darwin", () => { throw new Error("unused"); }), undefined); +}); + +test("removes packages built only for another libc and every link into them", () => { + const root = mkdtempSync(join(tmpdir(), "provider-pack-libc-")); + try { + const pack = join(root, "pack"); + pnpmPackage(pack, "@scope+native-linux-x64@1.0.0", "@scope/native-linux-x64", { libc: ["glibc"] }); + pnpmPackage(pack, "@scope+native-linux-x64-musl@1.0.0", "@scope/native-linux-x64-musl", { libc: ["musl"] }); + pnpmPackage(pack, "tool-linux-x64-musl@1.0.0", "tool-linux-x64-musl", { libc: ["musl"] }); + pnpmPackage(pack, "tool-linux-x64@1.0.0", "tool-linux-x64"); + pnpmPackage(pack, "consumer@1.0.0", "consumer", { optionalDependencies: { "tool-linux-x64": "1.0.0", "tool-linux-x64-musl": "1.0.0" } }); + const kept = [ + link(pack, ".pnpm/consumer@1.0.0/node_modules", "tool-linux-x64@1.0.0", "tool-linux-x64"), + link(pack, ".pnpm/node_modules", "@scope+native-linux-x64@1.0.0", "@scope/native-linux-x64"), + ]; + const pruned = [ + link(pack, ".pnpm/consumer@1.0.0/node_modules", "tool-linux-x64-musl@1.0.0", "tool-linux-x64-musl"), + link(pack, ".pnpm/node_modules", "@scope+native-linux-x64-musl@1.0.0", "@scope/native-linux-x64-musl"), + link(pack, "", "@scope+native-linux-x64-musl@1.0.0", "@scope/native-linux-x64-musl"), + ]; + + assert.deepEqual(removeForeignLibcPackagesFromProviderPack(pack, "glibc"), [ + "@scope+native-linux-x64-musl@1.0.0", "tool-linux-x64-musl@1.0.0", + ]); + assert.deepEqual(readdirSync(join(pack, "node_modules", ".pnpm")).sort(), [ + "@scope+native-linux-x64@1.0.0", "consumer@1.0.0", "node_modules", "tool-linux-x64@1.0.0", + ]); + for (const path of kept) assert.equal(existsSync(join(path, "package.json")), true); + for (const path of pruned) assert.equal(lstatExists(path), false); + // The emptied top-level scope goes too; the shared hoisted scope keeps its glibc link. + assert.equal(existsSync(join(pack, "node_modules", "@scope")), false); + assert.deepEqual(readdirSync(join(pack, "node_modules", ".pnpm", "node_modules", "@scope")), ["native-linux-x64"]); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("keeps every package when the pack has no libc or the libc matches", () => { + const root = mkdtempSync(join(tmpdir(), "provider-pack-libc-noop-")); + try { + const pack = join(root, "pack"); + pnpmPackage(pack, "tool-linux-x64-musl@1.0.0", "tool-linux-x64-musl", { libc: ["musl"] }); + assert.deepEqual(removeForeignLibcPackagesFromProviderPack(pack, undefined), []); + assert.deepEqual(removeForeignLibcPackagesFromProviderPack(pack, "musl"), []); + assert.equal(existsSync(join(pack, "node_modules", ".pnpm", "tool-linux-x64-musl@1.0.0")), true); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +test("refuses store entry names that escape the pnpm virtual store", () => { + const root = mkdtempSync(join(tmpdir(), "provider-pack-store-escape-")); + try { + const pack = join(root, "pack"); + mkdirSync(join(pack, "node_modules", ".pnpm", "node_modules"), { recursive: true }); + writeFileSync(join(pack, "node_modules", "outside"), "kept"); + for (const name of ["../outside", "node_modules", "a/b"]) { + assert.throws(() => removePnpmStoreEntries(pack, [name]), /Refusing to remove/); + } + assert.equal(readFileSync(join(pack, "node_modules", "outside"), "utf8"), "kept"); + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +function lstatExists(path) { + try { lstatSync(path); return true; } catch { return false; } +}