diff --git a/packages/paperclip-runner/scripts/check-clean-consumers.mjs b/packages/paperclip-runner/scripts/check-clean-consumers.mjs index bc4ee1ce9e2..eeefd1a5e31 100644 --- a/packages/paperclip-runner/scripts/check-clean-consumers.mjs +++ b/packages/paperclip-runner/scripts/check-clean-consumers.mjs @@ -15,6 +15,7 @@ import { import { basename, dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; +import { withInstalledPackagePackInput } from "./lib/installed-package-pack.mjs"; const runnerRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const scratchParent = process.env.PAPERCLIP_RUN_SCRATCH_DIR @@ -107,7 +108,8 @@ async function packRunnerRuntimeDependencies(destination) { const identity = `${manifest.name}@${manifest.version}`; let tarball = packed.get(identity); if (tarball === undefined) { - tarball = await pack(concreteRoot, destination); + tarball = await withInstalledPackagePackInput(concreteRoot, scratchRoot, + input => pack(input, destination)); packed.set(identity, tarball); } overrides[overrideKey] = tarball; diff --git a/packages/paperclip-runner/scripts/installed-package-pack.test.mjs b/packages/paperclip-runner/scripts/installed-package-pack.test.mjs new file mode 100644 index 00000000000..b091d4d3629 --- /dev/null +++ b/packages/paperclip-runner/scripts/installed-package-pack.test.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { withInstalledPackagePackInput } from "./lib/installed-package-pack.mjs"; + +async function fixture(t, extra = {}) { + const root = await mkdtemp(join(tmpdir(), "runner-pack-test-")); + t.after(() => rm(root, { recursive: true, force: true })); + const source = join(root, "node_modules/.pnpm/fixture/node_modules/fixture"); + await mkdir(source, { recursive: true }); + await writeFile(join(source, "package.json"), JSON.stringify({ name: "pack-fixture", version: "1.0.0", ...extra })); + return { root, source }; +} + +for (const extra of [{}, { bundleDependencies: false }, { bundleDependencies: [] }]) { + test(`stages unchanged non-bundled package ${JSON.stringify(extra)}`, async t => { + const { root, source } = await fixture(t, extra); + await mkdir(join(source, "node_modules")); + await writeFile(join(source, "node_modules/foreign"), "must not copy"); + await writeFile(join(source, "bin.js"), "#!/usr/bin/env node\n"); + await chmod(join(source, "bin.js"), 0o755); + await writeFile(join(source, ".npmignore"), "excluded.txt\n"); + await symlink("bin.js", join(source, "link")); + let staged; + const result = await withInstalledPackagePackInput(source, root, async (input, external) => { + staged = input; + assert.equal(external, true); + assert.ok(!input.includes("node_modules")); + assert.deepEqual(await readFile(join(input, "package.json")), await readFile(join(source, "package.json"))); + assert.deepEqual(await readFile(join(input, ".npmignore")), await readFile(join(source, ".npmignore"))); + assert.equal((await lstat(join(input, "bin.js"))).mode & 0o777, 0o755); + assert.equal(await readlink(join(input, "link")), "bin.js"); + await assert.rejects(lstat(join(input, "node_modules")), { code: "ENOENT" }); + return "tarball"; + }); + assert.equal(result, "tarball"); + await assert.rejects(lstat(staged), { code: "ENOENT" }); + assert.equal(await readFile(join(source, "node_modules/foreign"), "utf8"), "must not copy"); + }); +} + +for (const extra of [{ bundleDependencies: true }, { bundleDependencies: ["dependency"] }, { bundledDependencies: ["dependency"] }]) { + test(`preserves original bundled pack path ${JSON.stringify(extra)}`, async t => { + const { root, source } = await fixture(t, extra); + const before = await readdir(root); + assert.equal(await withInstalledPackagePackInput(source, root, async (input, external) => { + assert.equal(input, source); assert.equal(external, false); return "unchanged"; + }), "unchanged"); + assert.deepEqual(await readdir(root), before); + }); +} + +test("removes only its staging input when packing fails", async t => { + const { root, source } = await fixture(t); + let staged; + const error = new Error("pack failed"); + await assert.rejects(withInstalledPackagePackInput(source, root, async input => { + staged = input; throw error; + }), value => value === error); + await assert.rejects(lstat(staged), { code: "ENOENT" }); + assert.ok((await lstat(join(source, "package.json"))).isFile()); +}); + +for (const useFiles of [true, false]) { + test(`npm preserves ${useFiles ? "files and nested .npmignore" : ".gitignore fallback"} selection and package links`, async t => { + const { root, source } = await fixture(t, { ...(useFiles ? { files: ["lib", "bin.js"] } : {}), bin: { fixture: "bin.js" }, + devEngines: { runtime: { name: "node", version: ">=999.0.0", onFail: "error" } } }); + await mkdir(join(source, "lib")); + await writeFile(join(source, "bin.js"), "#!/usr/bin/env node\nconsole.log('fixture');\n"); + await chmod(join(source, "bin.js"), 0o755); + await writeFile(join(source, "lib/kept.js"), "export const value = 'unchanged';\n"); + await writeFile(join(source, "lib/ignored.js"), "not published"); + if (useFiles) await writeFile(join(source, "lib/.npmignore"), "ignored.js\n"); + else await writeFile(join(source, ".gitignore"), "outside.txt\nlib/ignored.js\n"); + await writeFile(join(source, "outside.txt"), "not in files list"); + await writeFile(join(source, "LICENSE"), "fixture license\n"); + await writeFile(join(root, "private.txt"), "must not dereference"); + await symlink(join(root, "private.txt"), join(source, "lib/link")); + const home = join(root, "home"); await mkdir(home); + for (const name of ["user.npmrc", "global.npmrc"]) await writeFile(join(home, name), ""); + const env = { PATH: process.env.PATH, HOME: home, TMPDIR: root, CI: "true", + NPM_CONFIG_USERCONFIG: join(home, "user.npmrc"), NPM_CONFIG_GLOBALCONFIG: join(home, "global.npmrc"), + NPM_CONFIG_CACHE: join(home, "cache"), NPM_CONFIG_UPDATE_NOTIFIER: "false" }; + const pack = async (input, label) => { + const output = join(root, label); await mkdir(output); + const rows = JSON.parse(execFileSync("npm", ["pack", input, "--ignore-scripts", "--json", "--pack-destination", output], + { cwd: output, env, encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 })); + assert.equal(rows.length, 1); + return { files: rows[0].files, archive: await readFile(join(output, rows[0].filename)) }; + }; + // Check actual archive bytes against the installed input; the failing npm + // directory traversal itself is reproduced separately on Linux CI. + let input; + const staged = await withInstalledPackagePackInput(source, root, async path => { + input = path; + return pack(path, "first"); + }); + for (const file of staged.files) { + const bytes = execFileSync("tar", ["-xzOf", "-", `package/${file.path}`], + { input: staged.archive, timeout: 5_000, maxBuffer: 1024 * 1024 }); + assert.deepEqual(bytes, await readFile(join(source, file.path))); + } + assert.deepEqual(staged.files.map(file => file.path).sort(), ["LICENSE", "bin.js", "lib/kept.js", "package.json"]); + assert.equal(staged.files.find(file => file.path === "bin.js").mode & 0o777, 0o755); + await assert.rejects(lstat(input), { code: "ENOENT" }); + assert.equal(await readFile(join(root, "private.txt"), "utf8"), "must not dereference"); + }); +} diff --git a/packages/paperclip-runner/scripts/lib/installed-package-pack.mjs b/packages/paperclip-runner/scripts/lib/installed-package-pack.mjs new file mode 100644 index 00000000000..e5e361d145f --- /dev/null +++ b/packages/paperclip-runner/scripts/lib/installed-package-pack.mjs @@ -0,0 +1,30 @@ +import { cp, mkdtemp, readFile, rm } from "node:fs/promises"; +import { join, resolve } from "node:path"; + +/** Keep npm's installed-tree traversal out of non-bundled package packing. + * Copy manifests and pack rules unchanged; npm still selects the tarball files. + * Bundled packages need their dependency tree, so keep their existing pack path. + */ +export async function withInstalledPackagePackInput(packageRoot, scratchRoot, pack) { + const manifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8")); + const hasBundle = [manifest.bundleDependencies, manifest.bundledDependencies].some( + value => value !== undefined && value !== false && !(Array.isArray(value) && value.length === 0), + ); + if (hasBundle) return pack(packageRoot, false); + + const stage = await mkdtemp(join(scratchRoot, "pack-input-")); + try { + const input = join(stage, "package"); + // node_modules is excluded by npm for non-bundled packages. Preserve every + // other entry, including ignore rules and links, without dereferencing it. + await cp(packageRoot, input, { + recursive: true, + dereference: false, + verbatimSymlinks: true, + filter: source => resolve(source) !== resolve(packageRoot, "node_modules"), + }); + return await pack(input, true); + } finally { + await rm(stage, { recursive: true, force: true }); + } +}