fix(runner): stage installed dependency pack inputs

Preserve non-bundled package contents outside the pnpm tree before npm packs them. Keep bundled and root-package behavior and clean-consumer assertions unchanged.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

packages/paperclip-runner/scripts/check-clean-consumers.mjs

@@ -15,6 +15,7 @@ import {
 import { basename, dirname, join, resolve } from "node:path";
 import { tmpdir } from "node:os";
 import { fileURLToPath } from "node:url";
+import { withInstalledPackagePackInput } from "./lib/installed-package-pack.mjs";
 
 const runnerRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const scratchParent = process.env.PAPERCLIP_RUN_SCRATCH_DIR
@@ -107,7 +108,8 @@ async function packRunnerRuntimeDependencies(destination) {
     const identity = `${manifest.name}@${manifest.version}`;
     let tarball = packed.get(identity);
     if (tarball === undefined) {
-      tarball = await pack(concreteRoot, destination);
+      tarball = await withInstalledPackagePackInput(concreteRoot, scratchRoot,
+        input => pack(input, destination));
       packed.set(identity, tarball);
     }
     overrides[overrideKey] = tarball;

packages/paperclip-runner/scripts/installed-package-pack.test.mjs

@@ -0,0 +1,111 @@
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { chmod, lstat, mkdir, mkdtemp, readFile, readdir, readlink, rm, symlink, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { test } from "node:test";
+import { withInstalledPackagePackInput } from "./lib/installed-package-pack.mjs";
+
+async function fixture(t, extra = {}) {
+  const root = await mkdtemp(join(tmpdir(), "runner-pack-test-"));
+  t.after(() => rm(root, { recursive: true, force: true }));
+  const source = join(root, "node_modules/.pnpm/fixture/node_modules/fixture");
+  await mkdir(source, { recursive: true });
+  await writeFile(join(source, "package.json"), JSON.stringify({ name: "pack-fixture", version: "1.0.0", ...extra }));
+  return { root, source };
+}
+
+for (const extra of [{}, { bundleDependencies: false }, { bundleDependencies: [] }]) {
+  test(`stages unchanged non-bundled package ${JSON.stringify(extra)}`, async t => {
+    const { root, source } = await fixture(t, extra);
+    await mkdir(join(source, "node_modules"));
+    await writeFile(join(source, "node_modules/foreign"), "must not copy");
+    await writeFile(join(source, "bin.js"), "#!/usr/bin/env node\n");
+    await chmod(join(source, "bin.js"), 0o755);
+    await writeFile(join(source, ".npmignore"), "excluded.txt\n");
+    await symlink("bin.js", join(source, "link"));
+    let staged;
+    const result = await withInstalledPackagePackInput(source, root, async (input, external) => {
+      staged = input;
+      assert.equal(external, true);
+      assert.ok(!input.includes("node_modules"));
+      assert.deepEqual(await readFile(join(input, "package.json")), await readFile(join(source, "package.json")));
+      assert.deepEqual(await readFile(join(input, ".npmignore")), await readFile(join(source, ".npmignore")));
+      assert.equal((await lstat(join(input, "bin.js"))).mode & 0o777, 0o755);
+      assert.equal(await readlink(join(input, "link")), "bin.js");
+      await assert.rejects(lstat(join(input, "node_modules")), { code: "ENOENT" });
+      return "tarball";
+    });
+    assert.equal(result, "tarball");
+    await assert.rejects(lstat(staged), { code: "ENOENT" });
+    assert.equal(await readFile(join(source, "node_modules/foreign"), "utf8"), "must not copy");
+  });
+}
+
+for (const extra of [{ bundleDependencies: true }, { bundleDependencies: ["dependency"] }, { bundledDependencies: ["dependency"] }]) {
+  test(`preserves original bundled pack path ${JSON.stringify(extra)}`, async t => {
+    const { root, source } = await fixture(t, extra);
+    const before = await readdir(root);
+    assert.equal(await withInstalledPackagePackInput(source, root, async (input, external) => {
+      assert.equal(input, source); assert.equal(external, false); return "unchanged";
+    }), "unchanged");
+    assert.deepEqual(await readdir(root), before);
+  });
+}
+
+test("removes only its staging input when packing fails", async t => {
+  const { root, source } = await fixture(t);
+  let staged;
+  const error = new Error("pack failed");
+  await assert.rejects(withInstalledPackagePackInput(source, root, async input => {
+    staged = input; throw error;
+  }), value => value === error);
+  await assert.rejects(lstat(staged), { code: "ENOENT" });
+  assert.ok((await lstat(join(source, "package.json"))).isFile());
+});
+
+for (const useFiles of [true, false]) {
+  test(`npm preserves ${useFiles ? "files and nested .npmignore" : ".gitignore fallback"} selection and package links`, async t => {
+    const { root, source } = await fixture(t, { ...(useFiles ? { files: ["lib", "bin.js"] } : {}), bin: { fixture: "bin.js" },
+      devEngines: { runtime: { name: "node", version: ">=999.0.0", onFail: "error" } } });
+    await mkdir(join(source, "lib"));
+    await writeFile(join(source, "bin.js"), "#!/usr/bin/env node\nconsole.log('fixture');\n");
+    await chmod(join(source, "bin.js"), 0o755);
+    await writeFile(join(source, "lib/kept.js"), "export const value = 'unchanged';\n");
+    await writeFile(join(source, "lib/ignored.js"), "not published");
+    if (useFiles) await writeFile(join(source, "lib/.npmignore"), "ignored.js\n");
+    else await writeFile(join(source, ".gitignore"), "outside.txt\nlib/ignored.js\n");
+    await writeFile(join(source, "outside.txt"), "not in files list");
+    await writeFile(join(source, "LICENSE"), "fixture license\n");
+    await writeFile(join(root, "private.txt"), "must not dereference");
+    await symlink(join(root, "private.txt"), join(source, "lib/link"));
+    const home = join(root, "home"); await mkdir(home);
+    for (const name of ["user.npmrc", "global.npmrc"]) await writeFile(join(home, name), "");
+    const env = { PATH: process.env.PATH, HOME: home, TMPDIR: root, CI: "true",
+      NPM_CONFIG_USERCONFIG: join(home, "user.npmrc"), NPM_CONFIG_GLOBALCONFIG: join(home, "global.npmrc"),
+      NPM_CONFIG_CACHE: join(home, "cache"), NPM_CONFIG_UPDATE_NOTIFIER: "false" };
+    const pack = async (input, label) => {
+      const output = join(root, label); await mkdir(output);
+      const rows = JSON.parse(execFileSync("npm", ["pack", input, "--ignore-scripts", "--json", "--pack-destination", output],
+        { cwd: output, env, encoding: "utf8", timeout: 60_000, maxBuffer: 1024 * 1024 }));
+      assert.equal(rows.length, 1);
+      return { files: rows[0].files, archive: await readFile(join(output, rows[0].filename)) };
+    };
+    // Check actual archive bytes against the installed input; the failing npm
+    // directory traversal itself is reproduced separately on Linux CI.
+    let input;
+    const staged = await withInstalledPackagePackInput(source, root, async path => {
+      input = path;
+      return pack(path, "first");
+    });
+    for (const file of staged.files) {
+      const bytes = execFileSync("tar", ["-xzOf", "-", `package/${file.path}`],
+        { input: staged.archive, timeout: 5_000, maxBuffer: 1024 * 1024 });
+      assert.deepEqual(bytes, await readFile(join(source, file.path)));
+    }
+    assert.deepEqual(staged.files.map(file => file.path).sort(), ["LICENSE", "bin.js", "lib/kept.js", "package.json"]);
+    assert.equal(staged.files.find(file => file.path === "bin.js").mode & 0o777, 0o755);
+    await assert.rejects(lstat(input), { code: "ENOENT" });
+    assert.equal(await readFile(join(root, "private.txt"), "utf8"), "must not dereference");
+  });
+}

packages/paperclip-runner/scripts/lib/installed-package-pack.mjs

@@ -0,0 +1,30 @@
+import { cp, mkdtemp, readFile, rm } from "node:fs/promises";
+import { join, resolve } from "node:path";
+
+/** Keep npm's installed-tree traversal out of non-bundled package packing.
+ * Copy manifests and pack rules unchanged; npm still selects the tarball files.
+ * Bundled packages need their dependency tree, so keep their existing pack path.
+ */
+export async function withInstalledPackagePackInput(packageRoot, scratchRoot, pack) {
+  const manifest = JSON.parse(await readFile(resolve(packageRoot, "package.json"), "utf8"));
+  const hasBundle = [manifest.bundleDependencies, manifest.bundledDependencies].some(
+    value => value !== undefined && value !== false && !(Array.isArray(value) && value.length === 0),
+  );
+  if (hasBundle) return pack(packageRoot, false);
+
+  const stage = await mkdtemp(join(scratchRoot, "pack-input-"));
+  try {
+    const input = join(stage, "package");
+    // node_modules is excluded by npm for non-bundled packages. Preserve every
+    // other entry, including ignore rules and links, without dereferencing it.
+    await cp(packageRoot, input, {
+      recursive: true,
+      dereference: false,
+      verbatimSymlinks: true,
+      filter: source => resolve(source) !== resolve(packageRoot, "node_modules"),
+    });
+    return await pack(input, true);
+  } finally {
+    await rm(stage, { recursive: true, force: true });
+  }
+}