fix(cli): stage git install workspace closure

Co-Authored-By: Paperclip <noreply@paperclip.ing>

cli/src/__tests__/install-command.test.ts

@@ -2,7 +2,15 @@ import fs from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
-import { installCommand, installGitPayload, resolveGitHubRef, resolveGitInstallRequest, resolveNpmInstallRequest } from "../commands/install.js";
+import {
+  installCommand,
+  installGitPayload,
+  resolveGitHubRef,
+  resolveGitInstallRequest,
+  resolveGitInstallWorkspacePackages,
+  resolveNpmInstallRequest,
+  runCommandWithDiagnostics,
+} from "../commands/install.js";
 import { uninstallCommand } from "../commands/uninstall.js";
 import { resolvePaperclipInstanceId } from "../config/home.js";
 import {
@@ -101,18 +109,35 @@ describe("managed install commands", () => {
     const runCommand = vi.fn(async (file: string, args: string[]) => {
       if (file === "curl" && !args.includes("--output")) return { stdout: JSON.stringify({ sha }), stderr: "" };
       if (file === "curl") { fs.writeFileSync(args[args.indexOf("--output") + 1], "archive"); return { stdout: "", stderr: "" }; }
-      if (file === "tar") { const checkout = args[args.indexOf("-C") + 1]; fs.mkdirSync(path.join(checkout, "cli"), { recursive: true }); fs.writeFileSync(path.join(checkout, "cli", "package.json"), JSON.stringify({ version: "0.3.1" })); return { stdout: "", stderr: "" }; }
+      if (file === "tar") {
+        const checkout = args[args.indexOf("-C") + 1];
+        const packages = [
+          { dir: "packages/shared", name: "@paperclipai/shared", packageJson: { name: "@paperclipai/shared", version: "0.3.1" } },
+          { dir: "packages/db", name: "@paperclipai/db", packageJson: { name: "@paperclipai/db", version: "0.3.1", dependencies: { "@paperclipai/shared": "workspace:*" }, bundleDependencies: ["embedded-postgres"] } },
+          { dir: "server", name: "@paperclipai/server", packageJson: { name: "@paperclipai/server", version: "0.3.1", dependencies: { "@paperclipai/db": "workspace:*" } } },
+        ];
+        fs.mkdirSync(path.join(checkout, "cli"), { recursive: true });
+        fs.writeFileSync(path.join(checkout, "cli", "package.json"), JSON.stringify({ version: "0.3.1" }));
+        fs.mkdirSync(path.join(checkout, "scripts"), { recursive: true });
+        fs.writeFileSync(path.join(checkout, "scripts", "release-package-manifest.json"), JSON.stringify(packages.map(({ dir, name }) => ({ dir, name }))));
+        for (const workspacePackage of packages) {
+          fs.mkdirSync(path.join(checkout, workspacePackage.dir), { recursive: true });
+          fs.writeFileSync(path.join(checkout, workspacePackage.dir, "package.json"), JSON.stringify(workspacePackage.packageJson));
+        }
+        return { stdout: "", stderr: "" };
+      }
       if (file === "corepack") {
         if (args.includes("pack")) {
           const destination = args[args.indexOf("--pack-destination") + 1];
-          const packageName = args.includes("packages/db") ? "paperclipai-db" : "paperclipai-server";
+          const packageDir = args[args.indexOf("--dir") + 1];
+          const packageName = packageDir === "server" ? "paperclipai-server" : "paperclipai-shared";
           fs.writeFileSync(path.join(destination, `${packageName}-0.3.1.tgz`), "package");
         }
         return { stdout: "", stderr: "" };
       }
       if (file === "bash") return { stdout: "", stderr: "" };
       if (file === "npm" && args[0] === "pack") {
-        const packageName = args[1]?.endsWith("db-package") ? "paperclipai-db" : "paperclipai";
+        const packageName = args[1]?.includes("workspace-package-") ? "paperclipai-db" : "paperclipai";
         fs.writeFileSync(path.join(args[args.indexOf("--pack-destination") + 1], `${packageName}-0.3.1.tgz`), "package");
         return { stdout: "", stderr: "" };
       }
@@ -132,9 +157,37 @@ describe("managed install commands", () => {
     expect(manifest?.payloadPath).toContain(path.join("git", sha.slice(0, 12)));
     expect(runCommand.mock.calls.filter(([command, args]) => command === "curl" && args.includes("--output"))).toHaveLength(1);
     expect(runCommand.mock.calls.filter(([command, args]) => command === "corepack" && args[1] === "install")).toHaveLength(1);
-    expect(runCommand.mock.calls.filter(([command, args]) => command === "corepack" && args.includes("pack"))).toHaveLength(1);
+    expect(runCommand.mock.calls.filter(([command, args]) => command === "corepack" && args.includes("pack"))).toHaveLength(2);
     expect(runCommand.mock.calls.filter(([command, args]) => command === process.execPath && args[0]?.endsWith("prepare-bundled-package.mjs"))).toHaveLength(1);
     expect(runCommand.mock.calls.filter(([command, args]) => command === "npm" && args[0] === "pack")).toHaveLength(2);
+    const installCall = runCommand.mock.calls.find(([command, args]) => command === "npm" && args[0] === "install");
+    expect(installCall?.[1].filter((arg) => arg.endsWith(".tgz"))).toHaveLength(4);
+  });
+
+  it("resolves the complete server workspace dependency closure in dependency order", () => {
+    const checkout = path.join(root, "checkout");
+    const packages = [
+      { dir: "packages/shared", name: "@paperclipai/shared", dependencies: {} },
+      { dir: "packages/db", name: "@paperclipai/db", dependencies: { "@paperclipai/shared": "workspace:*" } },
+      { dir: "server", name: "@paperclipai/server", dependencies: { "@paperclipai/db": "workspace:*" } },
+    ];
+    fs.mkdirSync(path.join(checkout, "scripts"), { recursive: true });
+    fs.writeFileSync(path.join(checkout, "scripts", "release-package-manifest.json"), JSON.stringify(packages.map(({ dir, name }) => ({ dir, name }))));
+    for (const workspacePackage of packages) {
+      fs.mkdirSync(path.join(checkout, workspacePackage.dir), { recursive: true });
+      fs.writeFileSync(path.join(checkout, workspacePackage.dir, "package.json"), JSON.stringify({ name: workspacePackage.name, dependencies: workspacePackage.dependencies }));
+    }
+
+    expect(resolveGitInstallWorkspacePackages(checkout).map(({ name }) => name)).toEqual([
+      "@paperclipai/shared",
+      "@paperclipai/db",
+      "@paperclipai/server",
+    ]);
+  });
+
+  it("includes child-process stderr in command failures", async () => {
+    await expect(runCommandWithDiagnostics(process.execPath, ["-e", "process.stderr.write('unsupported workspace dependency\\n'); process.exit(1)"]))
+      .rejects.toThrow("unsupported workspace dependency");
   });
 
   it("installs through the shim, reports provenance, and uninstalls without deleting user data", async () => {

cli/src/commands/install.ts

@@ -33,6 +33,55 @@ export type CommandRunner = (
   options?: Parameters<typeof execFileAsync>[2],
 ) => Promise<{ stdout: string; stderr: string }>;
 
+type ReleasePackageEntry = { dir: string; name: string };
+
+export async function runCommandWithDiagnostics(
+  file: string,
+  args: string[],
+  options?: Parameters<typeof execFileAsync>[2],
+): Promise<{ stdout: string; stderr: string }> {
+  try {
+    return await execFileAsync(file, args, { ...options, encoding: "utf8" });
+  } catch (error) {
+    const stderr = error && typeof error === "object" && "stderr" in error && typeof error.stderr === "string"
+      ? error.stderr.trim()
+      : "";
+    if (!stderr || (error instanceof Error && error.message.includes(stderr))) throw error;
+    throw new Error(`${error instanceof Error ? error.message : String(error)}\n${stderr}`, { cause: error });
+  }
+}
+
+export function resolveGitInstallWorkspacePackages(checkoutPath: string): ReleasePackageEntry[] {
+  const manifestPath = path.join(checkoutPath, "scripts", "release-package-manifest.json");
+  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as ReleasePackageEntry[];
+  const packageByName = new Map(manifest.map((entry) => [entry.name, entry]));
+  const visiting = new Set<string>();
+  const visited = new Set<string>();
+  const ordered: ReleasePackageEntry[] = [];
+
+  const visit = (packageName: string): void => {
+    if (visited.has(packageName)) return;
+    if (visiting.has(packageName)) throw new Error(`Circular workspace dependency while staging ${packageName}.`);
+    const entry = packageByName.get(packageName);
+    if (!entry) throw new Error(`Git install cannot stage workspace dependency ${packageName}; it is missing from scripts/release-package-manifest.json.`);
+    visiting.add(packageName);
+    const packageJson = JSON.parse(fs.readFileSync(path.join(checkoutPath, entry.dir, "package.json"), "utf8")) as Record<string, unknown>;
+    for (const section of ["dependencies", "optionalDependencies", "peerDependencies"] as const) {
+      const dependencies = packageJson[section];
+      if (!dependencies || typeof dependencies !== "object") continue;
+      for (const dependencyName of Object.keys(dependencies)) {
+        if (dependencyName.startsWith("@paperclipai/")) visit(dependencyName);
+      }
+    }
+    visiting.delete(packageName);
+    visited.add(packageName);
+    ordered.push(entry);
+  };
+
+  visit("@paperclipai/server");
+  return ordered;
+}
+
 function assertSupportedNodeVersion(): void {
   const major = Number(process.versions.node.split(".")[0]);
   if (!Number.isFinite(major) || major < 20) {
@@ -191,21 +240,30 @@ export async function installGitPayload(repo: string, sha: string, runCommand: C
     await runCommand("curl", ["--fail", "--silent", "--show-error", "--location", "--output", archivePath, `https://codeload.github.com/${repo}/tar.gz/${sha}`], { maxBuffer: 4 * 1024 * 1024 });
     await runCommand("tar", ["-xzf", archivePath, "--strip-components=1", "-C", checkoutPath], { maxBuffer: 4 * 1024 * 1024 });
     await runCommand("corepack", ["pnpm", "install", "--frozen-lockfile"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
-    await runCommand("bash", ["scripts/build-npm.sh", "--skip-checks"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
-    await runCommand("corepack", ["pnpm", "--filter", "@paperclipai/db", "build"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
-    await runCommand("corepack", ["pnpm", "--filter", "@paperclipai/server", "build"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
+    await runCommand("bash", ["scripts/build-npm.sh", "--skip-checks", "--skip-typecheck"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
+    await runCommand("corepack", ["pnpm", "-r", "--filter", "@paperclipai/server...", "--if-present", "run", "build"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
     const metadata = JSON.parse(fs.readFileSync(path.join(checkoutPath, "cli", "package.json"), "utf8")) as { version: string };
-    const stagedDbPackage = path.join(stagingRoot, "db-package");
-    await runCommand(process.execPath, [path.join(checkoutPath, "scripts", "prepare-bundled-package.mjs"), path.join(checkoutPath, "packages", "db"), stagedDbPackage], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
-    await runCommand("npm", ["pack", stagedDbPackage, "--pack-destination", stagingRoot], { cwd: checkoutPath, maxBuffer: 16 * 1024 * 1024 });
-    await runCommand("corepack", ["pnpm", "--dir", "server", "pack", "--pack-destination", stagingRoot], { cwd: checkoutPath, env: { ...process.env, PAPERCLIP_RELEASE_REUSE_UI_DIST: "1" }, maxBuffer: 32 * 1024 * 1024 });
+    const workspacePackages = resolveGitInstallWorkspacePackages(checkoutPath);
+    for (const [index, workspacePackage] of workspacePackages.entries()) {
+      const packageDir = path.join(checkoutPath, workspacePackage.dir);
+      const packageJson = JSON.parse(fs.readFileSync(path.join(packageDir, "package.json"), "utf8")) as { bundleDependencies?: string[]; bundledDependencies?: string[] };
+      const bundledDependencies = packageJson.bundleDependencies ?? packageJson.bundledDependencies ?? [];
+      if (bundledDependencies.length > 0) {
+        const stagedPackage = path.join(stagingRoot, `workspace-package-${index}`);
+        await runCommand(process.execPath, [path.join(checkoutPath, "scripts", "prepare-bundled-package.mjs"), packageDir, stagedPackage], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 });
+        await runCommand("npm", ["pack", stagedPackage, "--pack-destination", stagingRoot], { cwd: checkoutPath, maxBuffer: 16 * 1024 * 1024 });
+      } else {
+        await runCommand("corepack", ["pnpm", "--dir", workspacePackage.dir, "pack", "--pack-destination", stagingRoot], { cwd: checkoutPath, env: { ...process.env, PAPERCLIP_RELEASE_REUSE_UI_DIST: "1" }, maxBuffer: 32 * 1024 * 1024 });
+      }
+    }
     await runCommand("npm", ["pack", "--pack-destination", stagingRoot], { cwd: path.join(checkoutPath, "cli"), maxBuffer: 16 * 1024 * 1024 });
     const tarballs = fs.readdirSync(stagingRoot).filter((entry) => entry.endsWith(".tgz"));
-    const cliTarball = tarballs.find((entry) => /^paperclipai-(?!db-|server-)/.test(entry));
-    const dbTarball = tarballs.find((entry) => entry.startsWith("paperclipai-db-"));
-    const serverTarball = tarballs.find((entry) => entry.startsWith("paperclipai-server-"));
-    if (!cliTarball || !dbTarball || !serverTarball) throw new Error("Git install packaging did not produce paperclipai, @paperclipai/db, and @paperclipai/server tarballs.");
-    await runCommand("npm", ["install", "--prefix", stagedPayload, path.join(stagingRoot, cliTarball), path.join(stagingRoot, dbTarball), path.join(stagingRoot, serverTarball), "--no-audit", "--no-fund"], { cwd: stagingRoot, maxBuffer: 32 * 1024 * 1024 });
+    const cliTarball = tarballs.find((entry) => entry === `paperclipai-${metadata.version}.tgz`);
+    const workspaceTarballs = tarballs.filter((entry) => entry !== cliTarball);
+    if (!cliTarball || workspaceTarballs.length !== workspacePackages.length) {
+      throw new Error(`Git install packaging produced ${workspaceTarballs.length} workspace tarballs; expected ${workspacePackages.length}.`);
+    }
+    await runCommand("npm", ["install", "--prefix", stagedPayload, path.join(stagingRoot, cliTarball), ...workspaceTarballs.map((entry) => path.join(stagingRoot, entry)), "--no-audit", "--no-fund"], { cwd: stagingRoot, maxBuffer: 32 * 1024 * 1024 });
     await smokePayload(stagedPayload, metadata.version, runCommand);
     fs.renameSync(stagedPayload, payloadPath);
     return { payloadPath, reused: false, version: metadata.version };
@@ -266,7 +324,7 @@ export async function installCommand(
   dependencies: { runCommand?: CommandRunner; now?: () => Date } = {},
 ): Promise<void> {
   assertSupportedNodeVersion();
-  const runCommand = dependencies.runCommand ?? execFileAsync;
+  const runCommand = dependencies.runCommand ?? runCommandWithDiagnostics;
   const gitRequest = resolveGitInstallRequest(options);
   if (gitRequest) {
     await confirmGitInstall(options, gitRequest.repo, gitRequest.ref);

scripts/acpx-patch-packaging.test.mjs

@@ -30,6 +30,7 @@ const dbPackage = JSON.parse(
 );
 const releaseScript = await readFile(new URL("./release.sh", import.meta.url), "utf8");
 const releaseLib = await readFile(new URL("./release-lib.sh", import.meta.url), "utf8");
+const buildNpmScript = await readFile(new URL("./build-npm.sh", import.meta.url), "utf8");
 
 test("published packages preserve the patched ACPX runtime", () => {
   assert.equal(
@@ -189,3 +190,8 @@ test("bundled package dry runs preview without querying published versions", ()
   assert.match(releaseLib, /run_bundled_npm_publish publish --tag "\$dist_tag"/);
   assert.doesNotMatch(releaseLib, /run_bundled_npm_publish publish "\.\/\$tarball"/);
 });
+
+test("npm builds use corepack instead of requiring a global pnpm", () => {
+  assert.match(buildNpmScript, /corepack pnpm -r typecheck/);
+  assert.doesNotMatch(buildNpmScript, /^\s*pnpm -r typecheck/m);
+});

scripts/build-npm.sh

@@ -37,7 +37,7 @@ fi
 if [ "$skip_typecheck" = false ]; then
   echo "  [2/6] Type-checking..."
   cd "$REPO_ROOT"
-  pnpm -r typecheck
+  corepack pnpm -r typecheck
 else
   echo "  [2/6] Skipping type-check (--skip-typecheck)"
 fi