diff --git a/cli/src/__tests__/install-command.test.ts b/cli/src/__tests__/install-command.test.ts index e85531990fb..bfc2461239a 100644 --- a/cli/src/__tests__/install-command.test.ts +++ b/cli/src/__tests__/install-command.test.ts @@ -2,7 +2,15 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { installCommand, installGitPayload, resolveGitHubRef, resolveGitInstallRequest, resolveNpmInstallRequest } from "../commands/install.js"; +import { + installCommand, + installGitPayload, + resolveGitHubRef, + resolveGitInstallRequest, + resolveGitInstallWorkspacePackages, + resolveNpmInstallRequest, + runCommandWithDiagnostics, +} from "../commands/install.js"; import { uninstallCommand } from "../commands/uninstall.js"; import { resolvePaperclipInstanceId } from "../config/home.js"; import { @@ -101,18 +109,35 @@ describe("managed install commands", () => { const runCommand = vi.fn(async (file: string, args: string[]) => { if (file === "curl" && !args.includes("--output")) return { stdout: JSON.stringify({ sha }), stderr: "" }; if (file === "curl") { fs.writeFileSync(args[args.indexOf("--output") + 1], "archive"); return { stdout: "", stderr: "" }; } - if (file === "tar") { const checkout = args[args.indexOf("-C") + 1]; fs.mkdirSync(path.join(checkout, "cli"), { recursive: true }); fs.writeFileSync(path.join(checkout, "cli", "package.json"), JSON.stringify({ version: "0.3.1" })); return { stdout: "", stderr: "" }; } + if (file === "tar") { + const checkout = args[args.indexOf("-C") + 1]; + const packages = [ + { dir: "packages/shared", name: "@paperclipai/shared", packageJson: { name: "@paperclipai/shared", version: "0.3.1" } }, + { dir: "packages/db", name: "@paperclipai/db", packageJson: { name: "@paperclipai/db", version: "0.3.1", dependencies: { "@paperclipai/shared": "workspace:*" }, bundleDependencies: ["embedded-postgres"] } }, + { dir: "server", name: "@paperclipai/server", packageJson: { name: "@paperclipai/server", version: "0.3.1", dependencies: { "@paperclipai/db": "workspace:*" } } }, + ]; + fs.mkdirSync(path.join(checkout, "cli"), { recursive: true }); + fs.writeFileSync(path.join(checkout, "cli", "package.json"), JSON.stringify({ version: "0.3.1" })); + fs.mkdirSync(path.join(checkout, "scripts"), { recursive: true }); + fs.writeFileSync(path.join(checkout, "scripts", "release-package-manifest.json"), JSON.stringify(packages.map(({ dir, name }) => ({ dir, name })))); + for (const workspacePackage of packages) { + fs.mkdirSync(path.join(checkout, workspacePackage.dir), { recursive: true }); + fs.writeFileSync(path.join(checkout, workspacePackage.dir, "package.json"), JSON.stringify(workspacePackage.packageJson)); + } + return { stdout: "", stderr: "" }; + } if (file === "corepack") { if (args.includes("pack")) { const destination = args[args.indexOf("--pack-destination") + 1]; - const packageName = args.includes("packages/db") ? "paperclipai-db" : "paperclipai-server"; + const packageDir = args[args.indexOf("--dir") + 1]; + const packageName = packageDir === "server" ? "paperclipai-server" : "paperclipai-shared"; fs.writeFileSync(path.join(destination, `${packageName}-0.3.1.tgz`), "package"); } return { stdout: "", stderr: "" }; } if (file === "bash") return { stdout: "", stderr: "" }; if (file === "npm" && args[0] === "pack") { - const packageName = args[1]?.endsWith("db-package") ? "paperclipai-db" : "paperclipai"; + const packageName = args[1]?.includes("workspace-package-") ? "paperclipai-db" : "paperclipai"; fs.writeFileSync(path.join(args[args.indexOf("--pack-destination") + 1], `${packageName}-0.3.1.tgz`), "package"); return { stdout: "", stderr: "" }; } @@ -132,9 +157,37 @@ describe("managed install commands", () => { expect(manifest?.payloadPath).toContain(path.join("git", sha.slice(0, 12))); expect(runCommand.mock.calls.filter(([command, args]) => command === "curl" && args.includes("--output"))).toHaveLength(1); expect(runCommand.mock.calls.filter(([command, args]) => command === "corepack" && args[1] === "install")).toHaveLength(1); - expect(runCommand.mock.calls.filter(([command, args]) => command === "corepack" && args.includes("pack"))).toHaveLength(1); + expect(runCommand.mock.calls.filter(([command, args]) => command === "corepack" && args.includes("pack"))).toHaveLength(2); expect(runCommand.mock.calls.filter(([command, args]) => command === process.execPath && args[0]?.endsWith("prepare-bundled-package.mjs"))).toHaveLength(1); expect(runCommand.mock.calls.filter(([command, args]) => command === "npm" && args[0] === "pack")).toHaveLength(2); + const installCall = runCommand.mock.calls.find(([command, args]) => command === "npm" && args[0] === "install"); + expect(installCall?.[1].filter((arg) => arg.endsWith(".tgz"))).toHaveLength(4); + }); + + it("resolves the complete server workspace dependency closure in dependency order", () => { + const checkout = path.join(root, "checkout"); + const packages = [ + { dir: "packages/shared", name: "@paperclipai/shared", dependencies: {} }, + { dir: "packages/db", name: "@paperclipai/db", dependencies: { "@paperclipai/shared": "workspace:*" } }, + { dir: "server", name: "@paperclipai/server", dependencies: { "@paperclipai/db": "workspace:*" } }, + ]; + fs.mkdirSync(path.join(checkout, "scripts"), { recursive: true }); + fs.writeFileSync(path.join(checkout, "scripts", "release-package-manifest.json"), JSON.stringify(packages.map(({ dir, name }) => ({ dir, name })))); + for (const workspacePackage of packages) { + fs.mkdirSync(path.join(checkout, workspacePackage.dir), { recursive: true }); + fs.writeFileSync(path.join(checkout, workspacePackage.dir, "package.json"), JSON.stringify({ name: workspacePackage.name, dependencies: workspacePackage.dependencies })); + } + + expect(resolveGitInstallWorkspacePackages(checkout).map(({ name }) => name)).toEqual([ + "@paperclipai/shared", + "@paperclipai/db", + "@paperclipai/server", + ]); + }); + + it("includes child-process stderr in command failures", async () => { + await expect(runCommandWithDiagnostics(process.execPath, ["-e", "process.stderr.write('unsupported workspace dependency\\n'); process.exit(1)"])) + .rejects.toThrow("unsupported workspace dependency"); }); it("installs through the shim, reports provenance, and uninstalls without deleting user data", async () => { diff --git a/cli/src/commands/install.ts b/cli/src/commands/install.ts index ec769c38404..68376cb3118 100644 --- a/cli/src/commands/install.ts +++ b/cli/src/commands/install.ts @@ -33,6 +33,55 @@ export type CommandRunner = ( options?: Parameters[2], ) => Promise<{ stdout: string; stderr: string }>; +type ReleasePackageEntry = { dir: string; name: string }; + +export async function runCommandWithDiagnostics( + file: string, + args: string[], + options?: Parameters[2], +): Promise<{ stdout: string; stderr: string }> { + try { + return await execFileAsync(file, args, { ...options, encoding: "utf8" }); + } catch (error) { + const stderr = error && typeof error === "object" && "stderr" in error && typeof error.stderr === "string" + ? error.stderr.trim() + : ""; + if (!stderr || (error instanceof Error && error.message.includes(stderr))) throw error; + throw new Error(`${error instanceof Error ? error.message : String(error)}\n${stderr}`, { cause: error }); + } +} + +export function resolveGitInstallWorkspacePackages(checkoutPath: string): ReleasePackageEntry[] { + const manifestPath = path.join(checkoutPath, "scripts", "release-package-manifest.json"); + const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8")) as ReleasePackageEntry[]; + const packageByName = new Map(manifest.map((entry) => [entry.name, entry])); + const visiting = new Set(); + const visited = new Set(); + const ordered: ReleasePackageEntry[] = []; + + const visit = (packageName: string): void => { + if (visited.has(packageName)) return; + if (visiting.has(packageName)) throw new Error(`Circular workspace dependency while staging ${packageName}.`); + const entry = packageByName.get(packageName); + if (!entry) throw new Error(`Git install cannot stage workspace dependency ${packageName}; it is missing from scripts/release-package-manifest.json.`); + visiting.add(packageName); + const packageJson = JSON.parse(fs.readFileSync(path.join(checkoutPath, entry.dir, "package.json"), "utf8")) as Record; + for (const section of ["dependencies", "optionalDependencies", "peerDependencies"] as const) { + const dependencies = packageJson[section]; + if (!dependencies || typeof dependencies !== "object") continue; + for (const dependencyName of Object.keys(dependencies)) { + if (dependencyName.startsWith("@paperclipai/")) visit(dependencyName); + } + } + visiting.delete(packageName); + visited.add(packageName); + ordered.push(entry); + }; + + visit("@paperclipai/server"); + return ordered; +} + function assertSupportedNodeVersion(): void { const major = Number(process.versions.node.split(".")[0]); if (!Number.isFinite(major) || major < 20) { @@ -191,21 +240,30 @@ export async function installGitPayload(repo: string, sha: string, runCommand: C await runCommand("curl", ["--fail", "--silent", "--show-error", "--location", "--output", archivePath, `https://codeload.github.com/${repo}/tar.gz/${sha}`], { maxBuffer: 4 * 1024 * 1024 }); await runCommand("tar", ["-xzf", archivePath, "--strip-components=1", "-C", checkoutPath], { maxBuffer: 4 * 1024 * 1024 }); await runCommand("corepack", ["pnpm", "install", "--frozen-lockfile"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); - await runCommand("bash", ["scripts/build-npm.sh", "--skip-checks"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); - await runCommand("corepack", ["pnpm", "--filter", "@paperclipai/db", "build"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); - await runCommand("corepack", ["pnpm", "--filter", "@paperclipai/server", "build"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); + await runCommand("bash", ["scripts/build-npm.sh", "--skip-checks", "--skip-typecheck"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); + await runCommand("corepack", ["pnpm", "-r", "--filter", "@paperclipai/server...", "--if-present", "run", "build"], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); const metadata = JSON.parse(fs.readFileSync(path.join(checkoutPath, "cli", "package.json"), "utf8")) as { version: string }; - const stagedDbPackage = path.join(stagingRoot, "db-package"); - await runCommand(process.execPath, [path.join(checkoutPath, "scripts", "prepare-bundled-package.mjs"), path.join(checkoutPath, "packages", "db"), stagedDbPackage], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); - await runCommand("npm", ["pack", stagedDbPackage, "--pack-destination", stagingRoot], { cwd: checkoutPath, maxBuffer: 16 * 1024 * 1024 }); - await runCommand("corepack", ["pnpm", "--dir", "server", "pack", "--pack-destination", stagingRoot], { cwd: checkoutPath, env: { ...process.env, PAPERCLIP_RELEASE_REUSE_UI_DIST: "1" }, maxBuffer: 32 * 1024 * 1024 }); + const workspacePackages = resolveGitInstallWorkspacePackages(checkoutPath); + for (const [index, workspacePackage] of workspacePackages.entries()) { + const packageDir = path.join(checkoutPath, workspacePackage.dir); + const packageJson = JSON.parse(fs.readFileSync(path.join(packageDir, "package.json"), "utf8")) as { bundleDependencies?: string[]; bundledDependencies?: string[] }; + const bundledDependencies = packageJson.bundleDependencies ?? packageJson.bundledDependencies ?? []; + if (bundledDependencies.length > 0) { + const stagedPackage = path.join(stagingRoot, `workspace-package-${index}`); + await runCommand(process.execPath, [path.join(checkoutPath, "scripts", "prepare-bundled-package.mjs"), packageDir, stagedPackage], { cwd: checkoutPath, maxBuffer: 32 * 1024 * 1024 }); + await runCommand("npm", ["pack", stagedPackage, "--pack-destination", stagingRoot], { cwd: checkoutPath, maxBuffer: 16 * 1024 * 1024 }); + } else { + await runCommand("corepack", ["pnpm", "--dir", workspacePackage.dir, "pack", "--pack-destination", stagingRoot], { cwd: checkoutPath, env: { ...process.env, PAPERCLIP_RELEASE_REUSE_UI_DIST: "1" }, maxBuffer: 32 * 1024 * 1024 }); + } + } await runCommand("npm", ["pack", "--pack-destination", stagingRoot], { cwd: path.join(checkoutPath, "cli"), maxBuffer: 16 * 1024 * 1024 }); const tarballs = fs.readdirSync(stagingRoot).filter((entry) => entry.endsWith(".tgz")); - const cliTarball = tarballs.find((entry) => /^paperclipai-(?!db-|server-)/.test(entry)); - const dbTarball = tarballs.find((entry) => entry.startsWith("paperclipai-db-")); - const serverTarball = tarballs.find((entry) => entry.startsWith("paperclipai-server-")); - if (!cliTarball || !dbTarball || !serverTarball) throw new Error("Git install packaging did not produce paperclipai, @paperclipai/db, and @paperclipai/server tarballs."); - await runCommand("npm", ["install", "--prefix", stagedPayload, path.join(stagingRoot, cliTarball), path.join(stagingRoot, dbTarball), path.join(stagingRoot, serverTarball), "--no-audit", "--no-fund"], { cwd: stagingRoot, maxBuffer: 32 * 1024 * 1024 }); + const cliTarball = tarballs.find((entry) => entry === `paperclipai-${metadata.version}.tgz`); + const workspaceTarballs = tarballs.filter((entry) => entry !== cliTarball); + if (!cliTarball || workspaceTarballs.length !== workspacePackages.length) { + throw new Error(`Git install packaging produced ${workspaceTarballs.length} workspace tarballs; expected ${workspacePackages.length}.`); + } + await runCommand("npm", ["install", "--prefix", stagedPayload, path.join(stagingRoot, cliTarball), ...workspaceTarballs.map((entry) => path.join(stagingRoot, entry)), "--no-audit", "--no-fund"], { cwd: stagingRoot, maxBuffer: 32 * 1024 * 1024 }); await smokePayload(stagedPayload, metadata.version, runCommand); fs.renameSync(stagedPayload, payloadPath); return { payloadPath, reused: false, version: metadata.version }; @@ -266,7 +324,7 @@ export async function installCommand( dependencies: { runCommand?: CommandRunner; now?: () => Date } = {}, ): Promise { assertSupportedNodeVersion(); - const runCommand = dependencies.runCommand ?? execFileAsync; + const runCommand = dependencies.runCommand ?? runCommandWithDiagnostics; const gitRequest = resolveGitInstallRequest(options); if (gitRequest) { await confirmGitInstall(options, gitRequest.repo, gitRequest.ref); diff --git a/scripts/acpx-patch-packaging.test.mjs b/scripts/acpx-patch-packaging.test.mjs index 8ed4aafd773..227c77695b7 100644 --- a/scripts/acpx-patch-packaging.test.mjs +++ b/scripts/acpx-patch-packaging.test.mjs @@ -30,6 +30,7 @@ const dbPackage = JSON.parse( ); const releaseScript = await readFile(new URL("./release.sh", import.meta.url), "utf8"); const releaseLib = await readFile(new URL("./release-lib.sh", import.meta.url), "utf8"); +const buildNpmScript = await readFile(new URL("./build-npm.sh", import.meta.url), "utf8"); test("published packages preserve the patched ACPX runtime", () => { assert.equal( @@ -189,3 +190,8 @@ test("bundled package dry runs preview without querying published versions", () assert.match(releaseLib, /run_bundled_npm_publish publish --tag "\$dist_tag"/); assert.doesNotMatch(releaseLib, /run_bundled_npm_publish publish "\.\/\$tarball"/); }); + +test("npm builds use corepack instead of requiring a global pnpm", () => { + assert.match(buildNpmScript, /corepack pnpm -r typecheck/); + assert.doesNotMatch(buildNpmScript, /^\s*pnpm -r typecheck/m); +}); diff --git a/scripts/build-npm.sh b/scripts/build-npm.sh index 00b8acc2230..74d801fdfe7 100755 --- a/scripts/build-npm.sh +++ b/scripts/build-npm.sh @@ -37,7 +37,7 @@ fi if [ "$skip_typecheck" = false ]; then echo " [2/6] Type-checking..." cd "$REPO_ROOT" - pnpm -r typecheck + corepack pnpm -r typecheck else echo " [2/6] Skipping type-check (--skip-typecheck)" fi