From 9661a3050e2fd63d2b6768233d1a56ce4ee55357 Mon Sep 17 00:00:00 2001 From: Tonio Date: Thu, 27 Aug 2026 11:20:01 -0700 Subject: [PATCH] feat(dev): auto-install dependencies when the lockfile outruns the last install Pulling master with new dependencies left the dev server failing on unresolvable imports (motion/react, @sentry/browser) until someone manually ran pnpm install and cleared Vite's optimizer cache. The dev runner now detects a pnpm-lock.yaml newer than node_modules/.modules.yaml and runs pnpm install itself, then drops the stale Vite caches that otherwise keep serving 504 "Outdated Optimize Dep" responses. - boot (dev and watch modes): install before the migration preflight - dev mode: pnpm-lock.yaml joins the watched files; auto-restart installs first, and skips the restart instead of exiting if the install fails so the running server is never orphaned - watch mode: a lockfile watcher installs on change and nudges the server entry's mtime so tsx watch respawns onto the fresh node_modules - the manifest mtime is bumped after each install so a touched-but-unchanged lockfile cannot re-trigger installs forever Co-Authored-By: Claude Fable 5 --- scripts/dev-runner-install.ts | 72 +++++++++++ scripts/dev-runner.ts | 73 ++++++++++- .../src/__tests__/dev-runner-install.test.ts | 115 ++++++++++++++++++ 3 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 scripts/dev-runner-install.ts create mode 100644 server/src/__tests__/dev-runner-install.test.ts diff --git a/scripts/dev-runner-install.ts b/scripts/dev-runner-install.ts new file mode 100644 index 00000000000..02d42ca9cb6 --- /dev/null +++ b/scripts/dev-runner-install.ts @@ -0,0 +1,72 @@ +import { existsSync, rmSync, statSync, utimesSync } from "node:fs"; +import path from "node:path"; + +export interface StaleInstallCheck { + stale: boolean; + reason: string | null; +} + +function mtimeMs(filePath: string): number | null { + try { + return statSync(filePath).mtimeMs; + } catch { + return null; + } +} + +function modulesManifestPath(repoRoot: string): string { + return path.join(repoRoot, "node_modules", ".modules.yaml"); +} + +export function lockfilePath(repoRoot: string): string { + return path.join(repoRoot, "pnpm-lock.yaml"); +} + +// pnpm rewrites node_modules/.modules.yaml at the end of every install, so a +// lockfile newer than that manifest means the checkout picked up dependency +// changes (e.g. a pull) that were never installed. +export function detectStaleInstall(repoRoot: string): StaleInstallCheck { + const lockfileMtime = mtimeMs(lockfilePath(repoRoot)); + if (lockfileMtime === null) { + return { stale: false, reason: null }; + } + const manifestMtime = mtimeMs(modulesManifestPath(repoRoot)); + if (manifestMtime === null) { + return { stale: true, reason: "dependencies have never been installed in this checkout" }; + } + if (lockfileMtime > manifestMtime) { + return { stale: true, reason: "pnpm-lock.yaml changed after the last pnpm install" }; + } + return { stale: false, reason: null }; +} + +// The manifest mtime is the staleness marker, but a no-op install may skip +// rewriting it; bump it explicitly so a touched-but-unchanged lockfile cannot +// re-trigger installs forever. +export function markInstallFresh(repoRoot: string, now = new Date()): void { + try { + utimesSync(modulesManifestPath(repoRoot), now, now); + } catch { + // Manifest missing (unusual pnpm setup): the next boot re-runs a no-op install. + } +} + +export function resolveViteCachePaths(repoRoot: string): string[] { + return [ + path.join(repoRoot, "ui", "node_modules", ".vite"), + path.join(repoRoot, "node_modules", ".vite"), + ]; +} + +// Vite's optimizer keeps serving pre-bundled deps hashed against the old +// lockfile ("504 Outdated Optimize Dep") — drop the caches after an install so +// the next server boot re-bundles. +export function clearViteCaches(repoRoot: string): string[] { + const removed: string[] = []; + for (const cachePath of resolveViteCachePaths(repoRoot)) { + if (!existsSync(cachePath)) continue; + rmSync(cachePath, { recursive: true, force: true }); + removed.push(cachePath); + } + return removed; +} diff --git a/scripts/dev-runner.ts b/scripts/dev-runner.ts index 54e7198a74d..5f1eed63346 100644 --- a/scripts/dev-runner.ts +++ b/scripts/dev-runner.ts @@ -1,12 +1,13 @@ #!/usr/bin/env -S node --import tsx import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, rmSync, unwatchFile, utimesSync, watchFile, writeFileSync } from "node:fs"; import path from "node:path"; import { createInterface } from "node:readline/promises"; import { stdin, stdout } from "node:process"; import { createCapturedOutputBuffer, parseJsonResponseWithLimit } from "./dev-runner-output.ts"; import { applyDevRunnerOptions } from "./dev-runner-options.ts"; +import { clearViteCaches, detectStaleInstall, lockfilePath, markInstallFresh } from "./dev-runner-install.ts"; import { collectWatchedSnapshot as collectDevServerWatchedSnapshot, diffSnapshots } from "./dev-runner-snapshot.mjs"; import { createDevServiceIdentity, repoRoot } from "./dev-service-profile.ts"; import { bootstrapDevRunnerWorktreeEnv, isWorktreeSeedPending } from "../server/src/dev-runner-worktree.ts"; @@ -72,6 +73,7 @@ const watchedDirectories = [ const watchedFiles = [ ".env", "package.json", + "pnpm-lock.yaml", "pnpm-workspace.yaml", "tsconfig.base.json", "tsconfig.json", @@ -491,6 +493,42 @@ async function maybePreflightMigrations(options: { interactive?: boolean; autoAp await refreshPendingMigrations(); } +let installInFlight = false; + +async function maybeAutoInstallDependencies(options: { exitOnFailure?: boolean } = {}): Promise { + const exitOnFailure = options.exitOnFailure ?? true; + if (installInFlight) return true; + const check = detectStaleInstall(repoRoot); + if (!check.stale) return true; + + installInFlight = true; + try { + console.log(`[paperclip] ${check.reason}; running pnpm install...`); + const result = await runPnpm(["install"], { stdio: "inherit", cwd: repoRoot }); + if (result.signal) { + exitForSignal(result.signal); + return false; + } + if (result.code !== 0) { + console.error("[paperclip] pnpm install failed; run it manually, then restart `pnpm dev`"); + if (exitOnFailure) { + process.exit(result.code); + } + return false; + } + markInstallFresh(repoRoot); + const removedCaches = clearViteCaches(repoRoot); + if (removedCaches.length > 0) { + console.log( + `[paperclip] cleared stale Vite caches: ${removedCaches.map((cachePath) => path.relative(repoRoot, cachePath)).join(", ")}`, + ); + } + return true; + } finally { + installInFlight = false; + } +} + async function buildPluginSdk() { console.log("[paperclip] building plugin sdk..."); const result = await runPnpm( @@ -638,6 +676,12 @@ async function maybeAutoRestartChild() { } try { + // Never exit here on a failed install: the old server child is still + // running and would be orphaned. Skip the restart and keep serving. + const installed = await maybeAutoInstallDependencies({ exitOnFailure: false }); + if (!installed) { + return; + } await maybePreflightMigrations({ autoApply: true, interactive: false, @@ -654,6 +698,30 @@ async function maybeAutoRestartChild() { } } +// In watch mode the server child restarts itself (tsx watch) on source +// changes, but nothing reinstalls dependencies when a pull updates the +// lockfile — the server then respawns against stale node_modules and dies on +// unresolvable imports. Watch the lockfile, install, and nudge the server +// entry's mtime so tsx watch does one clean respawn on the fresh install. +function installWatchModeLockfileWatcher() { + if (mode !== "watch") return; + + const serverEntryPath = path.join(repoRoot, "server", "src", "index.ts"); + watchFile(lockfilePath(repoRoot), { interval: 2000 }, () => { + void (async () => { + if (!detectStaleInstall(repoRoot).stale) return; + const installed = await maybeAutoInstallDependencies({ exitOnFailure: false }); + if (!installed) return; + try { + const now = new Date(); + utimesSync(serverEntryPath, now, now); + } catch { + console.warn("[paperclip] dependencies reinstalled; restart the server to pick them up"); + } + })(); + }); +} + function installDevIntervals() { if (mode !== "dev") return; @@ -680,6 +748,7 @@ async function shutdown(signal: NodeJS.Signals) { if (shuttingDown) return; shuttingDown = true; clearDevIntervals(); + unwatchFile(lockfilePath(repoRoot)); clearDevServerStatus(); await removeLocalServiceRegistryRecord(devService.serviceKey); @@ -705,9 +774,11 @@ process.on("SIGTERM", () => { void shutdown("SIGTERM"); }); +await maybeAutoInstallDependencies(); await maybePreflightMigrations(); await startServerChild(); installDevIntervals(); +installWatchModeLockfileWatcher(); if (mode === "watch") { const exit = await waitForChildExit(); diff --git a/server/src/__tests__/dev-runner-install.test.ts b/server/src/__tests__/dev-runner-install.test.ts new file mode 100644 index 00000000000..e70ed02137d --- /dev/null +++ b/server/src/__tests__/dev-runner-install.test.ts @@ -0,0 +1,115 @@ +import { existsSync, mkdirSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + clearViteCaches, + detectStaleInstall, + markInstallFresh, + resolveViteCachePaths, +} from "../../../scripts/dev-runner-install.ts"; + +const tempRoots: string[] = []; + +function makeRepoRoot(): string { + const repoRoot = mkdtempSync(path.join(os.tmpdir(), "paperclip-dev-runner-install-")); + tempRoots.push(repoRoot); + return repoRoot; +} + +function writeLockfile(repoRoot: string, mtime: Date): void { + const lockfile = path.join(repoRoot, "pnpm-lock.yaml"); + writeFileSync(lockfile, "lockfileVersion: '9.0'\n", "utf8"); + utimesSync(lockfile, mtime, mtime); +} + +function writeModulesManifest(repoRoot: string, mtime: Date): void { + const manifest = path.join(repoRoot, "node_modules", ".modules.yaml"); + mkdirSync(path.dirname(manifest), { recursive: true }); + writeFileSync(manifest, "hoistPattern:\n - '*'\n", "utf8"); + utimesSync(manifest, mtime, mtime); +} + +afterEach(() => { + for (const repoRoot of tempRoots.splice(0)) { + rmSync(repoRoot, { recursive: true, force: true }); + } +}); + +describe("detectStaleInstall", () => { + it("is fresh when there is no lockfile to compare against", () => { + const repoRoot = makeRepoRoot(); + + expect(detectStaleInstall(repoRoot)).toEqual({ stale: false, reason: null }); + }); + + it("is stale when dependencies were never installed", () => { + const repoRoot = makeRepoRoot(); + writeLockfile(repoRoot, new Date("2026-08-27T10:00:00Z")); + + const check = detectStaleInstall(repoRoot); + expect(check.stale).toBe(true); + expect(check.reason).toMatch(/never been installed/); + }); + + it("is stale when the lockfile is newer than the last install", () => { + const repoRoot = makeRepoRoot(); + writeModulesManifest(repoRoot, new Date("2026-08-27T10:00:00Z")); + writeLockfile(repoRoot, new Date("2026-08-27T11:00:00Z")); + + const check = detectStaleInstall(repoRoot); + expect(check.stale).toBe(true); + expect(check.reason).toMatch(/pnpm-lock\.yaml changed/); + }); + + it("is fresh when the last install is newer than the lockfile", () => { + const repoRoot = makeRepoRoot(); + writeLockfile(repoRoot, new Date("2026-08-27T10:00:00Z")); + writeModulesManifest(repoRoot, new Date("2026-08-27T11:00:00Z")); + + expect(detectStaleInstall(repoRoot)).toEqual({ stale: false, reason: null }); + }); +}); + +describe("markInstallFresh", () => { + it("bumps the manifest so a touched lockfile stops reading as stale", () => { + const repoRoot = makeRepoRoot(); + writeModulesManifest(repoRoot, new Date("2026-08-27T10:00:00Z")); + writeLockfile(repoRoot, new Date("2026-08-27T11:00:00Z")); + expect(detectStaleInstall(repoRoot).stale).toBe(true); + + markInstallFresh(repoRoot, new Date("2026-08-27T12:00:00Z")); + + expect(detectStaleInstall(repoRoot)).toEqual({ stale: false, reason: null }); + }); + + it("tolerates a missing manifest", () => { + const repoRoot = makeRepoRoot(); + + expect(() => markInstallFresh(repoRoot)).not.toThrow(); + }); +}); + +describe("clearViteCaches", () => { + it("removes existing optimizer caches and reports them", () => { + const repoRoot = makeRepoRoot(); + const cachePaths = resolveViteCachePaths(repoRoot); + for (const cachePath of cachePaths) { + mkdirSync(path.join(cachePath, "deps"), { recursive: true }); + writeFileSync(path.join(cachePath, "deps", "_metadata.json"), "{}", "utf8"); + } + + const removed = clearViteCaches(repoRoot); + + expect(removed.sort()).toEqual([...cachePaths].sort()); + for (const cachePath of cachePaths) { + expect(existsSync(cachePath)).toBe(false); + } + }); + + it("is a no-op when no caches exist", () => { + const repoRoot = makeRepoRoot(); + + expect(clearViteCaches(repoRoot)).toEqual([]); + }); +});