diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 65f72fa8c56..2ff512638af 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -788,6 +788,25 @@ jobs: pnpm install --frozen-lockfile fi + # The public-install verifier at the end of this lane downloads the whole + # public registry graph into an empty npm cache: its consumer install + # took 119-129s of the 226-241s step, more than half of this ~415-446s + # lane, in runs 38426120144, 38432086560 and 38423713537 (2026-10-13). + # That graph is fixed by source manifests, so this fetches it in the + # background while the Rust restore, Evalbook and `pnpm build` run, about + # 150s before the verifier installs anything. Only the cache is handed + # over, and the verify step waits for this first. A failed or partial + # warm-up leaves the verifier to fetch what is missing as before. + - name: Start warming the public-install npm cache + if: ${{ hashFiles('scripts/warm-public-install-npm-cache.mjs') != '' }} + run: | + ( + # The default `bash -e` would skip the status line on a failure. + set +e + PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE="$RUNNER_TEMP/public-install-npm-cache" node scripts/warm-public-install-npm-cache.mjs + echo $? > "$RUNNER_TEMP/public-install-npm-cache.status" + ) > "$RUNNER_TEMP/public-install-npm-cache.log" 2>&1 & + # pnpm build reaches the Runner package's build:binary step; without the # shared Rust cache that is a ~3m40s cold compile of all third-party # crates (run 35036001734, 2026-09-15). Same restore-only contract as @@ -914,7 +933,20 @@ jobs: # docker-runner-check.yml runs the same verifier after `pnpm build`. - name: Verify built-in Grok from a clean public npm install if: ${{ hashFiles('scripts/verify-grok-npm-install.mjs') != '' }} - run: node scripts/verify-grok-npm-install.mjs + run: | + status="$RUNNER_TEMP/public-install-npm-cache.status" + if [ -f "$RUNNER_TEMP/public-install-npm-cache.log" ]; then + # Normally finished long before this; never let it hold the lane. + for _ in $(seq 180); do [ -f "$status" ] && break; sleep 1; done + if [ "$(cat "$status" 2>/dev/null)" = 0 ]; then + tail -n 1 "$RUNNER_TEMP/public-install-npm-cache.log" + else + echo '::notice title=Public-install npm cache::warm-up did not finish cleanly; the verifier fetches what is missing' + tail -n 20 "$RUNNER_TEMP/public-install-npm-cache.log" + fi + export PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE="$RUNNER_TEMP/public-install-npm-cache" + fi + node scripts/verify-grok-npm-install.mjs verify_serialized_server: name: Verify serialized server suites (${{ matrix.shard_label }}) diff --git a/package.json b/package.json index df1a2323f53..66cf20785c7 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index 8c43c72a133..a3cbfbd7f2b 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -9,6 +9,7 @@ import { fileURLToPath } from 'node:url'; import { isWorkspacePackage, parseWorkspaceEntries } from '../build-standalone-public-packages.mjs'; import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertInstalledCodexPlatformProvenance, assertInstalledNativeConsumerContinuity, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource, installedDaemonProbeSource, macNodeHeadersDirectory, macPublicInstallLifecycleProfile, macInstalledStartupProbeSource, runMacInitialInstallWithDiagnostics } from '../grok-public-install-sandbox.mjs'; import { verifyPiPackagedDaemon } from '../pi-public-install-probe.mjs'; +import { publicInstallClosure } from '../public-install-closure.mjs'; import { runnerBinaryTarget } from '../../packages/paperclip-runner/src/live/runner-binary.ts'; const paths = { assets: '/private/staging/assets', consumer: '/private/staging/consumer', cache: '/private/staging/cache', uid: 1001, gid: 1001 }; @@ -209,7 +210,7 @@ test('the image and Grok downloads start before staging and join without weakeni assert.ok(source.includes("const stagedGrok = join(prefetched, 'grok')")); assert.doesNotMatch(source, /provision-grok\.mjs'\), prerequisite\]/); // Both downloads start before any package is staged and before the first sandbox runs. - const staging = position("visit('@paperclipai/server')"); + const staging = position('publicInstallClosure(repo)'); const firstSandbox = position("isolated(['npm', 'install', '--ignore-scripts', '--omit=dev'"); assert.ok(imagePull < staging && grokProvision < staging); assert.ok(position("await awaited('image', imageReady)") < firstSandbox); @@ -284,17 +285,8 @@ test('the verifier stages only workspace packages, so it skips the standalone bu const retention = readFileSync(new URL('../retain-runner-qualification-packages.mjs', import.meta.url), 'utf8'); assert.ok(retention.includes("const pluginRelative = 'packages/plugins/sandbox-providers/daytona';") && retention.includes("cpSync(join(repo, pluginRelative, 'dist'), join(plugin, 'dist'), { recursive: true })"), 'retention still packs the Daytona plugin from its dist, which is why the gated build exists'); const repo = new URL('../../', import.meta.url); - const listing = spawnSync(process.execPath, [fileURLToPath(new URL('scripts/release-package-map.mjs', repo)), 'list'], { cwd: fileURLToPath(repo), encoding: 'utf8' }); - assert.equal(listing.status, 0, listing.stderr); - const packages = new Map(listing.stdout.trim().split('\n').map(line => line.split('\t')).map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(new URL(`${dir}/package.json`, repo), 'utf8')) }])); - const needed = new Set(); - const visit = name => { - if (needed.has(name)) return; - const entry = packages.get(name); assert.ok(entry, `missing public workspace dependency ${name}`); needed.add(name); - for (const [dep, spec] of Object.entries({ ...entry.manifest.dependencies, ...entry.manifest.optionalDependencies })) if (spec.startsWith('workspace:')) visit(dep); - }; - assert.ok(source.includes("visit('@paperclipai/server');\n visit('paperclipai');")); - visit('@paperclipai/server'); visit('paperclipai'); + assert.ok(source.includes('const { packages, needed } = publicInstallClosure(repo);')); + const { packages, needed } = publicInstallClosure(fileURLToPath(repo)); const workspace = parseWorkspaceEntries(readFileSync(new URL('pnpm-workspace.yaml', repo), 'utf8')); const standalone = [...packages.values()].filter(({ dir }) => !isWorkspacePackage(dir, workspace)).map(({ dir }) => dir); assert.ok(standalone.length > 0, 'the public package map must still carry standalone packages for this check to mean anything'); @@ -307,7 +299,7 @@ test('the verifier stages only workspace packages, so it skips the standalone bu assert.deepEqual(JSON.parse(readFileSync(new URL('server/package.json', repo), 'utf8')).files, ['dist', 'ui-dist', 'skills']); assert.ok(source.includes("PAPERCLIP_RELEASE_REUSE_UI_DIST: '1'")); const uiDist = position("run('bash', [join(repo, 'scripts/prepare-server-ui-dist.sh')], repo)"); - assert.ok(position("visit('paperclipai')") < gate && gate < uiDist && uiDist < position('timing.stage_ui_dist_ms =') && uiDist < position('const stageSource = ')); + assert.ok(position('publicInstallClosure(repo)') < gate && gate < uiDist && uiDist < position('timing.stage_ui_dist_ms =') && uiDist < position('const stageSource = ')); }); test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => { diff --git a/scripts/__tests__/public-install-closure.test.mjs b/scripts/__tests__/public-install-closure.test.mjs new file mode 100644 index 00000000000..9dd9ac2ddf5 --- /dev/null +++ b/scripts/__tests__/public-install-closure.test.mjs @@ -0,0 +1,62 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import test from 'node:test'; +import { fileURLToPath } from 'node:url'; +import { PUBLIC_INSTALL_ROOTS, publicInstallClosure, publicRegistryDependencies } from '../public-install-closure.mjs'; + +const repo = fileURLToPath(new URL('../../', import.meta.url)); + +test('the closure starts at the server and the CLI and follows only workspace specs', () => { + assert.deepEqual(PUBLIC_INSTALL_ROOTS, ['@paperclipai/server', 'paperclipai']); + const { packages, needed } = publicInstallClosure(repo); + const order = [...needed]; + assert.equal(order[0], '@paperclipai/server'); + assert.ok(needed.has('paperclipai')); + for (const name of needed) { + const { manifest } = packages.get(name); + for (const [dep, spec] of Object.entries({ ...manifest.dependencies, ...manifest.optionalDependencies })) { + assert.equal(needed.has(dep), spec.startsWith('workspace:'), `${name} -> ${dep}@${spec}`); + } + } +}); + +test('the warm-up graph is the closure\'s registry dependencies, first spec wins', () => { + const pkg = (dependencies, optionalDependencies) => ({ dir: 'x', manifest: { dependencies, optionalDependencies } }); + const packages = new Map([ + ['a', pkg({ b: 'workspace:*', left: '^1.0.0', shared: '^2.0.0', local: 'file:../local', linked: 'link:../linked' }, { native: '1.2.3' })], + ['b', pkg({ shared: '^3.0.0', right: '~4.0.0' }, { left: '^9.0.0' })], + ['unvisited', pkg({ never: '1.0.0' })], + ]); + assert.deepEqual(publicRegistryDependencies({ packages, needed: new Set(['a', 'b']) }), { + dependencies: { left: '^1.0.0', shared: '^2.0.0', right: '~4.0.0' }, + optionalDependencies: { native: '1.2.3' }, + }); + const real = publicRegistryDependencies(publicInstallClosure(repo)); + const names = [...Object.keys(real.dependencies), ...Object.keys(real.optionalDependencies)]; + assert.ok(names.length > 0); + for (const spec of [...Object.values(real.dependencies), ...Object.values(real.optionalDependencies)]) { + assert.doesNotMatch(spec, /^(workspace|file|link):/); + } +}); + +test('the warm-up refuses to run without the cache the verifier will read', () => { + const env = { ...process.env }; + delete env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE; + const result = spawnSync(process.execPath, [fileURLToPath(new URL('../warm-public-install-npm-cache.mjs', import.meta.url))], { env, encoding: 'utf8' }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE/); +}); + +test('the Build lane warms the cache before pnpm build and hands the same directory to the verifier', () => { + const workflow = readFileSync(new URL('../../.github/workflows/pr-trusted.yml', import.meta.url), 'utf8'); + const build = workflow.slice(workflow.indexOf('\n build:\n'), workflow.indexOf('\n verify_serialized_server:\n')); + const position = fragment => { const index = build.indexOf(fragment); assert.notEqual(index, -1, `Build job must contain ${fragment}`); return index; }; + const warm = position('PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE="$RUNNER_TEMP/public-install-npm-cache" node scripts/warm-public-install-npm-cache.mjs'); + assert.ok(position('- name: Install dependencies') < warm && warm < position('- name: Build\n'), 'the warm-up overlaps the Rust restore and pnpm build'); + assert.ok(build.includes('set +e\n'), 'a failed warm-up still records its status, so the verifier never waits it out'); + const verify = position('export PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE="$RUNNER_TEMP/public-install-npm-cache"'); + assert.ok(verify < position('node scripts/verify-grok-npm-install.mjs')); + const verifier = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + assert.ok(verifier.includes("const cache = process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE ? resolve(process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE) : join(root, 'cache');")); +}); diff --git a/scripts/public-install-closure.mjs b/scripts/public-install-closure.mjs new file mode 100644 index 00000000000..f1e375f6404 --- /dev/null +++ b/scripts/public-install-closure.mjs @@ -0,0 +1,49 @@ +// The public packages the public-install verifier stages: `@paperclipai/server` +// and `paperclipai` plus every public workspace package they reach through +// `workspace:` specs, in the depth-first order the verifier packs them. The +// npm cache warm-up reads the same closure from source manifests before +// `pnpm build` has produced anything, so both share this one definition. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const PUBLIC_INSTALL_ROOTS = ['@paperclipai/server', 'paperclipai']; + +// Returns `{ packages, needed }`: `packages` maps every public workspace +// package name to `{ dir, manifest }`, and `needed` is the closure in visit +// order. +export function publicInstallClosure(repo, roots = PUBLIC_INSTALL_ROOTS) { + const listing = execFileSync(process.execPath, [join(repo, 'scripts/release-package-map.mjs'), 'list'], { cwd: repo, stdio: 'pipe' }) + .toString().trim().split('\n').map(line => line.split('\t')); + const packages = new Map(listing.map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(join(repo, dir, 'package.json'), 'utf8')) }])); + const needed = new Set(); + function visit(name) { + if (needed.has(name)) return; + const entry = packages.get(name); assert.ok(entry, `Missing public workspace dependency ${name}`); needed.add(name); + for (const [dep, spec] of Object.entries({ ...entry.manifest.dependencies, ...entry.manifest.optionalDependencies })) { + if (spec.startsWith('workspace:')) visit(dep); + } + } + for (const root of roots) visit(root); + return { packages, needed }; +} + +// The registry dependencies the closure's packages declare, as one consumer +// manifest. A name two packages declare with different specs keeps the first +// spec: the warm-up only fills a cache, and the real install fetches whatever +// it still lacks. +export function publicRegistryDependencies({ packages, needed }) { + const dependencies = {}; + const optionalDependencies = {}; + for (const name of needed) { + const { manifest } = packages.get(name); + for (const [section, target] of [['dependencies', dependencies], ['optionalDependencies', optionalDependencies]]) { + for (const [dep, spec] of Object.entries(manifest[section] ?? {})) { + if (needed.has(dep) || /^(workspace|file|link):/.test(spec) || dep in dependencies || dep in optionalDependencies) continue; + target[dep] = spec; + } + } + } + return { dependencies, optionalDependencies }; +} diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index f7af462f8a3..ea01fa01bb0 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -13,6 +13,7 @@ import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs'; import { materializePublishManifest } from './prepare-bundled-package.mjs'; +import { publicInstallClosure } from './public-install-closure.mjs'; import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertInstalledNativeConsumerContinuity, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource, installedDaemonProbeSource, macNodeHeadersDirectory, macPublicInstallLifecycleProfile, macInstalledStartupProbeSource, runMacInitialInstallWithDiagnostics } from './grok-public-install-sandbox.mjs'; import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'; import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs'; @@ -229,18 +230,10 @@ try { // Provision as the unprivileged verification user, never into the host's /opt. // Only the positive probe sees this file at the canonical sandbox path. const grokReady = prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok]); - const listing = run(process.execPath, [join(repo, 'scripts/release-package-map.mjs'), 'list'], repo).toString().trim().split('\n').map(line => line.split('\t')); - const packages = new Map(listing.map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(join(repo, dir, 'package.json'), 'utf8')) }])); - const needed = new Set(); - function visit(name) { - if (needed.has(name)) return; - const entry = packages.get(name); assert.ok(entry, `Missing public workspace dependency ${name}`); needed.add(name); - for (const [dep, spec] of Object.entries({ ...entry.manifest.dependencies, ...entry.manifest.optionalDependencies })) { - if (spec.startsWith('workspace:')) visit(dep); - } - } - visit('@paperclipai/server'); - visit('paperclipai'); + // The server, the CLI and every public workspace package they reach through + // `workspace:` specs. The npm cache warm-up reads the same closure, so both + // take it from one module. + const { packages, needed } = publicInstallClosure(repo); // release.sh also builds the standalone public packages (the sandbox-provider // plugins outside the pnpm workspace) before it stages anything, because it // publishes them. Nothing staged here reaches them: `needed` is closed under @@ -340,7 +333,13 @@ try { } const assets = join(root, 'assets'); mkdirSync(assets, { mode: 0o755 }); const consumer = join(root, 'consumer'); mkdirSync(consumer); - const cache = join(root, 'cache'); mkdirSync(cache); + // CI can point this at a cache that scripts/warm-public-install-npm-cache.mjs + // filled with the same public registry graph while `pnpm build` ran. npm + // checks every cached tarball against the integrity its own resolution + // records, so a warm cache changes where bytes come from, never what is + // installed; the consumer directory and its lock still start empty. + const cache = process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE ? resolve(process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE) : join(root, 'cache'); + mkdirSync(cache, { recursive: true }); writeFileSync(join(consumer, 'package.json'), JSON.stringify({ private: true, type: 'module' })); for (const tarball of tarballs) { const destination = join(assets, basename(tarball)); cpSync(tarball, destination); chmodSync(destination, 0o644); diff --git a/scripts/warm-public-install-npm-cache.mjs b/scripts/warm-public-install-npm-cache.mjs new file mode 100755 index 00000000000..a6ee8f3b194 --- /dev/null +++ b/scripts/warm-public-install-npm-cache.mjs @@ -0,0 +1,37 @@ +#!/usr/bin/env node +// Fill an npm cache with the public registry graph of the packages the +// public-install verifier stages, while `pnpm build` still runs. The verifier +// starts after the build with an empty cache, and its consumer install then +// downloads the whole graph (106-127s in runs 38241687458 to 38250487646, +// 2026-10-12). The graph is fixed by source manifests, so it can be fetched +// before any tarball exists. +// +// This only downloads. Scripts are disabled, nothing here is verified, and the +// scratch consumer is removed afterwards. npm checks every cached tarball +// against the integrity its own resolution records, so a warm cache changes +// where bytes come from, never what the verifier installs. Any failure leaves +// a partial cache that the verifier completes from the registry as before. +import assert from 'node:assert/strict'; +import { execFileSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { publicInstallClosure, publicRegistryDependencies } from './public-install-closure.mjs'; + +const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); +const cache = process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE; +assert.ok(cache, 'Set PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE to the cache directory the verifier will use'); +mkdirSync(cache, { recursive: true }); +const started = performance.now(); +const scratch = mkdtempSync(join(tmpdir(), 'paperclip-public-install-warm-')); +try { + const { dependencies, optionalDependencies } = publicRegistryDependencies(publicInstallClosure(repo)); + writeFileSync(join(scratch, 'package.json'), JSON.stringify({ private: true, dependencies, optionalDependencies })); + execFileSync('npm', ['install', '--ignore-scripts', '--omit=dev', '--no-audit', '--no-fund', '--no-package-lock'], { + cwd: scratch, stdio: 'inherit', env: { ...process.env, npm_config_cache: cache, npm_config_update_notifier: 'false' }, + }); + console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.cache-warm.v1', dependencyCount: Object.keys(dependencies).length + Object.keys(optionalDependencies).length, warm_ms: Math.round(performance.now() - started) })); +} finally { + rmSync(scratch, { recursive: true, force: true }); +}