diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index 954e8fd597d..cf0ab7c636c 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -205,3 +205,19 @@ the public npm registry; public tarball URLs remain unchanged. The temporary registry closes before the existing offline lifecycle rebuild and runtime probes. No lifecycle, network-isolation, lock-preservation or provider checks are skipped. Native ARM and Mac qualification keep their existing file installs. + +The Linux verifier also shares `PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE` with the +producer installs that prepare bundled dependencies. These run after the existing +cache warm-up and reuse its downloaded archives instead of populating a second +host cache. Each producer still installs into a separate empty directory, applies +the release patches, and packs fresh output. npm resolution and integrity checks +remain enabled; cache misses download normally. Without an explicit shared cache, +producer installs keep their normal npm cache configuration. + +After the offline lifecycle finishes, Pi setup runs in its own download sandbox +while the independent Codex/Grok probes run offline. Pi admission still waits +for successful setup and an unchanged consumer lock. The setup container has a +unique verifier-owned name and is removed on early failure as well as completion. +`pi_setup_prefetch_ms` measures setup wall time; `pi_setup_wait_ms` measures only +the remaining wait after the other probes. These overlapping durations must not +be added when interpreting total verification time. diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index b2bf6dd1e6b..66401392a44 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -244,7 +244,7 @@ test('the bundled-dependency installs run together while the other packages are // cleanup as it starts, with the results put back into dependency order // before every tarball is checked. const bundledSources = position('for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry)'); - const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo })"); + const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo, env })"); const registered = position('prefetches.push(...installs.map(({ handle }) => handle))'); const otherSources = position('for (const entry of entries.filter(entry => !entry.bundled))'); const join = position("await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)))"); @@ -596,3 +596,19 @@ test('Pi admission verifies assembled host daemon identity and preserves the fla await assert.rejects(verifyPiPackagedDaemon(server, flat, target, runnerBinaryTarget), /manifest must be an object/); } finally { rmSync(root, { recursive: true, force: true }); } }); + + +test('Pi setup overlaps independent probes only after the offline lifecycle and joins before admission', () => { + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, fragment); return index; }; + const lifecycle = position("phase('lifecycle_rebuild')"); + const start = position("prefetch('pi_setup', 'docker', piSetupArgs)"); + const codex = position("const codex = JSON.parse(isolated(['node', '/packages/codex-probe.mjs'])"); + const grok = position("phase('grok_probes')"); + const join = position("await awaited('pi_setup', piSetupReady)"); + const receipt = position("assert.equal(receipt.status, 'installed_verified')"); + const probe = position("console.log(isolated(['node', '/packages/pi-public-install-probe.mjs'"); + assert.ok(lifecycle < start && start < codex && codex < grok && grok < join && join < receipt && receipt < probe); + assert.ok(source.includes("piSetupArgs.splice(1, 0, '--name', piSetupContainer)")); + assert.ok(position("await stopPrefetches(prefetches)") < position("['rm', '--force', piSetupContainer]")); +}); diff --git a/scripts/__tests__/stage-bundled-packages.test.mjs b/scripts/__tests__/stage-bundled-packages.test.mjs index d42ddf14cf7..c8d0c7fb609 100644 --- a/scripts/__tests__/stage-bundled-packages.test.mjs +++ b/scripts/__tests__/stage-bundled-packages.test.mjs @@ -47,18 +47,18 @@ test('the installs run at the same time, each with its own source and target', a if (Date.now() > deadline) { console.error('the other install never started'); process.exit(3); } Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); } - writeFileSync(join(target, 'package.json'), '{}'); + writeFileSync(join(target, 'package.json'), JSON.stringify({ cache: process.env.npm_config_cache })); `); const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db']); for (const { target } of entries) mkdirSync(target); - const installs = startBundledInstalls(entries, { logDir: root, cwd: root, script }); + const installs = startBundledInstalls(entries, { logDir: root, cwd: root, script, env: { ...process.env, npm_config_cache: join(root, 'shared-cache') } }); assert.deepEqual(installs.map(({ slug }) => slug), ['server', 'db']); await Promise.all(installs.map(({ handle }) => handle.done)); for (const [index, { stagedSource, target, slug, handle }] of installs.entries()) { assert.equal(handle.child.exitCode, 0); assert.equal(readFileSync(join(root, `bundled-${slug}.log`), 'utf8'), `install ${stagedSource} -> ${target}\n`); assert.equal(target, join(root, `package-${index}`)); - assert.ok(existsSync(join(target, 'package.json'))); + assert.equal(JSON.parse(readFileSync(join(target, 'package.json'), 'utf8')).cache, join(root, 'shared-cache')); } const timings = bundledInstallTimings(installs); assert.deepEqual(Object.keys(timings), ['stage_bundled_server_ms', 'stage_bundled_db_ms']); diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index 2ff9a5e8a8a..beb92bfa272 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -197,7 +197,9 @@ if (process.argv[2] === '--linux-arm64-consumer') { assert.equal(process.platform, 'linux', 'Run this verification on disposable EC2 Linux, not a developer host'); const root = mkdtempSync(join(tmpdir(), 'paperclip-grok-public-install-')); const prerequisite = join(root, 'native/grok'); -const env = { ...process.env, NODE_PATH: '', PAPERCLIP_RELEASE_REUSE_UI_DIST: '1', npm_config_ignore_scripts: 'false', npm_config_audit: 'false', npm_config_fund: 'false' }; +const cache = process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE ? resolve(process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE) : join(root, 'cache'); +mkdirSync(cache, { recursive: true }); +const env = { ...process.env, NODE_PATH: '', ...(process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE ? { npm_config_cache: cache } : {}), PAPERCLIP_RELEASE_REUSE_UI_DIST: '1', npm_config_ignore_scripts: 'false', npm_config_audit: 'false', npm_config_fund: 'false' }; const run = (cmd, args, cwd = root) => execFileSync(cmd, args, { cwd, env, stdio: 'pipe', maxBuffer: 32 * 1024 * 1024 }); const sourceRevision = run('git', ['rev-parse', 'HEAD'], repo).toString().trim(); const releaseVersion = `0.0.0-grok-verify.${sourceRevision.slice(0, 12)}`; @@ -215,6 +217,8 @@ const stagedGrok = join(prefetched, 'grok'); const prefetches = []; const downloads = []; const timing = {}; +const piSetupContainer = `paperclip-pi-setup-${basename(root)}`; +let piSetupStarted = false; // A prefetch failure surfaces where its result is awaited, never as an // unhandled rejection that would mask an earlier staging error. const prefetch = (name, cmd, args) => { @@ -291,7 +295,7 @@ try { const sourcesStarted = performance.now(); for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry); const installsStarted = performance.now(); - const installs = startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo }); + const installs = startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo, env }); prefetches.push(...installs.map(({ handle }) => handle)); for (const entry of entries.filter(entry => !entry.bundled)) { const releaseManifest = stageSource(entry); @@ -339,8 +343,6 @@ try { // checks every cached tarball against the integrity its own resolution // records, so a warm cache changes where bytes come from, never what is // installed; the consumer directory and its lock still start empty. - const cache = process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE ? resolve(process.env.PAPERCLIP_PUBLIC_INSTALL_NPM_CACHE) : join(root, 'cache'); - mkdirSync(cache, { recursive: true }); writeFileSync(join(consumer, 'package.json'), JSON.stringify({ private: true, type: 'module' })); for (const tarball of tarballs) { const destination = join(assets, basename(tarball)); cpSync(tarball, destination); chmodSync(destination, 0o644); @@ -374,6 +376,13 @@ try { assert.equal(readFileSync(sentinel, 'utf8'), 'ok', 'Offline lifecycle scripts must actually execute'); assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Lifecycle execution must preserve the resolved consumer lock'); phase('lifecycle_rebuild'); + const piSetupReceipt = join(consumer, '.paperclip-pi-setup.json'); + const piSetupArgs = grokConsumerDockerArgs({ assets, consumer, cache, uid: consumerUid, gid: process.getgid(), + command: ['sh', '-c', 'exec node /consumer/node_modules/paperclipai/dist/index.js runtime setup pi > /consumer/.paperclip-pi-setup.json'], + download: true, temporarySizeMiB: 2048 }); + piSetupArgs.splice(1, 0, '--name', piSetupContainer); + const piSetupReady = prefetch('pi_setup', 'docker', piSetupArgs); + piSetupStarted = true; for (const name of needed) { const installedManifest = JSON.parse(readFileSync(join(consumer, 'node_modules', name, 'package.json'), 'utf8')); assert.equal(installedManifest.version, releaseVersion, `Installed release version for ${name}`); @@ -422,8 +431,9 @@ try { // Pi assembles a bundled runtime in scratch space before atomic publication. // Keep the existing sandbox and memory bound; only this download needs more // temporary capacity than the smaller offline lifecycle probes. - const setup = isolated(['node', '/consumer/node_modules/paperclipai/dist/index.js', 'runtime', 'setup', 'pi'], { download: true, temporarySizeMiB: 2048 }).toString(); - const receipt = JSON.parse(setup.trim()); + await awaited('pi_setup', piSetupReady); + const receipt = JSON.parse(readFileSync(piSetupReceipt, 'utf8')); + rmSync(piSetupReceipt); phase('pi_setup'); assert.equal(receipt.status, 'installed_verified'); assert.equal(receipt.target, 'linux-x64'); @@ -453,6 +463,9 @@ try { // An earlier failure must not leave a prefetch running, and scratch is // removed only once both children have stopped writing into it. await stopPrefetches(prefetches); + if (piSetupStarted) { + try { execFileSync('docker', ['rm', '--force', piSetupContainer], { timeout: 10_000, stdio: 'ignore' }); } catch {} + } rmSync(root, { recursive: true, force: true }); } }