diff --git a/package.json b/package.json index 3e026aa985..5a5000234b 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index d084ee9be8..5a3dc487e3 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -81,6 +81,35 @@ test('verification never elevates PR-controlled provisioning or cleanup on the h assert.ok(source.includes("const prerequisite = join(root, 'native/grok')")); }); +test('the image and Grok downloads start before staging and join without weakening the npm checks', () => { + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; }; + // Exactly the pinned consumer image is pulled, for the platform the probes run on. + const imagePull = position("prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE])"); + // Grok is provisioned into the prefetch directory, never straight into `native/grok`. + const grokProvision = position("prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok])"); + assert.ok(source.includes("const stagedGrok = join(prefetched, 'grok')")); + assert.doesNotMatch(source, /provision-grok\.mjs'\), prerequisite\]/); + // Both downloads start before any package is staged and before the first sandbox runs. + const staging = position("visit('@paperclipai/server')"); + const firstSandbox = position("isolated(['npm', 'install', '--ignore-scripts', '--omit=dev'"); + assert.ok(imagePull < staging && grokProvision < staging); + assert.ok(position("await awaited('image', imageReady)") < firstSandbox); + // The binary moves to the canonical host path only after npm is proven not to + // have provisioned it, after the negative probe, and before the positive probe. + const npmCheck = position("'npm must not provision Grok'"); + const negativeProbe = position("isolated(['node', '/packages/probe.mjs', 'missing'])"); + const grokJoin = position("await awaited('grok', grokReady)"); + const move = position('renameSync(stagedGrok, prerequisite)'); + const positiveProbe = position("isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite })"); + assert.ok(npmCheck < negativeProbe && negativeProbe < grokJoin && grokJoin < move && move < positiveProbe); + // The children run through the helper covered by prefetch-child.test.mjs, and + // cleanup waits for them before removing the directory they write into. + assert.ok(source.includes("import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'")); + assert.ok(position('await stopPrefetches(prefetches)') < position('rmSync(root, { recursive: true, force: true })')); + assert.equal(source.match(/rmSync\(root/g).length, 1); +}); + test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => { const root = realpathSync(mkdtempSync(join(tmpdir(), 'installed-codex-probe-'))); try { diff --git a/scripts/__tests__/prefetch-child.test.mjs b/scripts/__tests__/prefetch-child.test.mjs new file mode 100644 index 0000000000..bd3d9864be --- /dev/null +++ b/scripts/__tests__/prefetch-child.test.mjs @@ -0,0 +1,93 @@ +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { startPrefetch, stopPrefetches } from '../prefetch-child.mjs'; + +// Small local Node children stand in for the image pull and the Grok download; +// none of these tests needs Docker or a network. +const withScratch = async body => { + const root = mkdtempSync(join(tmpdir(), 'prefetch-child-')); + try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const start = (root, name, script, cmd = process.execPath, args = ['-e', script]) => + startPrefetch({ name, cmd, args, cwd: root, env: process.env, logPath: join(root, `${name}.log`) }); +const blockFor = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms); +const untilLogHas = async (path, text) => { + for (let attempt = 0; attempt < 400; attempt++) { + try { if (readFileSync(path, 'utf8').includes(text)) return; } catch {} + await new Promise(resolve => setTimeout(resolve, 25)); + } + throw new Error(`log never contained ${text}`); +}; + +test('a successful prefetch resolves, keeps its output, and is left alone by stop', async () => withScratch(async root => { + const handle = start(root, 'ok', "setTimeout(() => process.stdout.write('fetched\\n'), 150)"); + await handle.done; + assert.equal(handle.child.exitCode, 0); + assert.equal(readFileSync(join(root, 'ok.log'), 'utf8'), 'fetched\n'); + assert.ok(handle.durationMs >= 100 && handle.durationMs < 5000, `durationMs ${handle.durationMs}`); + await stopPrefetches([handle]); + assert.equal(handle.child.signalCode, null); +})); + +test('the duration ends at the download, not when a blocked parent notices the exit', async () => withScratch(async root => { + const handle = start(root, 'early', "setTimeout(() => process.stdout.write('Status: Downloaded\\n'), 100)"); + // Synchronous work in the parent, as package staging is in the verifier. + blockFor(1500); + const noticed = Date.now(); + await handle.done; + assert.ok(Date.now() - noticed < 500, 'the exit was pending by the time the parent yielded'); + assert.ok(handle.durationMs < 900, `durationMs ${handle.durationMs} must not include the 1500ms the parent was blocked`); +})); + +test('a nonzero exit rejects with the log tail where the result is awaited', async () => withScratch(async root => { + const handle = start(root, 'bad', "process.stderr.write('registry unreachable\\n'); process.exit(3)"); + await assert.rejects(handle.done, /^Error: bad prefetch failed \(exit 3\): registry unreachable$/); + await stopPrefetches([handle]); +})); + +test('a command that cannot start rejects instead of hanging the join or the stop', async () => withScratch(async root => { + const handle = start(root, 'missing', '', join(root, 'no-such-command'), []); + await assert.rejects(handle.done, /missing prefetch failed \(.*ENOENT/); + await stopPrefetches([handle]); +})); + +test('stopping a running prefetch waits for it to exit so its directory can be removed safely', async () => withScratch(async root => { + const handle = start(root, 'slow', "process.stdout.write('started\\n'); setInterval(() => {}, 1000)"); + await untilLogHas(join(root, 'slow.log'), 'started'); + const stopping = Date.now(); + await stopPrefetches([handle]); + assert.ok(Date.now() - stopping < 4000); + assert.equal(handle.child.signalCode, 'SIGTERM'); + assert.equal(handle.child.exitCode, null); + await assert.rejects(handle.done, /slow prefetch failed \(SIGTERM\): started/); +})); + +test('a prefetch that ignores SIGTERM is killed after the grace period', async () => withScratch(async root => { + const handle = start(root, 'stubborn', "process.on('SIGTERM', () => {}); process.stdout.write('ignoring\\n'); setInterval(() => {}, 1000)"); + await untilLogHas(join(root, 'stubborn.log'), 'ignoring'); + const stopping = Date.now(); + await stopPrefetches([handle], { graceMs: 200 }); + assert.ok(Date.now() - stopping >= 200 && Date.now() - stopping < 4000); + assert.equal(handle.child.signalCode, 'SIGKILL'); +})); + +test('an unawaited failure never becomes an unhandled rejection', () => { + const script = ` + import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)}; + import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; + const root = mkdtempSync(join(tmpdir(), 'prefetch-unawaited-')); + try { + startPrefetch({ name: 'ignored', cmd: process.execPath, args: ['-e', 'process.exit(2)'], cwd: root, env: process.env, logPath: join(root, 'ignored.log') }); + await new Promise(resolve => setTimeout(resolve, 500)); + console.log('still running'); + } finally { rmSync(root, { recursive: true, force: true }); } + `; + const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { encoding: 'utf8', timeout: 10_000 }); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /still running/); + assert.doesNotMatch(result.stderr, /unhandled/i); +}); diff --git a/scripts/prefetch-child.mjs b/scripts/prefetch-child.mjs new file mode 100644 index 0000000000..bdaf3d25a8 --- /dev/null +++ b/scripts/prefetch-child.mjs @@ -0,0 +1,60 @@ +// Background downloads for verification scripts: start a child early with its +// output in a log file, join it where the result is needed, and stop it before +// the scratch directory it writes into is removed. No shell, no inherited +// terminal, and a failure is reported where the caller awaits it. +import { spawn } from 'node:child_process'; +import { closeSync, openSync, readFileSync, statSync } from 'node:fs'; + +const running = child => child.pid !== undefined && child.exitCode === null && child.signalCode === null; + +// Returns { name, child, done, durationMs }. `done` resolves on exit code 0 +// and otherwise rejects with the tail of the log; the rejection is observed +// here so an unawaited failure never surfaces as an unhandled rejection. +export function startPrefetch({ name, cmd, args, cwd, env, logPath }) { + const startedAt = Date.now(); + const log = openSync(logPath, 'w'); + let child; + try { + child = spawn(cmd, args, { cwd, env, stdio: ['ignore', log, log] }); + } finally { + closeSync(log); // the child holds its own descriptor + } + const handle = { name, child, done: null, durationMs: null }; + let settled = false; + handle.done = new Promise((settle, reject) => { + const finish = failure => { + if (settled) return; + settled = true; + // The caller may be blocked in synchronous work when the child exits, so + // this event can arrive long after the fact. When the child wrote + // anything, its last write is the end time; both downloads this serves + // print their final status line as they complete. A silent child falls + // back to the time the exit was observed. + let finishedAt = Date.now(); + try { + const { mtimeMs, size } = statSync(logPath); + if (size > 0) finishedAt = Math.min(finishedAt, mtimeMs); + } catch {} + handle.durationMs = Math.max(0, Math.round(finishedAt - startedAt)); + if (failure === null) return settle(); + let output = ''; + try { output = readFileSync(logPath, 'utf8').trim().slice(-2000); } catch {} + reject(new Error(`${name} prefetch failed (${failure}): ${output}`)); + }; + child.once('error', error => finish(error.message)); + child.once('exit', (code, signal) => finish(code === 0 ? null : (signal ?? `exit ${code}`))); + }); + handle.done.catch(() => {}); + return handle; +} + +// Stops every prefetch that is still running and resolves only once all of +// them have exited, so the caller can remove their working directory without +// racing a write. A child that ignores SIGTERM is killed after `graceMs`. +export async function stopPrefetches(prefetches, { graceMs = 5000 } = {}) { + await Promise.all(prefetches.map(async ({ child, done }) => { + if (running(child)) child.kill('SIGTERM'); + const escalate = setTimeout(() => { if (running(child)) child.kill('SIGKILL'); }, graceMs); + try { await done; } catch {} finally { clearTimeout(escalate); } + })); +} diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index b3c2db21d2..d35c40e818 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -4,12 +4,13 @@ // explicit and separate from npm installation, and is removed in finally. import assert from 'node:assert/strict'; import { execFileSync } from 'node:child_process'; -import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { materializePublishManifest, prepareBundledPackage } from './prepare-bundled-package.mjs'; import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from './grok-public-install-sandbox.mjs'; +import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'; import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs'; const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); assert.equal(process.platform, 'linux', 'Run this verification on disposable EC2 Linux, not a developer host'); @@ -19,7 +20,31 @@ const env = { ...process.env, NODE_PATH: '', PAPERCLIP_RELEASE_REUSE_UI_DIST: '1 const run = (cmd, args, cwd = root) => execFileSync(cmd, args, { cwd, env, stdio: 'pipe', maxBuffer: 32 * 1024 * 1024 }); const sourceRevision = run('git', ['rev-parse', 'HEAD'], repo).toString().trim(); const releaseVersion = `0.0.0-grok-verify.${sourceRevision.slice(0, 12)}`; +// The two network downloads the probes need do not depend on anything packed +// below, so they overlap the CPU-bound staging instead of running after it. +// The consumer image is pinned by digest and the Grok binary is checked against +// its pinned sha256 by the provisioning script, so an early start changes no +// verified property. Neither touches the consumer, the assets, nor the +// `native/grok` path that the post-install assertion inspects. +const prefetched = join(root, 'prefetch'); mkdirSync(prefetched); +const stagedGrok = join(prefetched, 'grok'); +const prefetches = []; +const timing = {}; +// A prefetch failure surfaces where its result is awaited, never as an +// unhandled rejection that would mask an earlier staging error. +const prefetch = (name, cmd, args) => { + const handle = startPrefetch({ name, cmd, args, cwd: root, env, logPath: join(prefetched, `${name}.log`) }); + prefetches.push(handle); + return handle.done; +}; +let phaseStarted = performance.now(); +const phase = name => { const now = performance.now(); timing[`${name}_ms`] = Math.round(now - phaseStarted); phaseStarted = now; }; +const awaited = async (name, promise) => { const started = performance.now(); await promise; timing[`${name}_wait_ms`] = Math.round(performance.now() - started); }; try { + const imageReady = prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE]); + // Provision as the unprivileged verification user, never into the host's /opt. + // Only the positive probe sees this file at the canonical sandbox path. + const grokReady = prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok]); const listing = run(process.execPath, [join(repo, 'scripts/release-package-map.mjs'), 'list'], repo).toString().trim().split('\n').map(line => line.split('\t')); const packages = new Map(listing.map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(join(repo, dir, 'package.json'), 'utf8')) }])); const needed = new Set(); @@ -81,9 +106,12 @@ try { const sentinel = join(consumer, 'node_modules/paperclip-verification-lifecycle-sentinel/lifecycle-ran'); const consumerUid = process.getuid(); const isolated = (command, options = {}) => run('docker', grokConsumerDockerArgs({ assets, consumer, cache, uid: consumerUid, gid: process.getgid(), command, ...options })); + phase('stage_packages'); + await awaited('image', imageReady); // npm resolution is intentionally the public consumer graph, not the pnpm // workspace graph. Freeze that result before any lifecycle script can run. isolated(['npm', 'install', '--ignore-scripts', '--omit=dev', ...readdirSync(assets).filter(file => file.endsWith('.tgz')).map(file => `/packages/${file}`)], { download: true }); + phase('consumer_install'); assert.equal(existsSync(sentinel), false, 'Dependency download must not run lifecycle scripts'); const consumerLock = readFileSync(join(consumer, 'package-lock.json'), 'utf8'); // npm ci rejects bundled optional platform dependencies absent from its own @@ -92,6 +120,7 @@ try { isolated(GROK_PUBLIC_INSTALL_LIFECYCLE); assert.equal(readFileSync(sentinel, 'utf8'), 'ok', 'Offline lifecycle scripts must actually execute'); assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Lifecycle execution must preserve the resolved consumer lock'); + phase('lifecycle_rebuild'); for (const name of needed) { const installedManifest = JSON.parse(readFileSync(join(consumer, 'node_modules', name, 'package.json'), 'utf8')); assert.equal(installedManifest.version, releaseVersion, `Installed release version for ${name}`); @@ -104,6 +133,7 @@ try { writeFileSync(join(assets, 'codex-probe.mjs'), installedCodexProbeSource( '/consumer/node_modules/@paperclipai/server/dist/vendor/paperclip-runner/index.js', '/consumer', codexVersion), { mode: 0o644 }); const codex = JSON.parse(isolated(['node', '/packages/codex-probe.mjs']).toString().trim()); + phase('codex_probe'); assert.equal(codex.codexConsumerHostOnly, true); assert.equal(codex.codexPlatformPackageSource, 'official npm'); assert.ok(existsSync(join(installed, 'providers/grok/launcher.cjs'))); @@ -122,10 +152,14 @@ try { `; writeFileSync(join(assets, 'probe.mjs'), probe, { mode: 0o644 }); isolated(['node', '/packages/probe.mjs', 'missing']); - // Provision as the unprivileged verification user, never into the host's /opt. - // Only the positive probe sees this file at the canonical sandbox path. - run(process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), prerequisite]); + // The binary was provisioned outside `native/grok` while the packages were + // staged, so the assertion above still proves npm put nothing there. It moves + // into place only now, so only the positive probe sees it. + await awaited('grok', grokReady); + mkdirSync(dirname(prerequisite), { recursive: true }); + renameSync(stagedGrok, prerequisite); isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite }); + phase('grok_probes'); console.log(JSON.stringify({ schema: 'paperclip.grok.public-npm-install.v1', sourceRevision, releaseVersion, lifecycleScriptsEnabled: true, lifecycleSentinelVerified: true, lifecycleNetwork: 'none', consumerImage: GROK_PUBLIC_INSTALL_IMAGE, consumerUid, consumerLockPreserved: true, cleanNpmInstall: true, packageCount: needed.size, codexPackedPayloadsOmitted: true, codexPackedPackagesChecked: tarballs.length, ...codex, builtinLauncherPresent: true, separateGrokPackage: false, npmProvisionedBinary: false, missingPrerequisiteRejected: true, provisionedBinaryVerified: true, commandLeaseVerified: true, providerCalls: 0 })); // Exercise Pi's public CLI and installed server, never a private workspace // package or binary override. Public dependency downloads are explicit and @@ -137,6 +171,7 @@ try { // temporary capacity than the smaller offline lifecycle probes. const setup = isolated(['node', '/consumer/node_modules/paperclipai/dist/index.js', 'runtime', 'setup', 'pi'], { download: true, temporarySizeMiB: 2048 }).toString(); const receipt = JSON.parse(setup.trim()); + phase('pi_setup'); assert.equal(receipt.status, 'installed_verified'); assert.equal(receipt.target, 'linux-x64'); assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Pi setup must preserve the consumer dependency graph'); @@ -144,6 +179,13 @@ try { // Docker defaults tmpfs to noexec. The offline admission probe must execute // its verified private snapshot while keeping lifecycle/download scratch noexec. console.log(isolated(['node', '/packages/pi-public-install-probe.mjs', '/consumer/node_modules/@paperclipai/server'], { temporarySizeMiB: 2048, temporaryExecutable: true }).toString().trim()); + phase('pi_probe'); + // Wall time per phase, so the slow part of this step is visible in the CI log + // without re-running it. `*_prefetch_ms` runs from a download's start to its + // last output, and `*_wait_ms` is how long the main flow waited for a + // prefetch that had not finished yet; zero means the overlap hid it entirely. + const downloads = Object.fromEntries(prefetches.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs])); + console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloads })); if (process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR) { const retained = retainRunnerQualificationPackages({ repo, output: process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR, sourceRevision, releaseVersion, env, publicArchives: [...needed].map((name, index) => ({ name, file: tarballs[index] })), @@ -151,5 +193,8 @@ try { console.log(JSON.stringify(retained)); } } finally { + // An earlier failure must not leave a prefetch running, and scratch is + // removed only once both children have stopped writing into it. + await stopPrefetches(prefetches); rmSync(root, { recursive: true, force: true }); }