diff --git a/package.json b/package.json index 5a5000234b..29b999b28e 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", + "test:release-registry": "node --test scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/__tests__/grok-public-install-sandbox.test.mjs b/scripts/__tests__/grok-public-install-sandbox.test.mjs index 5a3dc487e3..11fde52433 100644 --- a/scripts/__tests__/grok-public-install-sandbox.test.mjs +++ b/scripts/__tests__/grok-public-install-sandbox.test.mjs @@ -110,6 +110,31 @@ test('the image and Grok downloads start before staging and join without weakeni assert.equal(source.match(/rmSync\(root/g).length, 1); }); +test('the bundled-dependency installs run together while the other packages are copied, and pack in order', () => { + const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8'); + const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; }; + // The installs are children of the script's own CLI entry through the helper + // covered by stage-bundled-packages.test.mjs, never in-process calls. + assert.doesNotMatch(source, /prepareBundledPackage/); + assert.ok(source.includes("import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs'")); + // Only the bundled packages' sources are staged before the installs start; + // the installs are registered for cleanup at once, before anything else can + // fail; the other packages are copied while the installs run; the join comes + // before the first pack; and the pack loop walks every package in order. + const bundledSources = position('for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry)'); + const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo })"); + const registered = position('prefetches.push(...installs.map(({ handle }) => handle))'); + const otherSources = position('for (const entry of entries.filter(entry => !entry.bundled))'); + const join = position("await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)))"); + const packLoop = position('for (const { target } of entries)'); + const firstPack = position("run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target)"); + assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packLoop && packLoop < firstPack); + assert.equal(source.match(/run\('npm', \['pack'/g).length, 2, 'one package pack loop plus the lifecycle sentinel'); + // The timing line reports the split staging phase and each install. + for (const key of ['stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms']) assert.ok(source.includes(`timing.${key} =`)); + assert.ok(source.includes('...bundledInstallTimings(installs)')); +}); + test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => { const root = realpathSync(mkdtempSync(join(tmpdir(), 'installed-codex-probe-'))); try { diff --git a/scripts/__tests__/prefetch-child.test.mjs b/scripts/__tests__/prefetch-child.test.mjs index bd3d9864be..c508ef0290 100644 --- a/scripts/__tests__/prefetch-child.test.mjs +++ b/scripts/__tests__/prefetch-child.test.mjs @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { spawnSync } from 'node:child_process'; +import { spawn, spawnSync } from 'node:child_process'; import { mkdtempSync, readFileSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -25,6 +25,8 @@ const untilLogHas = async (path, text) => { test('a successful prefetch resolves, keeps its output, and is left alone by stop', async () => withScratch(async root => { const handle = start(root, 'ok', "setTimeout(() => process.stdout.write('fetched\\n'), 150)"); + // The parent keeps its default, immediate termination: no handler of ours. + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) assert.equal(process.listenerCount(signal), 0, `${signal} handler installed in the parent`); await handle.done; assert.equal(handle.child.exitCode, 0); assert.equal(readFileSync(join(root, 'ok.log'), 'utf8'), 'fetched\n'); @@ -66,7 +68,7 @@ test('stopping a running prefetch waits for it to exit so its directory can be r await assert.rejects(handle.done, /slow prefetch failed \(SIGTERM\): started/); })); -test('a prefetch that ignores SIGTERM is killed after the grace period', async () => withScratch(async root => { +test('a prefetch that ignores SIGTERM is killed after the grace period', { skip: process.platform === 'win32' && 'SIGTERM cannot be ignored on Windows' }, async () => withScratch(async root => { const handle = start(root, 'stubborn', "process.on('SIGTERM', () => {}); process.stdout.write('ignoring\\n'); setInterval(() => {}, 1000)"); await untilLogHas(join(root, 'stubborn.log'), 'ignoring'); const stopping = Date.now(); @@ -75,6 +77,97 @@ test('a prefetch that ignores SIGTERM is killed after the grace period', async ( assert.equal(handle.child.signalCode, 'SIGKILL'); })); +// A process is gone once the kernel no longer knows it, or once it is a zombie +// that only waits to be reaped: either way it can no longer write anywhere. +const gone = pid => { + try { process.kill(pid, 0); } catch (error) { return error.code === 'ESRCH'; } + try { return /^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return false; } +}; +const untilGone = async pid => { + for (let attempt = 0; attempt < 200; attempt++) { + if (gone(pid)) return true; + await new Promise(resolve => setTimeout(resolve, 25)); + } + return false; +}; +// The child starts a process of its own that announces its pid and then runs +// on, the way the bundled-dependency installs block in `npm install`. +const GRANDCHILD_SCRIPT = ` + const { spawn } = require('node:child_process'); + const grandchild = spawn(process.execPath, ['-e', "process.stdout.write('grandchild ' + process.pid + '\\\\n'); setInterval(() => {}, 1000)"], { stdio: 'inherit' }); + grandchild.on('exit', () => process.exit(0)); +`; +const grandchildPid = async log => { await untilLogHas(log, 'grandchild '); return Number(readFileSync(log, 'utf8').match(/grandchild (\d+)/)[1]); }; + +test('stopping a prefetch also stops the processes it started', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + const handle = start(root, 'tree', GRANDCHILD_SCRIPT); + const pid = await grandchildPid(join(root, 'tree.log')); + assert.ok(!gone(pid), 'the grandchild runs before the stop'); + await stopPrefetches([handle]); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived stopPrefetches`); + assert.notEqual(handle.child.signalCode ?? handle.child.exitCode, null); +})); + +test('a stop request written to the pipe ends the command and the processes it started, with no signal to the supervisor', { skip: process.platform === 'win32' && 'the grandchild check reads /proc' }, async () => withScratch(async root => { + // This is the stop path on a platform without process groups: there `kill` + // would end the supervisor outright and leave its command running, so + // `stopPrefetches` writes a request instead. The supervisor answers it + // the way it answers the parent's exit, and its own exit still reports the + // command's fate. + const handle = start(root, 'tree', GRANDCHILD_SCRIPT); + const pid = await grandchildPid(join(root, 'tree.log')); + handle.child.stdin.end('stop\n'); + await assert.rejects(handle.done, /tree prefetch failed \(SIGTERM\)/); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived the stop request`); + assert.match(readFileSync(join(root, 'tree.log'), 'utf8'), /stop requested; stopping/); + assert.doesNotMatch(readFileSync(join(root, 'tree.log'), 'utf8'), /parent exited/); + await stopPrefetches([handle]); +})); + +// A separate parent process stands in for the verifier: it starts a prefetch +// whose child starts a grandchild, then does `work` (a module-scope statement). +const parentScript = (root, work) => ` + import { execFileSync } from 'node:child_process'; + import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)}; + startPrefetch({ name: 'tree', cmd: process.execPath, args: ['-e', ${JSON.stringify(GRANDCHILD_SCRIPT)}], cwd: ${JSON.stringify(root)}, env: process.env, logPath: ${JSON.stringify(join(root, 'tree.log'))} }); + ${work} +`; + +for (const signal of ['SIGINT', 'SIGKILL']) { + test(`${signal} to the parent ends the processes a prefetch started, and the parent dies of it at once`, { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + // The parent runs no handler, so it dies of the signal immediately, and the + // prefetch tree stops because the parent is gone, not because it was told: + // SIGKILL leaves the parent no chance to tell anyone anything. + const parent = spawn(process.execPath, ['--input-type=module', '-e', parentScript(root, 'setInterval(() => {}, 1000);')], { cwd: root, stdio: 'ignore' }); + const exited = new Promise(resolve => parent.once('exit', (code, signal) => resolve({ code, signal }))); + const pid = await grandchildPid(join(root, 'tree.log')); + parent.kill(signal); + assert.deepEqual(await exited, { code: null, signal }); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived the parent's ${signal}`); + assert.match(readFileSync(join(root, 'tree.log'), 'utf8'), /parent exited; stopping/); + })); +} + +test('Ctrl-C while the parent is blocked in synchronous work stops it at once, and the prefetch tree with it', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + // The verifier runs `execFileSync` for the standalone builds while prefetches + // run, and a JavaScript signal handler cannot run until that returns. The + // parent gets its own group here so the signal can be sent the way a + // terminal sends Ctrl-C: to the foreground group, which is the parent and + // its synchronous command, and not the detached prefetch tree. + const parent = spawn(process.execPath, ['--input-type=module', '-e', parentScript(root, "execFileSync(process.execPath, ['-e', 'setTimeout(() => {}, 20000)'], { stdio: 'ignore' });")], { cwd: root, stdio: 'ignore', detached: true }); + try { + const exited = new Promise(resolve => parent.once('exit', (code, signal) => resolve({ code, signal }))); + const pid = await grandchildPid(join(root, 'tree.log')); + const signalled = Date.now(); + process.kill(-parent.pid, 'SIGINT'); + assert.deepEqual(await exited, { code: null, signal: 'SIGINT' }); + assert.ok(Date.now() - signalled < 2000, `the parent took ${Date.now() - signalled}ms to die: it waited for the synchronous command`); + assert.ok(await untilGone(pid), `grandchild ${pid} outlived the parent's Ctrl-C`); + } finally { + try { process.kill(-parent.pid, 'SIGKILL'); } catch {} + } +})); + test('an unawaited failure never becomes an unhandled rejection', () => { const script = ` import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)}; diff --git a/scripts/__tests__/stage-bundled-packages.test.mjs b/scripts/__tests__/stage-bundled-packages.test.mjs new file mode 100644 index 0000000000..d42ddf14cf --- /dev/null +++ b/scripts/__tests__/stage-bundled-packages.test.mjs @@ -0,0 +1,104 @@ +import assert from 'node:assert/strict'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { stopPrefetches } from '../prefetch-child.mjs'; +import { PREPARE_BUNDLED_PACKAGE_SCRIPT, bundledInstallTimings, bundledPackageSlug, startBundledInstalls } from '../stage-bundled-packages.mjs'; + +// A small local script stands in for `prepare-bundled-package.mjs`; no test +// here needs npm or a network. +const withScratch = async body => { + const root = mkdtempSync(join(tmpdir(), 'stage-bundled-packages-')); + try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); } +}; +const entriesFor = (root, names) => names.map((name, index) => ({ name, stagedSource: join(root, `source-${index}`), target: join(root, `package-${index}`) })); +const fakeScript = (root, body) => { const script = join(root, 'fake-prepare.mjs'); writeFileSync(script, body); return script; }; + +test('package names map to stable log and timing slugs', () => { + assert.equal(bundledPackageSlug('@paperclipai/server'), 'server'); + assert.equal(bundledPackageSlug('@paperclipai/adapter-utils'), 'adapter_utils'); + assert.equal(bundledPackageSlug('@paperclipai/db'), 'db'); +}); + +test('the children run the CLI entry release.sh uses, never an in-process call', () => { + assert.ok(existsSync(PREPARE_BUNDLED_PACKAGE_SCRIPT)); + const source = readFileSync(PREPARE_BUNDLED_PACKAGE_SCRIPT, 'utf8'); + assert.ok(source.includes('if (process.argv[1] === fileURLToPath(import.meta.url))')); + assert.ok(source.includes('prepareBundledPackage(resolve(sourceDir), resolve(destinationDir))')); + const release = readFileSync(new URL('../release.sh', import.meta.url), 'utf8'); + assert.match(release, /node "\$REPO_ROOT\/scripts\/prepare-bundled-package\.mjs" "\$REPO_ROOT\/\$pkg_dir" "\$publish_dir"/); +}); + +test('the installs run at the same time, each with its own source and target', async () => withScratch(async root => { + // Each child records its arguments, announces itself, and exits only once + // the other child has announced itself too. Serial children would deadlock + // here and the five-second guard would turn that into a failure. + const script = fakeScript(root, ` + import { existsSync, writeFileSync } from 'node:fs'; + import { basename, join } from 'node:path'; + const [source, target] = process.argv.slice(2); + const root = join(target, '..'); + writeFileSync(join(root, 'started-' + basename(target)), ''); + console.log('install ' + source + ' -> ' + target); + const other = basename(target) === 'package-0' ? 'package-1' : 'package-0'; + const deadline = Date.now() + 5000; + while (!existsSync(join(root, 'started-' + other))) { + if (Date.now() > deadline) { console.error('the other install never started'); process.exit(3); } + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20); + } + writeFileSync(join(target, 'package.json'), '{}'); + `); + const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db']); + for (const { target } of entries) mkdirSync(target); + const installs = startBundledInstalls(entries, { logDir: root, cwd: root, script }); + assert.deepEqual(installs.map(({ slug }) => slug), ['server', 'db']); + await Promise.all(installs.map(({ handle }) => handle.done)); + for (const [index, { stagedSource, target, slug, handle }] of installs.entries()) { + assert.equal(handle.child.exitCode, 0); + assert.equal(readFileSync(join(root, `bundled-${slug}.log`), 'utf8'), `install ${stagedSource} -> ${target}\n`); + assert.equal(target, join(root, `package-${index}`)); + assert.ok(existsSync(join(target, 'package.json'))); + } + const timings = bundledInstallTimings(installs); + assert.deepEqual(Object.keys(timings), ['stage_bundled_server_ms', 'stage_bundled_db_ms']); + for (const value of Object.values(timings)) assert.ok(Number.isInteger(value) && value >= 0); +})); + +test('stopping the installs after one fails also stops the npm process the other started', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => { + // The real wrapper blocks in `execFileSync('npm', ['install', ...])`; here + // the stand-in npm announces its pid and runs on, and the wrapper blocks on + // it the same way. Stopping only the wrapper would leave it writing. + const script = fakeScript(root, ` + import { execFileSync } from 'node:child_process'; + const [, target] = process.argv.slice(2); + if (target.endsWith('package-1')) { console.error('bundled install exploded'); process.exit(2); } + execFileSync(process.execPath, ['-e', "process.stdout.write('npm ' + process.pid + '\\\\n'); setInterval(() => {}, 1000)"], { stdio: 'inherit' }); + `); + const installs = startBundledInstalls(entriesFor(root, ['@paperclipai/server', '@paperclipai/db']), { logDir: root, cwd: root, script }); + await assert.rejects(Promise.all(installs.map(({ handle }) => handle.done)), /bundled-db prefetch failed \(exit 2\)/); + const log = join(root, 'bundled-server.log'); + const logHasPid = () => existsSync(log) && /npm \d+/.test(readFileSync(log, 'utf8')); + for (let attempt = 0; attempt < 200 && !logHasPid(); attempt++) await new Promise(resolve => setTimeout(resolve, 25)); + const npmPid = Number(readFileSync(log, 'utf8').match(/npm (\d+)/)[1]); + const alive = pid => { try { process.kill(pid, 0); return !/^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch (error) { return error.code !== 'ESRCH'; } }; + assert.ok(alive(npmPid), 'the stand-in npm runs before the stop'); + await stopPrefetches(installs.map(({ handle }) => handle)); + let stopped = false; + for (let attempt = 0; attempt < 200 && !(stopped = !alive(npmPid)); attempt++) await new Promise(resolve => setTimeout(resolve, 25)); + assert.ok(stopped, `stand-in npm ${npmPid} outlived stopPrefetches`); + assert.notEqual(installs[0].handle.child.signalCode ?? installs[0].handle.child.exitCode, null); +})); + +test('a failed install rejects with the tail of its own log and leaves the others to be stopped', async () => withScratch(async root => { + const script = fakeScript(root, ` + const [, target] = process.argv.slice(2); + if (target.endsWith('package-1')) { console.error('bundled install exploded'); process.exit(2); } + setTimeout(() => {}, 30_000); + `); + const installs = startBundledInstalls(entriesFor(root, ['@paperclipai/server', '@paperclipai/adapter-utils']), { logDir: root, cwd: root, script }); + await assert.rejects(Promise.all(installs.map(({ handle }) => handle.done)), /bundled-adapter_utils prefetch failed \(exit 2\): bundled install exploded/); + assert.equal(installs[0].handle.child.exitCode, null, 'the other install is still running when the failure surfaces'); + await stopPrefetches(installs.map(({ handle }) => handle)); + assert.notEqual(installs[0].handle.child.signalCode ?? installs[0].handle.child.exitCode, null); +})); diff --git a/scripts/prefetch-child.mjs b/scripts/prefetch-child.mjs index bdaf3d25a8..926bd3c257 100644 --- a/scripts/prefetch-child.mjs +++ b/scripts/prefetch-child.mjs @@ -2,23 +2,72 @@ // output in a log file, join it where the result is needed, and stop it before // the scratch directory it writes into is removed. No shell, no inherited // terminal, and a failure is reported where the caller awaits it. +// +// A child may start processes of its own: the bundled-dependency installs are +// a Node wrapper blocked in `npm install`. Signalling only the wrapper would +// leave npm writing into the scratch root while it is removed, so on POSIX +// each command runs under `prefetch-supervisor.mjs` as the leader of its own +// process group, and stopping signals and waits for the whole group. A group +// of its own no longer receives the terminal's Ctrl-C, and the parent cannot +// forward it: the verifier spends most of its time blocked in `execFileSync`, +// where no JavaScript signal handler can run, and installing one would also +// replace the parent's immediate default termination. Instead the supervisor +// holds a pipe from the parent and stops its group when the pipe closes, which +// the kernel does when the parent exits for any reason. The parent installs +// no signal handlers. +// +// Windows has no process groups and `kill` there ends a process outright, so a +// signalled supervisor could pass nothing on and its command would run on +// after cleanup returned. There a deliberate stop goes through the same pipe +// as a written request, which the supervisor answers by ending its command, +// and only the SIGKILL escalation targets the supervisor itself. import { spawn } from 'node:child_process'; import { closeSync, openSync, readFileSync, statSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +export const PREFETCH_SUPERVISOR_SCRIPT = fileURLToPath(new URL('./prefetch-supervisor.mjs', import.meta.url)); + +const GROUPS = process.platform !== 'win32'; const running = child => child.pid !== undefined && child.exitCode === null && child.signalCode === null; +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); -// Returns { name, child, done, durationMs }. `done` resolves on exit code 0 -// and otherwise rejects with the tail of the log; the rejection is observed -// here so an unawaited failure never surfaces as an unhandled rejection. +// Signals the child's whole process group (POSIX). Without groups the +// supervisor is asked to stop its command through the pipe it watches, since +// `kill` would end the supervisor at once and leave the command running; only +// SIGKILL is sent to the supervisor itself, after it has had its chance. +const signalTree = (child, signal) => { + if (child.pid === undefined) return; + if (!GROUPS) { + if (!running(child)) return; + if (signal === 'SIGKILL') child.kill(signal); + else if (child.stdin?.writable) child.stdin.end('stop\n'); + return; + } + try { process.kill(-child.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; } +}; +// True while any member of the child's group exists. On a platform without +// groups only the child itself is known. +const treeAlive = child => { + if (child.pid === undefined) return false; + if (!GROUPS) return running(child); + try { process.kill(-child.pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; } +}; + +// Returns { name, child, done, durationMs }. `child` is the supervisor, whose +// exit code or signal is the command's. `done` resolves on exit code 0 and +// otherwise rejects with the tail of the log; the rejection is observed here +// so an unawaited failure never surfaces as an unhandled rejection. export function startPrefetch({ name, cmd, args, cwd, env, logPath }) { const startedAt = Date.now(); const log = openSync(logPath, 'w'); let child; try { - child = spawn(cmd, args, { cwd, env, stdio: ['ignore', log, log] }); + // stdin is the pipe the supervisor watches; nothing is ever written to it. + child = spawn(process.execPath, [PREFETCH_SUPERVISOR_SCRIPT, cmd, ...args], { cwd, env, stdio: ['pipe', log, log], detached: GROUPS }); } finally { closeSync(log); // the child holds its own descriptor } + child.stdin?.on('error', () => {}); const handle = { name, child, done: null, durationMs: null }; let settled = false; handle.done = new Promise((settle, reject) => { @@ -48,13 +97,22 @@ export function startPrefetch({ name, cmd, args, cwd, env, logPath }) { return handle; } -// Stops every prefetch that is still running and resolves only once all of -// them have exited, so the caller can remove their working directory without -// racing a write. A child that ignores SIGTERM is killed after `graceMs`. +// Stops every prefetch that is still running, together with any process it +// started, and resolves only once all of them have exited, so the caller can +// remove their working directory without racing a write. A tree that ignores +// SIGTERM is killed after `graceMs`; one that still exists `graceMs` after +// that is given up on so cleanup cannot hang. export async function stopPrefetches(prefetches, { graceMs = 5000 } = {}) { await Promise.all(prefetches.map(async ({ child, done }) => { - if (running(child)) child.kill('SIGTERM'); - const escalate = setTimeout(() => { if (running(child)) child.kill('SIGKILL'); }, graceMs); - try { await done; } catch {} finally { clearTimeout(escalate); } + const started = Date.now(); + let termed = false, killed = false; + while (treeAlive(child)) { + const elapsed = Date.now() - started; + if (!termed) { signalTree(child, 'SIGTERM'); termed = true; } + else if (!killed && elapsed >= graceMs) { signalTree(child, 'SIGKILL'); killed = true; } + else if (killed && elapsed >= 2 * graceMs) break; + await sleep(25); + } + try { await done; } catch {} })); } diff --git a/scripts/prefetch-supervisor.mjs b/scripts/prefetch-supervisor.mjs new file mode 100644 index 0000000000..0cd29765c6 --- /dev/null +++ b/scripts/prefetch-supervisor.mjs @@ -0,0 +1,78 @@ +// Child side of prefetch-child.mjs: runs one prefetch command as the leader of +// its own process group and stops the whole group when the parent that started +// it is gone. +// +// The parent holds the write end of this process's stdin. The kernel closes +// that end when the parent exits for any reason: a Ctrl-C delivered while the +// parent is blocked in synchronous work, a plain `kill`, or SIGKILL. Reading +// EOF here is therefore the only parent-death signal needed, and the parent +// runs no signal handler of its own, so its default termination stays +// immediate. The parent writes to the pipe for one reason only: to ask for a +// deliberate stop where it cannot signal the group, which is Windows, where +// `kill` ends a process outright and a signalled supervisor could not pass +// anything on. Group-wide signals the parent sends through `stopPrefetches` +// on POSIX reach this process too; it passes them on to the command and stays +// alive until the command exits, so its own exit reports the command's +// outcome (exit code, or the same signal). +import { spawn } from 'node:child_process'; +import { fstatSync } from 'node:fs'; +import { constants } from 'node:os'; + +const GROUPS = process.platform !== 'win32'; +const GRACE_MS = Number(process.env.PAPERCLIP_PREFETCH_GRACE_MS) || 5000; +const [cmd, ...args] = process.argv.slice(2); +if (!cmd) { + process.stderr.write('prefetch supervisor: usage: prefetch-supervisor.mjs [args...]\n'); + process.exit(64); +} + +const child = spawn(cmd, args, { stdio: ['ignore', 'inherit', 'inherit'] }); +const signalChild = signal => { try { child.kill(signal); } catch {} }; +// On POSIX this process leads the group (the parent starts it detached), so +// the group id is its own pid. Without groups only the command is known. +const signalGroup = signal => { + if (!GROUPS) return signalChild(signal); + try { process.kill(-process.pid, signal); } catch (error) { if (error.code === 'ESRCH') signalChild(signal); } +}; + +// A signal to the group arrives here as well as at the command; passing it on +// is harmless when the command already has it, and covers a signal sent to +// this process alone. The parent owns the SIGKILL escalation in that case. +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => signalChild(signal)); + +let stopping = false; +const stopGroup = reason => { + if (stopping) return; + stopping = true; + process.stderr.write(`prefetch supervisor: ${reason}; stopping ${cmd}\n`); + signalGroup('SIGTERM'); + setTimeout(() => signalGroup('SIGKILL'), GRACE_MS).unref(); +}; +// Only a pipe can carry a stop request or report the parent's exit (Node +// creates its stdio pipes as socket pairs). Run by hand with a terminal or +// /dev/null on stdin, there is no parent to watch and nothing is read. +let stdinIsPipe = false; +try { const stdin = fstatSync(0); stdinIsPipe = stdin.isFIFO() || stdin.isSocket(); } catch {} +if (stdinIsPipe) { + process.stdin.on('data', () => stopGroup('stop requested')); + const parentGone = () => stopGroup('parent exited'); + process.stdin.on('end', parentGone); + process.stdin.on('close', parentGone); + process.stdin.on('error', parentGone); + process.stdin.resume(); +} + +child.on('error', error => { + process.stderr.write(`prefetch supervisor: ${error.message}\n`); + process.exit(127); +}); +child.on('exit', (code, signal) => { + if (signal !== null) { + // Die of the same signal, so the parent sees the command's fate on this + // process. Dropping the listener restores the default disposition first. + process.removeAllListeners(signal); + try { process.kill(process.pid, signal); } catch {} + process.exit(128 + (constants.signals[signal] ?? 1)); + } + process.exit(code ?? 1); +}); diff --git a/scripts/stage-bundled-packages.mjs b/scripts/stage-bundled-packages.mjs new file mode 100644 index 0000000000..9e3abe6acb --- /dev/null +++ b/scripts/stage-bundled-packages.mjs @@ -0,0 +1,32 @@ +// The public-install verifier stages three packages that bundle dependencies +// (`@paperclipai/server`, `@paperclipai/adapter-utils`, `@paperclipai/db`). +// Each one is a host-side `npm install` of that package's bundled closure into +// its own pack directory, and none reads what another produces, so they run as +// concurrent children instead of one after another inside the staging loop. +// Each child is the CLI entry of `prepare-bundled-package.mjs`, the same path +// release.sh runs, with its output in a log file that a failure reports. +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { startPrefetch } from './prefetch-child.mjs'; + +export const PREPARE_BUNDLED_PACKAGE_SCRIPT = fileURLToPath(new URL('./prepare-bundled-package.mjs', import.meta.url)); + +// `@paperclipai/adapter-utils` -> `adapter_utils`, for log names and timing keys. +export const bundledPackageSlug = name => name.replace(/^@paperclipai\//, '').replace(/[^a-z0-9]+/gi, '_'); + +// Starts one child per entry (`{ name, stagedSource, target }`) at once and +// returns the entries with their `slug` and prefetch `handle`. The caller joins +// `handle.done` where the packed output is needed and registers the handles +// for cleanup, so an earlier failure stops every install still running. +export function startBundledInstalls(entries, { logDir, cwd, env = process.env, script = PREPARE_BUNDLED_PACKAGE_SCRIPT }) { + return entries.map(entry => { + const slug = bundledPackageSlug(entry.name); + const handle = startPrefetch({ name: `bundled-${slug}`, cmd: process.execPath, args: [script, entry.stagedSource, entry.target], cwd, env, logPath: join(logDir, `bundled-${slug}.log`) }); + return { ...entry, slug, handle }; + }); +} + +// Per-package wall time for the timing line. The helper measures from the +// start to the child's last output; the parent is idle while it waits for the +// installs, so the exit is observed as it happens. +export const bundledInstallTimings = installs => Object.fromEntries(installs.map(({ slug, handle }) => [`stage_bundled_${slug}_ms`, handle.durationMs])); diff --git a/scripts/verify-grok-npm-install.mjs b/scripts/verify-grok-npm-install.mjs index d35c40e818..a5844b220b 100644 --- a/scripts/verify-grok-npm-install.mjs +++ b/scripts/verify-grok-npm-install.mjs @@ -8,10 +8,11 @@ import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, re import { tmpdir } from 'node:os'; import { basename, dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; -import { materializePublishManifest, prepareBundledPackage } from './prepare-bundled-package.mjs'; +import { materializePublishManifest } from './prepare-bundled-package.mjs'; import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource } from './grok-public-install-sandbox.mjs'; import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'; import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs'; +import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs'; const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..'); assert.equal(process.platform, 'linux', 'Run this verification on disposable EC2 Linux, not a developer host'); const root = mkdtempSync(join(tmpdir(), 'paperclip-grok-public-install-')); @@ -28,13 +29,18 @@ const releaseVersion = `0.0.0-grok-verify.${sourceRevision.slice(0, 12)}`; // `native/grok` path that the post-install assertion inspects. const prefetched = join(root, 'prefetch'); mkdirSync(prefetched); const stagedGrok = join(prefetched, 'grok'); +// Every background child, downloads and bundled-dependency installs alike, is +// stopped in `finally`; `downloads` is the subset the timing line reports as +// prefetches. const prefetches = []; +const downloads = []; const timing = {}; // A prefetch failure surfaces where its result is awaited, never as an // unhandled rejection that would mask an earlier staging error. const prefetch = (name, cmd, args) => { const handle = startPrefetch({ name, cmd, args, cwd: root, env, logPath: join(prefetched, `${name}.log`) }); prefetches.push(handle); + downloads.push(handle); return handle.done; }; let phaseStarted = performance.now(); @@ -61,11 +67,21 @@ try { // Source manifests remain untouched, including independently versioned SDKs. run(process.execPath, [join(repo, 'scripts/build-standalone-public-packages.mjs')], repo); run('bash', [join(repo, 'scripts/prepare-server-ui-dist.sh')], repo); - const tarballs = []; - for (const [index, name] of [...needed].entries()) { + // Each package's published files are staged into `source-` and its + // pack input into `package-`, with in dependency order so the + // tarball list keeps matching `needed`. The packages that bundle dependencies + // each need a host-side `npm install` of their bundled closure; those installs + // write only their own `package-` and read nothing another package + // produces, so they start first and run together while the other packages + // are copied, instead of one after another inside one loop. Packing stays + // serial and in order, because each tarball is checked as it is produced. + const entries = [...needed].map((name, index) => { const { dir, manifest } = packages.get(name); - const target = join(root, `package-${index}`); mkdirSync(target); - const stagedSource = join(root, `source-${index}`); mkdirSync(stagedSource); + const bundled = (manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length > 0; + return { name, dir, manifest, bundled, target: join(root, `package-${index}`), stagedSource: join(root, `source-${index}`) }; + }); + const stageSource = ({ dir, manifest, stagedSource }) => { + mkdirSync(stagedSource); for (const file of manifest.files ?? ['dist']) { // release.sh stages runtime skills into these public packages before pack. const releaseSkills = file === 'skills' && ['server', 'packages/adapters/claude-local', 'packages/adapters/codex-local'].includes(dir); @@ -76,12 +92,25 @@ try { } const releaseManifest = { ...manifest, version: releaseVersion }; writeFileSync(join(stagedSource, 'package.json'), JSON.stringify(releaseManifest)); - if ((manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length) { - prepareBundledPackage(stagedSource, target); - } else { - cpSync(stagedSource, target, { recursive: true }); - writeFileSync(join(target, 'package.json'), JSON.stringify(materializePublishManifest(releaseManifest))); - } + return releaseManifest; + }; + const sourcesStarted = performance.now(); + for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry); + const installsStarted = performance.now(); + const installs = startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo }); + prefetches.push(...installs.map(({ handle }) => handle)); + for (const entry of entries.filter(entry => !entry.bundled)) { + const releaseManifest = stageSource(entry); + mkdirSync(entry.target); + cpSync(entry.stagedSource, entry.target, { recursive: true }); + writeFileSync(join(entry.target, 'package.json'), JSON.stringify(materializePublishManifest(releaseManifest))); + } + timing.stage_sources_ms = Math.round(performance.now() - sourcesStarted); + await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done))); + timing.stage_bundled_ms = Math.round(performance.now() - installsStarted); + const packStarted = performance.now(); + const tarballs = []; + for (const { target } of entries) { run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target); const packed = readdirSync(root).filter(f => f.endsWith('.tgz') && !tarballs.includes(join(root, f))); assert.equal(packed.length, 1); @@ -89,6 +118,7 @@ try { assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).toString().trim().split('\n')); tarballs.push(tarball); } + timing.stage_pack_ms = Math.round(performance.now() - packStarted); const assets = join(root, 'assets'); mkdirSync(assets, { mode: 0o755 }); const consumer = join(root, 'consumer'); mkdirSync(consumer); const cache = join(root, 'cache'); mkdirSync(cache); @@ -184,8 +214,10 @@ try { // without re-running it. `*_prefetch_ms` runs from a download's start to its // last output, and `*_wait_ms` is how long the main flow waited for a // prefetch that had not finished yet; zero means the overlap hid it entirely. - const downloads = Object.fromEntries(prefetches.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs])); - console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloads })); + // `stage_sources_ms`, `stage_bundled_ms` and `stage_pack_ms` split the + // staging phase, and `stage_bundled__ms` is each concurrent install. + const downloaded = Object.fromEntries(downloads.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs])); + console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloaded, ...bundledInstallTimings(installs) })); if (process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR) { const retained = retainRunnerQualificationPackages({ repo, output: process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR, sourceRevision, releaseVersion, env, publicArchives: [...needed].map((name, index) => ({ name, file: tarballs[index] })),