mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-30 05:16:50 +01:00
Merge master b189f586d into Codex runner default candidate
Preserve master transport, lifecycle, and release verification controls alongside macOS qualification. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
commit
f4abdc2cfc
749 files changed
+124181
-68418
No files matched your search
@@ -0,0 +1,147 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { chmodSync, existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import net from "node:net";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..");
|
||||
|
||||
// Exercise the real startup helper and child-process boundary without registry
|
||||
// traffic. The npm fixture installs a real executable into its isolated prefix.
|
||||
export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false,
|
||||
config = "public", metadata = "valid", version = "2026.1009.0-canary.1" }) {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "canary-startup-test-"));
|
||||
const calls = path.join(root, "calls.jsonl");
|
||||
const fixtureBin = path.join(root, "bin");
|
||||
mkdirSync(fixtureBin);
|
||||
const npmFixture = path.join(fixtureBin, "npm");
|
||||
const port = await new Promise((resolve, reject) => {
|
||||
const server = net.createServer();
|
||||
server.on("error", reject);
|
||||
server.listen(0, "127.0.0.1", () => { const port = server.address().port; server.close(() => resolve(port)); });
|
||||
});
|
||||
const cli = `#!/usr/bin/env node
|
||||
const { appendFileSync } = require("node:fs");
|
||||
appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"onboard", pid:process.pid, args:process.argv.slice(2)}) + "\\n");
|
||||
if (process.env.FIXTURE_ONBOARD === "fail") { console.error("npm error code ETARGET"); process.exit(17); }
|
||||
if (process.env.FIXTURE_ONBOARD === "hang") { setTimeout(() => {}, 10000); }
|
||||
if (process.env.FIXTURE_ONBOARD === "serve") {
|
||||
const { spawn } = require("node:child_process");
|
||||
const child = spawn(process.execPath, ["-e", "process.on('SIGTERM',()=>{}); setTimeout(()=>process.exit(0),10000)"], {stdio:"ignore"});
|
||||
appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"descendant", pid:child.pid}) + "\\n");
|
||||
process.on("SIGTERM", () => {});
|
||||
setTimeout(() => process.exit(0), 10000);
|
||||
require("node:http").createServer((req,res) => {res.end("ok");}).listen(Number(process.env.FIXTURE_PORT), "127.0.0.1");
|
||||
}
|
||||
`;
|
||||
writeFileSync(npmFixture, `#!/usr/bin/env node
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const args = process.argv.slice(2);
|
||||
if (args[0] === "config") {
|
||||
fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config", pid:process.pid, args}) + "\\n");
|
||||
if (process.env.FIXTURE_CONFIG === "hang") {
|
||||
const { spawn } = require("node:child_process");
|
||||
const child = spawn(process.execPath, ["-e", "process.on('SIGTERM',()=>{}); setTimeout(()=>process.exit(0),10000)"], {stdio:"ignore"});
|
||||
fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config-descendant", pid:child.pid}) + "\\n");
|
||||
process.on("SIGTERM", () => {});
|
||||
setTimeout(() => process.exit(0), 10000);
|
||||
} else {
|
||||
if (process.env.FIXTURE_CONFIG === "large") console.log("x".repeat(8192));
|
||||
console.log("registry=" + (process.env.FIXTURE_CONFIG === "private" ? "https://fixture-user:fixture-password@registry.example.invalid/" : "https://registry.npmjs.org/") + "\\n@paperclipai:registry=undefined");
|
||||
console.error("fixture config diagnostic must stay private");
|
||||
}
|
||||
} else {
|
||||
const entries = fs.existsSync(process.env.FIXTURE_CALLS) ? fs.readFileSync(process.env.FIXTURE_CALLS, "utf8").trim().split("\\n").filter(Boolean).map(JSON.parse) : [];
|
||||
const attempt = entries.filter(e => e.kind === "npm").length + 1;
|
||||
fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"npm", pid:process.pid, args, cache:process.env.npm_config_cache}) + "\\n");
|
||||
const mode = process.env.FIXTURE_MODE;
|
||||
if (mode === "hang") { setTimeout(() => {}, 10000); }
|
||||
else if (mode === "permanent" || (mode === "recover" && attempt === 1)) { console.error("npm error code ETARGET"); process.exit(1); }
|
||||
else if (mode === "multibyte-recover" && attempt === 1) { console.error("npm error code ETARGET\\n" + "界".repeat(30 * 1024)); process.exit(1); }
|
||||
else if (mode === "auth") { console.error("npm error code E401"); process.exit(1); }
|
||||
else if (mode === "mixed") { console.error("npm error code ETARGET\\nnpm error code E401"); process.exit(1); }
|
||||
else if (mode === "network") { console.error("npm error code ECONNRESET"); process.exit(1); }
|
||||
else if (mode === "tarball-permanent" || mode === "tarball-large" || (mode === "tarball-recover" && attempt === 1)) {
|
||||
const url = "https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz";
|
||||
if (mode === "tarball-large") console.error("npm error code E401\\n" + "x".repeat(70 * 1024));
|
||||
console.error("npm error code E404\\nnpm error 404 Not Found - GET " + url + " - Not found"); process.exit(1);
|
||||
}
|
||||
else {
|
||||
const prefix = args[args.indexOf("--prefix") + 1];
|
||||
const bin = path.join(prefix, "node_modules", ".bin");
|
||||
fs.mkdirSync(bin, {recursive:true});
|
||||
fs.writeFileSync(path.join(bin, "paperclipai"), ${JSON.stringify(cli).replaceAll('$','\$')});
|
||||
fs.chmodSync(path.join(bin, "paperclipai"), 0o755);
|
||||
}
|
||||
}
|
||||
`);
|
||||
chmodSync(npmFixture, 0o755);
|
||||
const helper = new URL("../../tests/canary-onboarding/start-published-canary.mjs", import.meta.url).href;
|
||||
const driver = path.join(root, "driver.mjs");
|
||||
writeFileSync(driver, `import { startPublishedCanary } from ${JSON.stringify(helper)};
|
||||
import { appendFileSync } from "node:fs";
|
||||
globalThis.fetch = async (url, options) => {
|
||||
appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"metadata", url, redirect:options.redirect, credentials:options.credentials, headers:options.headers}) + "\\n");
|
||||
const manifest = {name:"@paperclipai/server",version:process.env.PAPERCLIPAI_VERSION,dist:{tarball:"https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz",integrity:"sha512-" + Buffer.alloc(64).toString("base64")}};
|
||||
const body = process.env.FIXTURE_METADATA === "hang" ? new ReadableStream({
|
||||
start(controller) { options.signal.addEventListener("abort", () => controller.error(options.signal.reason), {once:true}); },
|
||||
}) : JSON.stringify(manifest);
|
||||
const response = new Response(body, {status:process.env.FIXTURE_METADATA === "missing" ? 404 : 200});
|
||||
Object.defineProperty(response,"url",{value:url});
|
||||
return response;
|
||||
};
|
||||
try { await startPublishedCanary({version:${JSON.stringify(version)}, workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); }
|
||||
catch (error) { console.error(error.message); process.exitCode = 1; }
|
||||
`);
|
||||
const playwrightConfig = path.join(root, "playwright.config.mjs");
|
||||
if (playwright) {
|
||||
const originalConfig = path.join(repoRoot, "tests/canary-onboarding/playwright.config.ts");
|
||||
const testApi = new URL("../../node_modules/@playwright/test/index.mjs", import.meta.url).href;
|
||||
writeFileSync(path.join(root,"fixture.spec.mjs"), `import { test } from ${JSON.stringify(testApi)}; test("fixture", async()=>{});`);
|
||||
writeFileSync(playwrightConfig, `import original from ${JSON.stringify(originalConfig)};
|
||||
export default {...original, testDir:${JSON.stringify(root)}, testMatch:"fixture.spec.mjs", reporter:"list", projects:[{name:"fixture"}], webServer:{...original.webServer, timeout:${mode === "hang" ? 2500 : 300000}}};`);
|
||||
}
|
||||
const started = Date.now();
|
||||
try {
|
||||
const result = await new Promise((resolve, reject) => {
|
||||
const child = spawn(process.execPath, playwright
|
||||
? [path.join(repoRoot, "node_modules/@playwright/test/cli.js"), "test", "--config", playwrightConfig]
|
||||
: cancelAtKind
|
||||
? [fileURLToPath(helper), "2026.1009.0-canary.1", root, path.join(root, "data")]
|
||||
: [driver], { cwd: repoRoot, env: {
|
||||
...process.env, PATH: fixtureBin + path.delimiter + process.env.PATH,
|
||||
FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port), FIXTURE_CONFIG: config, FIXTURE_METADATA: metadata,
|
||||
PAPERCLIPAI_VERSION: version, PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"),
|
||||
npm_config_cache: path.join(root, "fresh-cache"),
|
||||
}, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let output = "";
|
||||
const cancellation = cancelAtKind && setInterval(() => {
|
||||
if (!existsSync(calls)) return;
|
||||
const rows = readFileSync(calls, "utf8").trim().split("\n").filter(Boolean).map(JSON.parse);
|
||||
if (rows.some(row => row.kind === cancelAtKind)) {
|
||||
clearInterval(cancellation);
|
||||
child.kill("SIGTERM");
|
||||
}
|
||||
}, 10);
|
||||
child.stdout.on("data", (data) => { output += data; });
|
||||
child.stderr.on("data", (data) => { output += data; });
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => { clearInterval(cancellation); resolve({code, output}); });
|
||||
});
|
||||
if (playwright && existsSync(path.join(root,"server.log"))) result.output += readFileSync(path.join(root,"server.log"), "utf8");
|
||||
const records = existsSync(calls) ? readFileSync(calls,"utf8").trim().split("\n").filter(Boolean).map(JSON.parse) : [];
|
||||
const prefixes = records.filter(call => call.kind === "npm").map(call => call.args[call.args.indexOf("--prefix") + 1]);
|
||||
if (playwright) {
|
||||
const deadline = Date.now() + 2000;
|
||||
while (prefixes.some(prefix => existsSync(prefix)) && Date.now() < deadline) {
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
}
|
||||
assert.deepEqual(prefixes.filter(prefix => existsSync(prefix)), [], "helper-owned install prefixes must be removed");
|
||||
return { ...result, elapsed: Date.now()-started, calls: records };
|
||||
|
||||
} finally { rmSync(root, {recursive:true, force:true}); }
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { canaryStartup } from "./canary-startup-fixture.mjs";
|
||||
|
||||
test("actual Playwright teardown stops the CLI and descendants that ignore SIGTERM", async () => {
|
||||
const result = await canaryStartup({ mode: "success", onboard: "serve", playwright: true });
|
||||
assert.equal(result.code, 0, result.output);
|
||||
assert.equal(result.calls.filter(call => call.kind === "onboard").length, 1);
|
||||
const processes = result.calls.filter(call => ["onboard", "descendant"].includes(call.kind));
|
||||
assert.equal(processes.length, 2);
|
||||
for (const call of processes) {
|
||||
const deadline = Date.now() + 2000;
|
||||
while (Date.now() < deadline) {
|
||||
try { process.kill(call.pid, 0); } catch (error) { if (error.code === "ESRCH") break; throw error; }
|
||||
await new Promise(resolve => setTimeout(resolve, 20));
|
||||
}
|
||||
assert.throws(() => process.kill(call.pid, 0), {code:"ESRCH"}, `fixture ${call.kind} outlived bounded teardown`);
|
||||
}
|
||||
});
|
||||
|
||||
test("actual Playwright startup timeout cancels a detached npm install", async () => {
|
||||
const result = await canaryStartup({ mode: "hang", playwright: true });
|
||||
assert.equal(result.code, 1);
|
||||
const installs = result.calls.filter(call => call.kind === "npm");
|
||||
assert.equal(installs.length, 1, result.output);
|
||||
assert.equal(result.calls.filter(call => call.kind === "onboard").length, 0);
|
||||
assert.throws(() => process.kill(installs[0].pid,0), {code:"ESRCH"});
|
||||
assert.ok(result.elapsed < 10000, `startup timeout did not stop npm: ${result.elapsed}ms`);
|
||||
});
|
||||
@@ -4,6 +4,8 @@ import { spawnSync } from 'node:child_process';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { isWorkspacePackage, parseWorkspaceEntries } from '../build-standalone-public-packages.mjs';
|
||||
import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertInstalledCodexPlatformProvenance, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource, installedDaemonProbeSource, macPublicInstallLifecycleProfile, macInstalledStartupProbeSource } from '../grok-public-install-sandbox.mjs';
|
||||
|
||||
const paths = { assets: '/private/staging/assets', consumer: '/private/staging/consumer', cache: '/private/staging/cache', uid: 1001, gid: 1001 };
|
||||
@@ -81,6 +83,118 @@ test('verification never elevates PR-controlled provisioning or cleanup on the h
|
||||
assert.ok(source.includes("const prerequisite = join(root, 'native/grok')"));
|
||||
});
|
||||
|
||||
test('the image and Grok downloads start before staging and join without weakening the npm checks', () => {
|
||||
const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8');
|
||||
// The optional Mac consumer owns separate scratch and no Linux prefetches.
|
||||
const linuxSource = source.slice(source.indexOf("assert.equal(process.platform, 'linux'"));
|
||||
const position = fragment => { const index = linuxSource.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; };
|
||||
// Exactly the pinned consumer image is pulled, for the platform the probes run on.
|
||||
const imagePull = position("prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE])");
|
||||
// Grok is provisioned into the prefetch directory, never straight into `native/grok`.
|
||||
const grokProvision = position("prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok])");
|
||||
assert.ok(source.includes("const stagedGrok = join(prefetched, 'grok')"));
|
||||
assert.doesNotMatch(source, /provision-grok\.mjs'\), prerequisite\]/);
|
||||
// Both downloads start before any package is staged and before the first sandbox runs.
|
||||
const staging = position("visit('@paperclipai/server')");
|
||||
const firstSandbox = position("isolated(['npm', 'install', '--ignore-scripts', '--omit=dev'");
|
||||
assert.ok(imagePull < staging && grokProvision < staging);
|
||||
assert.ok(position("await awaited('image', imageReady)") < firstSandbox);
|
||||
// The binary moves to the canonical host path only after npm is proven not to
|
||||
// have provisioned it, after the negative probe, and before the positive probe.
|
||||
const npmCheck = position("'npm must not provision Grok'");
|
||||
const negativeProbe = position("isolated(['node', '/packages/probe.mjs', 'missing'])");
|
||||
const grokJoin = position("await awaited('grok', grokReady)");
|
||||
const move = position('renameSync(stagedGrok, prerequisite)');
|
||||
const positiveProbe = position("isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite })");
|
||||
assert.ok(npmCheck < negativeProbe && negativeProbe < grokJoin && grokJoin < move && move < positiveProbe);
|
||||
// The children run through the helper covered by prefetch-child.test.mjs, and
|
||||
// cleanup waits for them before removing the directory they write into.
|
||||
assert.ok(source.includes("import { startPrefetch, stopPrefetches } from './prefetch-child.mjs'"));
|
||||
assert.ok(position('await stopPrefetches(prefetches)') < position('rmSync(root, { recursive: true, force: true })'));
|
||||
assert.equal(linuxSource.match(/rmSync\(root/g).length, 1);
|
||||
});
|
||||
|
||||
test('the bundled-dependency installs run together while the other packages are copied, and pack in order', () => {
|
||||
const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8');
|
||||
const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; };
|
||||
// The installs are children of the script's own CLI entry through the helper
|
||||
// covered by stage-bundled-packages.test.mjs, never in-process calls.
|
||||
assert.doesNotMatch(source, /prepareBundledPackage/);
|
||||
assert.ok(source.includes("import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs'"));
|
||||
// Only the bundled packages' sources are staged before the installs start;
|
||||
// the installs are registered for cleanup at once, before anything else can
|
||||
// fail; the other packages are copied while the installs run; the join comes
|
||||
// before the packs start; and the packs run through the helper covered by
|
||||
// pack-staged-packages.test.mjs, bundled packages first, each registered for
|
||||
// cleanup as it starts, with the results put back into dependency order
|
||||
// before every tarball is checked.
|
||||
const bundledSources = position('for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry)');
|
||||
const start = position("startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo })");
|
||||
const registered = position('prefetches.push(...installs.map(({ handle }) => handle))');
|
||||
const otherSources = position('for (const entry of entries.filter(entry => !entry.bundled))');
|
||||
const join = position("await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)))");
|
||||
assert.ok(source.includes("import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs'"));
|
||||
const packs = position("await packStagedPackages([...entries].sort((a, b) => Number(b.bundled) - Number(a.bundled)), { logDir: prefetched, env, concurrency, register: handle => prefetches.push(handle) })");
|
||||
const reordered = position('packs.sort((a, b) => a.index - b.index)');
|
||||
const checks = position("for (const tarball of tarballs) assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball])");
|
||||
const copied = position('for (const tarball of tarballs) {');
|
||||
assert.ok(bundledSources < start && start < registered && registered < otherSources && otherSources < join && join < packs && packs < reordered && reordered < checks && checks < copied);
|
||||
assert.equal(source.match(/run\('npm', \['pack'/g).length, 1, 'only the lifecycle sentinel is packed in-process');
|
||||
assert.ok(source.includes("destination: join(root, `pack-${index}`)"), 'each pack writes into its own directory');
|
||||
// The timing line reports the split staging phase, each install, and the
|
||||
// pack step with what the packs would have cost one after another.
|
||||
for (const key of ['stage_ui_dist_ms', 'stage_sources_ms', 'stage_bundled_ms', 'stage_pack_ms', 'stage_pack_check_ms']) assert.ok(source.includes(`timing.${key} =`));
|
||||
assert.ok(source.includes('...bundledInstallTimings(installs)'));
|
||||
assert.ok(source.includes('Object.assign(timing, packTimings(packs, concurrency))'));
|
||||
});
|
||||
|
||||
test('the verifier stages only workspace packages, so it skips the standalone build release.sh needs', () => {
|
||||
const source = readFileSync(new URL('../verify-grok-npm-install.mjs', import.meta.url), 'utf8');
|
||||
const position = fragment => { const index = source.indexOf(fragment); assert.notEqual(index, -1, `verifier must contain ${fragment}`); return index; };
|
||||
// The standalone public packages are the ones outside the pnpm workspace
|
||||
// (the sandbox-provider plugins). release.sh builds them because it publishes
|
||||
// them; the verifier packs only the closure of the server and the CLI under
|
||||
// `workspace:` specs, and a `workspace:` spec can only name a workspace
|
||||
// package, so no tarball here ever contains or depends on a standalone build.
|
||||
// The one reader of that build is qualification-package retention, which
|
||||
// packs the Daytona plugin from its `dist`, so the build runs exactly when
|
||||
// retention is requested: the single reference sits inside that gate, and
|
||||
// the same flag is what later decides whether retention runs at all.
|
||||
const standaloneBuild = "run(process.execPath, [join(repo, 'scripts/build-standalone-public-packages.mjs')], repo)";
|
||||
assert.equal(source.split('build-standalone-public-packages').length - 1, 1, 'the standalone build is referenced once, inside the retention gate');
|
||||
const gate = position("const retainQualificationPackages = Boolean(process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR);\n if (retainQualificationPackages) {\n const standaloneStarted = performance.now();\n " + standaloneBuild);
|
||||
assert.ok(source.includes('timing.stage_standalone_build_ms ='), 'the gated build is timed under its own key');
|
||||
assert.ok(source.includes('if (retainQualificationPackages) {\n const retained = retainRunnerQualificationPackages({'), 'retention is gated by the same flag as the build');
|
||||
assert.doesNotMatch(source, /if \(process\.env\.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR\)/, 'no second reading of the retention flag can diverge from the build gate');
|
||||
const retention = readFileSync(new URL('../retain-runner-qualification-packages.mjs', import.meta.url), 'utf8');
|
||||
assert.ok(retention.includes("const pluginRelative = 'packages/plugins/sandbox-providers/daytona';") && retention.includes("cpSync(join(repo, pluginRelative, 'dist'), join(plugin, 'dist'), { recursive: true })"), 'retention still packs the Daytona plugin from its dist, which is why the gated build exists');
|
||||
const repo = new URL('../../', import.meta.url);
|
||||
const listing = spawnSync(process.execPath, [fileURLToPath(new URL('scripts/release-package-map.mjs', repo)), 'list'], { cwd: fileURLToPath(repo), encoding: 'utf8' });
|
||||
assert.equal(listing.status, 0, listing.stderr);
|
||||
const packages = new Map(listing.stdout.trim().split('\n').map(line => line.split('\t')).map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(new URL(`${dir}/package.json`, repo), 'utf8')) }]));
|
||||
const needed = new Set();
|
||||
const visit = name => {
|
||||
if (needed.has(name)) return;
|
||||
const entry = packages.get(name); assert.ok(entry, `missing public workspace dependency ${name}`); needed.add(name);
|
||||
for (const [dep, spec] of Object.entries({ ...entry.manifest.dependencies, ...entry.manifest.optionalDependencies })) if (spec.startsWith('workspace:')) visit(dep);
|
||||
};
|
||||
assert.ok(source.includes("visit('@paperclipai/server');\n visit('paperclipai');"));
|
||||
visit('@paperclipai/server'); visit('paperclipai');
|
||||
const workspace = parseWorkspaceEntries(readFileSync(new URL('pnpm-workspace.yaml', repo), 'utf8'));
|
||||
const standalone = [...packages.values()].filter(({ dir }) => !isWorkspacePackage(dir, workspace)).map(({ dir }) => dir);
|
||||
assert.ok(standalone.length > 0, 'the public package map must still carry standalone packages for this check to mean anything');
|
||||
assert.ok(standalone.includes('packages/plugins/sandbox-providers/daytona'), 'the Daytona plugin retention packs is a standalone package, so pnpm build alone cannot produce its dist');
|
||||
for (const name of needed) assert.ok(isWorkspacePackage(packages.get(name).dir, workspace), `${name} is staged by the verifier but is not a workspace package`);
|
||||
// The server publishes `ui-dist`, so that copy stays, before any package is
|
||||
// staged, reusing the UI that `pnpm build` produced, and it is timed. The
|
||||
// gated standalone build keeps release.sh's order: after the closure is
|
||||
// known and before the UI copy and any staged source.
|
||||
assert.deepEqual(JSON.parse(readFileSync(new URL('server/package.json', repo), 'utf8')).files, ['dist', 'ui-dist', 'skills']);
|
||||
assert.ok(source.includes("PAPERCLIP_RELEASE_REUSE_UI_DIST: '1'"));
|
||||
const uiDist = position("run('bash', [join(repo, 'scripts/prepare-server-ui-dist.sh')], repo)");
|
||||
assert.ok(position("visit('paperclipai')") < gate && gate < uiDist && uiDist < position('timing.stage_ui_dist_ms =') && uiDist < position('const stageSource = '));
|
||||
});
|
||||
|
||||
test('the installed Codex probe exercises the public export and rejects incomplete or mismatched packages', () => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'installed-codex-probe-')));
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { stopPrefetches } from '../prefetch-child.mjs';
|
||||
import { PACK_ARGS, packConcurrency, packStagedPackages, packTimings } from '../pack-staged-packages.mjs';
|
||||
|
||||
// A small local script stands in for npm; no test here needs npm or a network.
|
||||
// It is run as `node <script> pack --ignore-scripts --pack-destination <dir>`,
|
||||
// the same argument list the real npm receives.
|
||||
const withScratch = async body => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'pack-staged-packages-'));
|
||||
try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); }
|
||||
};
|
||||
const entriesFor = (root, names) => names.map((name, index) => {
|
||||
const target = join(root, `package-${index}`);
|
||||
mkdirSync(target);
|
||||
return { name, index, target, destination: join(root, `pack-${index}`) };
|
||||
});
|
||||
const fakeNpm = (root, body) => { const script = join(root, 'fake-npm.mjs'); writeFileSync(script, body); return [process.execPath, script]; };
|
||||
const settle = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
test('the concurrency defaults to one child per core', () => {
|
||||
assert.ok(Number.isInteger(packConcurrency()) && packConcurrency() >= 1);
|
||||
assert.deepEqual(PACK_ARGS, ['pack', '--ignore-scripts', '--pack-destination']);
|
||||
});
|
||||
|
||||
test('packs run together up to the concurrency, in the order given, and the tarballs come back in entry order', async () => withScratch(async root => {
|
||||
// Each stand-in records when it started and finished, writes one tarball
|
||||
// named after its package, and prints the tarball name as npm does. The
|
||||
// first two packs are slow and the last two fast, so with two children at
|
||||
// a time the slow pair runs together and the fast pair starts only once one
|
||||
// of them has finished; the fast packs then finish out of entry order.
|
||||
const command = fakeNpm(root, `
|
||||
import { basename, join } from 'node:path';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
const args = process.argv.slice(2);
|
||||
const destination = args[args.length - 1];
|
||||
const name = basename(process.cwd());
|
||||
const started = Date.now();
|
||||
const delay = name === 'package-0' || name === 'package-1' ? 400 : 50;
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay);
|
||||
writeFileSync(join(destination, name + '-1.0.0.tgz'), 'tarball');
|
||||
writeFileSync(join(destination, '..', 'span-' + name), started + ' ' + Date.now());
|
||||
console.log(args.join(' '));
|
||||
console.log(name + '-1.0.0.tgz');
|
||||
`);
|
||||
const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db', '@paperclipai/shared', 'paperclipai']);
|
||||
const registered = [];
|
||||
const packs = await packStagedPackages(entries, { logDir: root, concurrency: 2, register: handle => registered.push(handle), command });
|
||||
assert.deepEqual(packs.map(({ name }) => name), entries.map(({ name }) => name));
|
||||
assert.deepEqual(packs.map(({ index }) => index), [0, 1, 2, 3]);
|
||||
for (const [index, pack] of packs.entries()) {
|
||||
assert.equal(pack.tarball, join(root, `pack-${index}`, `package-${index}-1.0.0.tgz`));
|
||||
assert.equal(readFileSync(pack.tarball, 'utf8'), 'tarball');
|
||||
assert.ok(Number.isInteger(pack.durationMs) && pack.durationMs >= 0);
|
||||
const log = readFileSync(join(root, `pack-${index}.log`), 'utf8');
|
||||
assert.equal(log.split('\n')[0], `pack --ignore-scripts --pack-destination ${join(root, `pack-${index}`)}`);
|
||||
}
|
||||
assert.equal(registered.length, 4, 'every pack is registered for cleanup as it starts');
|
||||
for (const handle of registered) assert.equal(handle.child.exitCode, 0);
|
||||
const span = name => readFileSync(join(root, `span-${name}`), 'utf8').split(' ').map(Number);
|
||||
const [slowA, slowB, fastA, fastB] = ['package-0', 'package-1', 'package-2', 'package-3'].map(span);
|
||||
assert.ok(slowA[0] < slowB[1] && slowB[0] < slowA[1], 'the two slow packs ran at the same time');
|
||||
const firstSlowEnd = Math.min(slowA[1], slowB[1]);
|
||||
assert.ok(fastA[0] >= firstSlowEnd - 50 && fastB[0] >= firstSlowEnd - 50, 'a third pack starts only once a slot is free');
|
||||
const timings = packTimings(packs, 2);
|
||||
assert.equal(timings.pack_concurrency, 2);
|
||||
assert.equal(timings.pack_serial_ms, packs.reduce((total, { durationMs }) => total + durationMs, 0));
|
||||
assert.equal(timings.pack_longest_ms, Math.max(...packs.map(({ durationMs }) => durationMs)));
|
||||
}));
|
||||
|
||||
test('a failed pack rejects with the tail of its log, launches nothing more, and leaves the running pack to be stopped', async () => withScratch(async root => {
|
||||
const command = fakeNpm(root, `
|
||||
import { basename } from 'node:path';
|
||||
if (basename(process.cwd()) === 'package-1') { console.error('pack exploded'); process.exit(2); }
|
||||
setTimeout(() => {}, 30_000);
|
||||
`);
|
||||
const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db', '@paperclipai/shared', 'paperclipai']);
|
||||
const registered = [];
|
||||
await assert.rejects(packStagedPackages(entries, { logDir: root, concurrency: 2, register: handle => registered.push(handle), command }), /pack-1 prefetch failed \(exit 2\): pack exploded/);
|
||||
await settle(100);
|
||||
assert.equal(registered.length, 2, 'the two queued packs never start after the failure');
|
||||
assert.equal(registered[0].child.exitCode, null, 'the other pack is still running when the failure surfaces');
|
||||
assert.ok(!existsSync(join(root, 'pack-2')) && !existsSync(join(root, 'pack-3')));
|
||||
await stopPrefetches(registered);
|
||||
assert.notEqual(registered[0].child.signalCode ?? registered[0].child.exitCode, null);
|
||||
}));
|
||||
|
||||
test('a pack that exits without producing exactly one tarball is a failure', async () => withScratch(async root => {
|
||||
const command = fakeNpm(root, `
|
||||
import { basename, join } from 'node:path';
|
||||
import { writeFileSync } from 'node:fs';
|
||||
const destination = process.argv[process.argv.length - 1];
|
||||
const name = basename(process.cwd());
|
||||
if (name === 'package-0') { writeFileSync(join(destination, 'a.tgz'), ''); writeFileSync(join(destination, 'b.tgz'), ''); }
|
||||
`);
|
||||
const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db']);
|
||||
await assert.rejects(packStagedPackages(entries, { logDir: root, concurrency: 1, command }), /@paperclipai\/server pack produced 2 tarballs/);
|
||||
assert.deepEqual(readdirSync(root).filter(name => name.startsWith('pack-') && !name.endsWith('.log')), ['pack-0'], 'the second pack never started');
|
||||
await assert.rejects(packStagedPackages([entries[1]], { logDir: root, concurrency: 1, command }), /@paperclipai\/db pack produced 0 tarballs/);
|
||||
}));
|
||||
|
||||
test('no entries resolve at once', async () => withScratch(async root => {
|
||||
assert.deepEqual(await packStagedPackages([], { logDir: root, concurrency: 2, command: [process.execPath, '-e', ''] }), []);
|
||||
}));
|
||||
@@ -0,0 +1,186 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { startPrefetch, stopPrefetches } from '../prefetch-child.mjs';
|
||||
|
||||
// Small local Node children stand in for the image pull and the Grok download;
|
||||
// none of these tests needs Docker or a network.
|
||||
const withScratch = async body => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'prefetch-child-'));
|
||||
try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); }
|
||||
};
|
||||
const start = (root, name, script, cmd = process.execPath, args = ['-e', script]) =>
|
||||
startPrefetch({ name, cmd, args, cwd: root, env: process.env, logPath: join(root, `${name}.log`) });
|
||||
const blockFor = ms => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
||||
const untilLogHas = async (path, text) => {
|
||||
for (let attempt = 0; attempt < 400; attempt++) {
|
||||
try { if (readFileSync(path, 'utf8').includes(text)) return; } catch {}
|
||||
await new Promise(resolve => setTimeout(resolve, 25));
|
||||
}
|
||||
throw new Error(`log never contained ${text}`);
|
||||
};
|
||||
|
||||
test('a successful prefetch resolves, keeps its output, and is left alone by stop', async () => withScratch(async root => {
|
||||
const handle = start(root, 'ok', "setTimeout(() => process.stdout.write('fetched\\n'), 150)");
|
||||
// The parent keeps its default, immediate termination: no handler of ours.
|
||||
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) assert.equal(process.listenerCount(signal), 0, `${signal} handler installed in the parent`);
|
||||
await handle.done;
|
||||
assert.equal(handle.child.exitCode, 0);
|
||||
assert.equal(readFileSync(join(root, 'ok.log'), 'utf8'), 'fetched\n');
|
||||
assert.ok(handle.durationMs >= 100 && handle.durationMs < 5000, `durationMs ${handle.durationMs}`);
|
||||
await stopPrefetches([handle]);
|
||||
assert.equal(handle.child.signalCode, null);
|
||||
}));
|
||||
|
||||
test('the duration ends at the download, not when a blocked parent notices the exit', async () => withScratch(async root => {
|
||||
const handle = start(root, 'early', "setTimeout(() => process.stdout.write('Status: Downloaded\\n'), 100)");
|
||||
// Synchronous work in the parent, as package staging is in the verifier.
|
||||
blockFor(1500);
|
||||
const noticed = Date.now();
|
||||
await handle.done;
|
||||
assert.ok(Date.now() - noticed < 500, 'the exit was pending by the time the parent yielded');
|
||||
assert.ok(handle.durationMs < 900, `durationMs ${handle.durationMs} must not include the 1500ms the parent was blocked`);
|
||||
}));
|
||||
|
||||
test('a nonzero exit rejects with the log tail where the result is awaited', async () => withScratch(async root => {
|
||||
const handle = start(root, 'bad', "process.stderr.write('registry unreachable\\n'); process.exit(3)");
|
||||
await assert.rejects(handle.done, /^Error: bad prefetch failed \(exit 3\): registry unreachable$/);
|
||||
await stopPrefetches([handle]);
|
||||
}));
|
||||
|
||||
test('a command that cannot start rejects instead of hanging the join or the stop', async () => withScratch(async root => {
|
||||
const handle = start(root, 'missing', '', join(root, 'no-such-command'), []);
|
||||
await assert.rejects(handle.done, /missing prefetch failed \(.*ENOENT/);
|
||||
await stopPrefetches([handle]);
|
||||
}));
|
||||
|
||||
test('stopping a running prefetch waits for it to exit so its directory can be removed safely', async () => withScratch(async root => {
|
||||
const handle = start(root, 'slow', "process.stdout.write('started\\n'); setInterval(() => {}, 1000)");
|
||||
await untilLogHas(join(root, 'slow.log'), 'started');
|
||||
const stopping = Date.now();
|
||||
await stopPrefetches([handle]);
|
||||
assert.ok(Date.now() - stopping < 4000);
|
||||
assert.equal(handle.child.signalCode, 'SIGTERM');
|
||||
assert.equal(handle.child.exitCode, null);
|
||||
await assert.rejects(handle.done, /slow prefetch failed \(SIGTERM\): started/);
|
||||
}));
|
||||
|
||||
test('a prefetch that ignores SIGTERM is killed after the grace period', { skip: process.platform === 'win32' && 'SIGTERM cannot be ignored on Windows' }, async () => withScratch(async root => {
|
||||
const handle = start(root, 'stubborn', "process.on('SIGTERM', () => {}); process.stdout.write('ignoring\\n'); setInterval(() => {}, 1000)");
|
||||
await untilLogHas(join(root, 'stubborn.log'), 'ignoring');
|
||||
const stopping = Date.now();
|
||||
await stopPrefetches([handle], { graceMs: 200 });
|
||||
assert.ok(Date.now() - stopping >= 200 && Date.now() - stopping < 4000);
|
||||
assert.equal(handle.child.signalCode, 'SIGKILL');
|
||||
}));
|
||||
|
||||
// A process is gone once the kernel no longer knows it, or once it is a zombie
|
||||
// that only waits to be reaped: either way it can no longer write anywhere.
|
||||
const gone = pid => {
|
||||
try { process.kill(pid, 0); } catch (error) { return error.code === 'ESRCH'; }
|
||||
try { return /^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch { return false; }
|
||||
};
|
||||
const untilGone = async pid => {
|
||||
for (let attempt = 0; attempt < 200; attempt++) {
|
||||
if (gone(pid)) return true;
|
||||
await new Promise(resolve => setTimeout(resolve, 25));
|
||||
}
|
||||
return false;
|
||||
};
|
||||
// The child starts a process of its own that announces its pid and then runs
|
||||
// on, the way the bundled-dependency installs block in `npm install`.
|
||||
const GRANDCHILD_SCRIPT = `
|
||||
const { spawn } = require('node:child_process');
|
||||
const grandchild = spawn(process.execPath, ['-e', "process.stdout.write('grandchild ' + process.pid + '\\\\n'); setInterval(() => {}, 1000)"], { stdio: 'inherit' });
|
||||
grandchild.on('exit', () => process.exit(0));
|
||||
`;
|
||||
const grandchildPid = async log => { await untilLogHas(log, 'grandchild '); return Number(readFileSync(log, 'utf8').match(/grandchild (\d+)/)[1]); };
|
||||
|
||||
test('stopping a prefetch also stops the processes it started', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => {
|
||||
const handle = start(root, 'tree', GRANDCHILD_SCRIPT);
|
||||
const pid = await grandchildPid(join(root, 'tree.log'));
|
||||
assert.ok(!gone(pid), 'the grandchild runs before the stop');
|
||||
await stopPrefetches([handle]);
|
||||
assert.ok(await untilGone(pid), `grandchild ${pid} outlived stopPrefetches`);
|
||||
assert.notEqual(handle.child.signalCode ?? handle.child.exitCode, null);
|
||||
}));
|
||||
|
||||
test('a stop request written to the pipe ends the command and the processes it started, with no signal to the supervisor', { skip: process.platform === 'win32' && 'the grandchild check reads /proc' }, async () => withScratch(async root => {
|
||||
// This is the stop path on a platform without process groups: there `kill`
|
||||
// would end the supervisor outright and leave its command running, so
|
||||
// `stopPrefetches` writes a request instead. The supervisor answers it
|
||||
// the way it answers the parent's exit, and its own exit still reports the
|
||||
// command's fate.
|
||||
const handle = start(root, 'tree', GRANDCHILD_SCRIPT);
|
||||
const pid = await grandchildPid(join(root, 'tree.log'));
|
||||
handle.child.stdin.end('stop\n');
|
||||
await assert.rejects(handle.done, /tree prefetch failed \(SIGTERM\)/);
|
||||
assert.ok(await untilGone(pid), `grandchild ${pid} outlived the stop request`);
|
||||
assert.match(readFileSync(join(root, 'tree.log'), 'utf8'), /stop requested; stopping/);
|
||||
assert.doesNotMatch(readFileSync(join(root, 'tree.log'), 'utf8'), /parent exited/);
|
||||
await stopPrefetches([handle]);
|
||||
}));
|
||||
|
||||
// A separate parent process stands in for the verifier: it starts a prefetch
|
||||
// whose child starts a grandchild, then does `work` (a module-scope statement).
|
||||
const parentScript = (root, work) => `
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)};
|
||||
startPrefetch({ name: 'tree', cmd: process.execPath, args: ['-e', ${JSON.stringify(GRANDCHILD_SCRIPT)}], cwd: ${JSON.stringify(root)}, env: process.env, logPath: ${JSON.stringify(join(root, 'tree.log'))} });
|
||||
${work}
|
||||
`;
|
||||
|
||||
for (const signal of ['SIGINT', 'SIGKILL']) {
|
||||
test(`${signal} to the parent ends the processes a prefetch started, and the parent dies of it at once`, { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => {
|
||||
// The parent runs no handler, so it dies of the signal immediately, and the
|
||||
// prefetch tree stops because the parent is gone, not because it was told:
|
||||
// SIGKILL leaves the parent no chance to tell anyone anything.
|
||||
const parent = spawn(process.execPath, ['--input-type=module', '-e', parentScript(root, 'setInterval(() => {}, 1000);')], { cwd: root, stdio: 'ignore' });
|
||||
const exited = new Promise(resolve => parent.once('exit', (code, signal) => resolve({ code, signal })));
|
||||
const pid = await grandchildPid(join(root, 'tree.log'));
|
||||
parent.kill(signal);
|
||||
assert.deepEqual(await exited, { code: null, signal });
|
||||
assert.ok(await untilGone(pid), `grandchild ${pid} outlived the parent's ${signal}`);
|
||||
assert.match(readFileSync(join(root, 'tree.log'), 'utf8'), /parent exited; stopping/);
|
||||
}));
|
||||
}
|
||||
|
||||
test('Ctrl-C while the parent is blocked in synchronous work stops it at once, and the prefetch tree with it', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => {
|
||||
// The verifier runs `execFileSync` for the standalone builds while prefetches
|
||||
// run, and a JavaScript signal handler cannot run until that returns. The
|
||||
// parent gets its own group here so the signal can be sent the way a
|
||||
// terminal sends Ctrl-C: to the foreground group, which is the parent and
|
||||
// its synchronous command, and not the detached prefetch tree.
|
||||
const parent = spawn(process.execPath, ['--input-type=module', '-e', parentScript(root, "execFileSync(process.execPath, ['-e', 'setTimeout(() => {}, 20000)'], { stdio: 'ignore' });")], { cwd: root, stdio: 'ignore', detached: true });
|
||||
try {
|
||||
const exited = new Promise(resolve => parent.once('exit', (code, signal) => resolve({ code, signal })));
|
||||
const pid = await grandchildPid(join(root, 'tree.log'));
|
||||
const signalled = Date.now();
|
||||
process.kill(-parent.pid, 'SIGINT');
|
||||
assert.deepEqual(await exited, { code: null, signal: 'SIGINT' });
|
||||
assert.ok(Date.now() - signalled < 2000, `the parent took ${Date.now() - signalled}ms to die: it waited for the synchronous command`);
|
||||
assert.ok(await untilGone(pid), `grandchild ${pid} outlived the parent's Ctrl-C`);
|
||||
} finally {
|
||||
try { process.kill(-parent.pid, 'SIGKILL'); } catch {}
|
||||
}
|
||||
}));
|
||||
|
||||
test('an unawaited failure never becomes an unhandled rejection', () => {
|
||||
const script = `
|
||||
import { startPrefetch } from ${JSON.stringify(new URL('../prefetch-child.mjs', import.meta.url).href)};
|
||||
import { mkdtempSync, rmSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path';
|
||||
const root = mkdtempSync(join(tmpdir(), 'prefetch-unawaited-'));
|
||||
try {
|
||||
startPrefetch({ name: 'ignored', cmd: process.execPath, args: ['-e', 'process.exit(2)'], cwd: root, env: process.env, logPath: join(root, 'ignored.log') });
|
||||
await new Promise(resolve => setTimeout(resolve, 500));
|
||||
console.log('still running');
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
`;
|
||||
const result = spawnSync(process.execPath, ['--input-type=module', '-e', script], { encoding: 'utf8', timeout: 10_000 });
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.match(result.stdout, /still running/);
|
||||
assert.doesNotMatch(result.stderr, /unhandled/i);
|
||||
});
|
||||
@@ -1,10 +1,14 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { copyFileSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { tmpdir } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
import { createServer } from "node:http";
|
||||
import { canaryStartup } from "./canary-startup-fixture.mjs";
|
||||
import { canRetryPublishedTarball404, TARBALL_METADATA_TIMEOUT_MS } from "../../tests/canary-onboarding/start-published-canary.mjs";
|
||||
|
||||
const repoRoot = path.resolve(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
@@ -16,6 +20,45 @@ function readWorkflow(name) {
|
||||
return readFileSync(path.join(repoRoot, ".github/workflows", name), "utf8");
|
||||
}
|
||||
|
||||
test("Docker Hub mirrors apply to both image builders without widening publication access", () => {
|
||||
const dockerWorkflow = readWorkflow("docker.yml");
|
||||
const releaseWorkflow = readWorkflow("release.yml");
|
||||
const docker = dockerWorkflow.split(" build-and-push:\n")[1].split("\n merge-and-push:")[0];
|
||||
const preview = releaseWorkflow.split(" image_preview:\n")[1].split("\n publish_image_preview:")[0];
|
||||
|
||||
for (const job of [docker, preview]) {
|
||||
const steps = job.split(/\n(?= - )/);
|
||||
const builder = steps.find((step) => /uses: docker\/setup-buildx-action@/.test(step));
|
||||
const build = steps.find((step) => /uses: docker\/build-push-action@/.test(step));
|
||||
assert.ok(builder && build, "the mirror must configure the builder that runs the image build");
|
||||
assert.ok(steps.indexOf(builder) < steps.indexOf(build));
|
||||
const config = builder.match(/buildkitd-config-inline: \|\n((?: {12,}[^\n]*(?:\n|$))+)/)?.[1];
|
||||
assert.ok(config, "configure the BuildKit container, not only the host Docker daemon");
|
||||
assert.deepEqual([...config.matchAll(/^\s*\[registry\."([^"]+)"\]/gm)].map((match) => match[1]), ["docker.io"],
|
||||
"GHCR publishing and cache traffic must not use the pull mirror");
|
||||
const mirrors = config.match(/^\s*mirrors = (\[[^\n]+\])$/m)?.[1];
|
||||
assert.ok(mirrors);
|
||||
assert.deepEqual(JSON.parse(mirrors), ["mirror.gcr.io"]);
|
||||
assert.doesNotMatch(config, /^\s*(?:http|insecure|ca|keypair)\s*=/m,
|
||||
"keep verified HTTPS and anonymous public-image pulls");
|
||||
assert.doesNotMatch(builder, /driver-opts:|buildkitd-flags:|secrets\./,
|
||||
"do not change bootstrap images, entitlements, or credentials");
|
||||
}
|
||||
|
||||
assert.match(docker, /platform: linux\/amd64/);
|
||||
assert.match(docker, /platform: linux\/arm64/);
|
||||
assert.match(docker, /permissions:\n\s+contents: read\n\s+packages: write/);
|
||||
assert.match(docker, /outputs: type=image,name=ghcr\.io\/\$\{\{ github\.repository \}\},push-by-digest=true,name-canonical=true,push=true/);
|
||||
assert.match(docker, /cache-to: type=registry,ref=ghcr\.io\/\$\{\{ github\.repository \}\}:buildcache-\$\{\{ matrix\.arch \}\},mode=max/);
|
||||
assert.match(preview, /permissions:\n\s+contents: read\n\s+steps:/);
|
||||
assert.match(preview, /platforms: linux\/amd64\n\s+push: false/);
|
||||
assert.match(preview, /outputs: type=docker,dest=\$\{\{ runner\.temp \}\}\/preview-image\.tar/);
|
||||
assert.doesNotMatch(preview, /docker\/login-action|secrets\.|packages: write|id-token: write/);
|
||||
assert.match(releaseWorkflow, /publish_image_preview:\n[\s\S]*?needs: \[plan_preview, image_preview\]/);
|
||||
assert.doesNotMatch(dockerWorkflow.split(" merge-and-push:\n")[1], /buildkitd-config-inline/,
|
||||
"the registry-only manifest merge does not need a Docker Hub mirror");
|
||||
});
|
||||
|
||||
test("chaos verification isolates callers that verify the same source commit", () => {
|
||||
const chaosWorkflow = readWorkflow("runner-chaos-evals.yml");
|
||||
const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1];
|
||||
@@ -62,6 +105,131 @@ test("canary reuses exact-source proof while stable keeps full verification", ()
|
||||
);
|
||||
});
|
||||
|
||||
function canaryAttachmentScript() {
|
||||
const publish = readWorkflow("release.yml").split(" publish_canary:\n")[1].split(" smoke_canary_onboarding:\n")[0];
|
||||
const step = publish.split(" - name: Attach verified canary source to local master\n")[1]?.split("\n - ")[0];
|
||||
assert.ok(step, "the trusted publisher must attach its pinned checkout before releasing");
|
||||
assert.match(step, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
|
||||
const script = step.match(/run: \|\n((?: {10}[^\n]*\n)+)/)?.[1];
|
||||
assert.ok(script, "missing inline canary branch attachment shell");
|
||||
return script.replace(/^ {10}/gm, "");
|
||||
}
|
||||
|
||||
function withCanaryGitFixture(check) {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "canary-publisher-git-"));
|
||||
const cwd = path.join(root, "repository with spaces");
|
||||
mkdirSync(cwd);
|
||||
const git = (...args) => {
|
||||
const result = spawnSync("git", args, { cwd, encoding: "utf8" });
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
return result.stdout.trim();
|
||||
};
|
||||
try {
|
||||
git("init", "--initial-branch=master");
|
||||
git("config", "user.name", "Release fixture");
|
||||
git("config", "user.email", "release-fixture@example.invalid");
|
||||
git("config", "commit.gpgsign", "false");
|
||||
git("config", "core.hooksPath", "/dev/null");
|
||||
writeFileSync(path.join(cwd, "source.txt"), "pinned source\n");
|
||||
git("add", "source.txt");
|
||||
git("commit", "-m", "Create pinned source");
|
||||
const pinned = git("rev-parse", "HEAD");
|
||||
writeFileSync(path.join(cwd, "source.txt"), "newer master source\n");
|
||||
git("commit", "-am", "Advance master");
|
||||
const newer = git("rev-parse", "HEAD");
|
||||
git("update-ref", "refs/remotes/origin/master", newer);
|
||||
git("checkout", "--detach", pinned);
|
||||
const env = {
|
||||
...process.env, REPO_ROOT: cwd, SOURCE_SHA: pinned,
|
||||
GITHUB_ACTIONS: "true", GITHUB_REPOSITORY: "paperclipai/paperclip",
|
||||
GITHUB_EVENT_NAME: "push", GITHUB_REF: "refs/heads/master",
|
||||
};
|
||||
const attach = (overrides = {}) => spawnSync("bash", ["--noprofile", "--norc", "-c", canaryAttachmentScript()], {
|
||||
cwd, env: { ...env, ...overrides }, encoding: "utf8", timeout: 5_000,
|
||||
});
|
||||
// Run the original release guard without running release.sh, npm, or a publisher.
|
||||
const guard = () => spawnSync("bash", ["-c", 'source "$1"; require_on_master_branch', "guard", path.join(repoRoot, "scripts/release-lib.sh")], {
|
||||
cwd, env, encoding: "utf8", timeout: 5_000,
|
||||
});
|
||||
check({ cwd, git, pinned, newer, attach, guard });
|
||||
} finally {
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test("canary publisher attaches the exact source while local and remote master are newer", () => {
|
||||
withCanaryGitFixture(({ cwd, git, pinned, newer, attach, guard }) => {
|
||||
assert.equal(guard().status, 1, "the unchanged release guard must reject detached HEAD even in Actions");
|
||||
assert.equal(git("rev-parse", "master"), newer);
|
||||
const result = attach();
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.equal(git("symbolic-ref", "--short", "HEAD"), "master");
|
||||
assert.equal(git("rev-parse", "HEAD"), pinned);
|
||||
assert.equal(git("rev-parse", "master"), pinned);
|
||||
assert.equal(git("rev-parse", "origin/master"), newer, "do not fetch or follow floating remote master");
|
||||
assert.equal(readFileSync(path.join(cwd, "source.txt"), "utf8"), "pinned source\n");
|
||||
assert.equal(git("status", "--porcelain"), "");
|
||||
assert.equal(guard().status, 0, "the real release guard must accept the pinned local master");
|
||||
});
|
||||
});
|
||||
|
||||
for (const [name, overrides] of [
|
||||
["fork repository", { GITHUB_REPOSITORY: "example/paperclip" }],
|
||||
["manual dispatch", { GITHUB_EVENT_NAME: "workflow_dispatch" }],
|
||||
["scheduled release", { GITHUB_EVENT_NAME: "schedule" }],
|
||||
["non-master push", { GITHUB_REF: "refs/heads/topic" }],
|
||||
["tag push", { GITHUB_REF: "refs/tags/canary/v1" }],
|
||||
["missing source SHA", { SOURCE_SHA: "" }],
|
||||
["abbreviated source SHA", { SOURCE_SHA: "a".repeat(12) }],
|
||||
["malformed source SHA", { SOURCE_SHA: "z".repeat(40) }],
|
||||
]) {
|
||||
test(`canary publisher rejects ${name} before changing any Git reference`, () => {
|
||||
withCanaryGitFixture(({ git, pinned, attach, guard }) => {
|
||||
const refs = git("show-ref");
|
||||
const result = attach(overrides);
|
||||
assert.equal(result.status, 1, result.stderr);
|
||||
assert.equal(git("show-ref"), refs);
|
||||
assert.equal(git("rev-parse", "HEAD"), pinned);
|
||||
assert.equal(guard().status, 1, "the rejected checkout must remain detached");
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
test("canary publisher rejects a different valid source SHA before changing Git references", () => {
|
||||
withCanaryGitFixture(({ git, pinned, newer, attach, guard }) => {
|
||||
const refs = git("show-ref");
|
||||
const result = attach({ SOURCE_SHA: newer });
|
||||
assert.equal(result.status, 1, result.stderr);
|
||||
assert.equal(git("show-ref"), refs);
|
||||
assert.equal(git("rev-parse", "HEAD"), pinned);
|
||||
assert.equal(guard().status, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test("the original canary release guard still rejects a non-master local branch in Actions", () => {
|
||||
withCanaryGitFixture(({ git, pinned, guard }) => {
|
||||
git("checkout", "-b", "topic", pinned);
|
||||
const refs = git("show-ref");
|
||||
assert.equal(guard().status, 1);
|
||||
assert.equal(git("symbolic-ref", "--short", "HEAD"), "topic");
|
||||
assert.equal(git("show-ref"), refs);
|
||||
});
|
||||
});
|
||||
|
||||
test("canary branch attachment keeps exact-source checkout and artifact order in its publishing lane", () => {
|
||||
const workflow = readWorkflow("release.yml");
|
||||
const publish = workflow.split(" publish_canary:\n")[1].split(" smoke_canary_onboarding:\n")[0];
|
||||
assert.equal(workflow.split("name: Attach verified canary source to local master").length, 2);
|
||||
assert.match(publish, /ref: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(publish, /name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}/);
|
||||
const steps = ["Checkout repository", "Attach verified canary source to local master", "Install dependencies", "Save canary smoke lockfile", "Restore tracked install-time changes", "Publish canary"];
|
||||
for (let i = 1; i < steps.length; i++) {
|
||||
assert.ok(publish.indexOf(`name: ${steps[i - 1]}`) < publish.indexOf(`name: ${steps[i]}`));
|
||||
}
|
||||
assert.match(publish, /run: \.\/scripts\/release\.sh canary --skip-verify/);
|
||||
assert.doesNotMatch(canaryAttachmentScript(), /git fetch|git pull|checkout (?:master|origin\/master)(?:\s|$)/);
|
||||
});
|
||||
|
||||
test("source proof requires every source check and does not wait on image publication", () => {
|
||||
const readiness = readWorkflow("cloud-readiness.yml");
|
||||
const proof = readiness.split(" source_verified:\n")[1];
|
||||
@@ -171,6 +339,99 @@ test("published canaries are gated by the exact-version onboarding browser smoke
|
||||
assert.match(releaseWorkflow, /tests\/canary-onboarding\/playwright-report/);
|
||||
});
|
||||
|
||||
test("canary smoke consumes its publisher's source-bound lockfile before frozen installation", () => {
|
||||
const workflow = readWorkflow("release.yml");
|
||||
const publish = workflow.split(" publish_canary:\n")[1].split(" smoke_canary_onboarding:\n")[0];
|
||||
const smoke = workflow.split(" smoke_canary_onboarding:\n")[1].split(" # ----- Nightly lane")[0];
|
||||
for (const job of [publish, smoke]) {
|
||||
assert.match(job, /name: Checkout repository\n\s+uses: actions\/checkout@[^\n]+\n\s+with:\n\s+ref: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(job, /name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}/);
|
||||
assert.match(job, /version: 9\.15\.4/);
|
||||
}
|
||||
assert.match(publish, /name: Save canary smoke lockfile\n\s+uses: actions\/upload-artifact@[0-9a-f]{40} # v4\n\s+with:\n\s+name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}\n\s+path: pnpm-lock\.yaml\n\s+if-no-files-found: error\n\s+overwrite: true\n\s+retention-days: 14/);
|
||||
assert.ok(publish.indexOf("name: Install dependencies") < publish.indexOf("name: Save canary smoke lockfile"));
|
||||
assert.ok(publish.indexOf("name: Save canary smoke lockfile") < publish.indexOf("name: Restore tracked install-time changes"));
|
||||
assert.ok(publish.indexOf("name: Restore tracked install-time changes") < publish.indexOf("name: Publish canary"));
|
||||
assert.match(smoke, /name: Restore canary smoke lockfile\n\s+uses: actions\/download-artifact@[0-9a-f]{40} # v4\n\s+with:\n\s+name: canary-smoke-lockfile-\$\{\{ github\.sha \}\}/);
|
||||
assert.ok(smoke.indexOf("name: Restore canary smoke lockfile") < smoke.indexOf("name: Install test dependencies"));
|
||||
assert.doesNotMatch(smoke, /run-id:|github-token:|repository:|continue-on-error|--no-frozen-lockfile/);
|
||||
});
|
||||
|
||||
for (const drift of ["patch", "manifest"]) {
|
||||
test(`a publisher lockfile lets a fresh smoke install retain validation after ${drift} drift`, (t) => {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "canary-smoke-lockfile-test-"));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const source = path.join(root, "package");
|
||||
mkdirSync(source);
|
||||
writeFileSync(path.join(source, "package.json"), JSON.stringify({ name: "smoke-fixture-dependency", version: "1.0.0" }));
|
||||
writeFileSync(path.join(source, "index.js"), 'module.exports = "original";\n');
|
||||
const pack = spawnSync("npm", ["pack", "--ignore-scripts", "--pack-destination", root], { cwd: source, encoding: "utf8" });
|
||||
assert.equal(pack.status, 0, pack.stderr);
|
||||
const manifest = {
|
||||
name: "canary-smoke-fixture", private: true,
|
||||
packageManager: "pnpm@9.15.4",
|
||||
dependencies: { "smoke-fixture-dependency": "file:smoke-fixture-dependency-1.0.0.tgz" },
|
||||
pnpm: { patchedDependencies: { "smoke-fixture-dependency@1.0.0": "fixture.patch" } },
|
||||
};
|
||||
writeFileSync(path.join(root, "package.json"), JSON.stringify(manifest));
|
||||
writeFileSync(path.join(root, "pnpm-workspace.yaml"), "packages: []\n");
|
||||
const patch = (value) => writeFileSync(path.join(root, "fixture.patch"), [
|
||||
"diff --git a/index.js b/index.js", "index 1111111..2222222 100644",
|
||||
"--- a/index.js", "+++ b/index.js", "@@ -1 +1 @@",
|
||||
'-module.exports = "original";', `+module.exports = "${value}";`, "",
|
||||
].join("\n"));
|
||||
const workflow = readWorkflow("release.yml");
|
||||
const publish = workflow.split(" publish_canary:\n")[1].split(" smoke_canary_onboarding:\n")[0];
|
||||
const smoke = workflow.split(" smoke_canary_onboarding:\n")[1].split(" # ----- Nightly lane")[0];
|
||||
const installArgs = (name) => {
|
||||
const job = name === "Install dependencies" ? publish : smoke;
|
||||
const command = job.match(new RegExp(`name: ${name}\\n\\s+run: (pnpm install[^\\n]+)`))?.[1];
|
||||
assert.ok(command, `missing ${name}`);
|
||||
return [...command.split(" ").slice(1), "--offline", "--ignore-scripts"];
|
||||
};
|
||||
const install = (args) => spawnSync("pnpm", args, {
|
||||
cwd: root, encoding: "utf8", env: { ...process.env, CI: "true" },
|
||||
});
|
||||
const success = (result) => assert.equal(result.status, 0, `${result.stdout}\n${result.stderr}`);
|
||||
patch(drift === "patch" ? "old-patch" : "published-patch");
|
||||
success(install(installArgs("Install dependencies")));
|
||||
const staleLock = readFileSync(path.join(root, "pnpm-lock.yaml"));
|
||||
if (drift === "patch") {
|
||||
patch("published-patch");
|
||||
} else {
|
||||
copyFileSync(path.join(root, "smoke-fixture-dependency-1.0.0.tgz"), path.join(root, "updated-dependency.tgz"));
|
||||
manifest.dependencies["smoke-fixture-dependency"] = "file:updated-dependency.tgz";
|
||||
writeFileSync(path.join(root, "package.json"), JSON.stringify(manifest));
|
||||
}
|
||||
const before = install(installArgs("Install test dependencies"));
|
||||
assert.notEqual(before.status, 0);
|
||||
assert.match(before.stdout + before.stderr, drift === "patch" ? /ERR_PNPM_LOCKFILE_CONFIG_MISMATCH/ : /ERR_PNPM_OUTDATED_LOCKFILE/);
|
||||
|
||||
// Capture the publisher's resolution, then reproduce its tracked-file restore
|
||||
// and a fresh smoke runner with no installed workspace dependencies.
|
||||
success(install(installArgs("Install dependencies")));
|
||||
const artifact = path.join(root, "publisher-lock.yaml");
|
||||
if (publish.includes("name: Save canary smoke lockfile")) {
|
||||
copyFileSync(path.join(root, "pnpm-lock.yaml"), artifact);
|
||||
}
|
||||
writeFileSync(path.join(root, "pnpm-lock.yaml"), staleLock);
|
||||
rmSync(path.join(root, "node_modules"), { recursive: true, force: true });
|
||||
if (smoke.includes("name: Restore canary smoke lockfile")) {
|
||||
copyFileSync(artifact, path.join(root, "pnpm-lock.yaml"));
|
||||
}
|
||||
success(install(installArgs("Install test dependencies")));
|
||||
assert.equal(readFileSync(path.join(root, "node_modules/smoke-fixture-dependency/index.js"), "utf8"), 'module.exports = "published-patch";\n');
|
||||
assert.deepEqual(readFileSync(path.join(root, "pnpm-lock.yaml")), readFileSync(artifact));
|
||||
|
||||
// A lockfile from the wrong source still fails closed; restoring an artifact
|
||||
// must not disable hash validation or resolve another version in the smoke job.
|
||||
patch("different-source");
|
||||
const wrongSource = install(installArgs("Install test dependencies"));
|
||||
assert.notEqual(wrongSource.status, 0);
|
||||
assert.match(wrongSource.stdout + wrongSource.stderr, /ERR_PNPM_LOCKFILE_CONFIG_MISMATCH/);
|
||||
});
|
||||
}
|
||||
|
||||
test("every lane's tag push degrades to recovery instructions when rejected", () => {
|
||||
const releaseWorkflow = readWorkflow("release.yml");
|
||||
|
||||
@@ -245,7 +506,7 @@ test("release verify workflow covers the same split test surface as stable PR ve
|
||||
assert.match(buildJob, /persist-credentials: false/);
|
||||
assert.doesNotMatch(buildJob, /cache: pnpm/);
|
||||
|
||||
for (const group of ["general-server-without-chat-or-native-runner", "general-chat", "general-workspaces-a", "general-workspaces-b"]) {
|
||||
for (const group of ["general-server-without-chat-or-native-runner", "general-chat", "general-workspaces-a", "general-workspaces-b", "general-workspaces-c"]) {
|
||||
assert.match(verifyWorkflow, new RegExp(`group: ${group}`));
|
||||
}
|
||||
for (const [group, count] of [["general-server-without-chat-or-native-runner", 10], ["general-chat", 3]]) {
|
||||
@@ -563,3 +824,418 @@ test("historical promotion sources keep their packaging path while new sources r
|
||||
assert.match(block, /Download exact-source release daemon receipts\n\s+if: needs.release_runner_source.outputs.required == 'true'/);
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
|
||||
test("canary startup refreshes an incomplete dependency publication, then onboards once", async () => {
|
||||
const result = await canaryStartup({ mode: "recover" });
|
||||
assert.equal(result.code, 0, result.output);
|
||||
const installs = result.calls.filter(call => call.kind === "npm");
|
||||
const onboarding = result.calls.filter(call => call.kind === "onboard");
|
||||
assert.equal(installs.length, 2);
|
||||
assert.equal(onboarding.length, 1);
|
||||
for (const call of installs) {
|
||||
assert.equal(call.args[0], "install");
|
||||
assert.ok(call.args.includes("paperclipai@2026.1009.0-canary.1"));
|
||||
assert.ok(call.args.includes("--no-package-lock"));
|
||||
assert.ok(call.args.includes("--no-save"));
|
||||
}
|
||||
assert.ok(!installs[0].args.includes("--prefer-online"));
|
||||
assert.ok(installs[1].args.includes("--prefer-online"));
|
||||
assert.notEqual(installs[0].args[2], installs[1].args[2]);
|
||||
assert.equal(installs[0].cache, installs[1].cache);
|
||||
assert.deepEqual(onboarding[0].args.slice(0, 3), ["onboard", "--yes", "--data-dir"]);
|
||||
});
|
||||
|
||||
test("permanent missing versions stop after three attempts without onboarding", async () => {
|
||||
const result = await canaryStartup({ mode: "permanent" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter(call => call.kind === "npm").length, 3);
|
||||
assert.equal(result.calls.filter(call => call.kind === "onboard").length, 0);
|
||||
assert.match(result.output, /installation failed.*onboarding was not started/);
|
||||
});
|
||||
|
||||
test("npm failures other than ETARGET fail immediately without onboarding", async () => {
|
||||
const result = await canaryStartup({ mode: "auth" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.length, 1);
|
||||
assert.match(result.output, /E401/);
|
||||
});
|
||||
|
||||
test("onboarding failures are never retried, even when they mention ETARGET", async () => {
|
||||
const result = await canaryStartup({ mode: "success", onboard: "fail" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter(call => call.kind === "npm").length, 1);
|
||||
assert.equal(result.calls.filter(call => call.kind === "onboard").length, 1);
|
||||
assert.match(result.output, /onboarding failed \(exit 17\)/);
|
||||
});
|
||||
|
||||
test("a hanging npm install is stopped within the shared acquisition budget", async () => {
|
||||
const result = await canaryStartup({ mode: "hang", budget: 1000 });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter(call => call.kind === "npm").length, 1);
|
||||
assert.equal(result.calls.filter(call => call.kind === "onboard").length, 0);
|
||||
assert.ok(result.elapsed < 4000, `install exceeded bounded cancellation: ${result.elapsed}ms`);
|
||||
assert.match(result.output, /exceeded its startup budget/);
|
||||
});
|
||||
|
||||
test("backoff consumes the shared budget and does not begin another install after expiry", async () => {
|
||||
const result = await canaryStartup({ mode: "permanent", budget: 1000, delay: 5000 });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.length, 1);
|
||||
assert.match(result.output, /exceeded its startup budget/);
|
||||
});
|
||||
|
||||
|
||||
test("Playwright cancellation stops an onboarding process without replay", async () => {
|
||||
const result = await canaryStartup({ mode: "success", onboard: "hang", cancelAtKind: "onboard" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter(call => call.kind === "npm").length, 1);
|
||||
const onboarding = result.calls.filter(call => call.kind === "onboard");
|
||||
assert.equal(onboarding.length, 1);
|
||||
assert.throws(() => process.kill(onboarding[0].pid, 0), {code:"ESRCH"});
|
||||
assert.match(result.output, /stopped by SIGTERM/);
|
||||
assert.ok(result.elapsed < 2500, `onboarding cancellation took ${result.elapsed}ms`);
|
||||
});
|
||||
|
||||
test("canary install retries retain the existing overall Playwright startup deadline", async () => {
|
||||
const { INSTALL_BUDGET_MS } = await import("../../tests/canary-onboarding/start-published-canary.mjs");
|
||||
const config = readFileSync(path.join(repoRoot, "tests/canary-onboarding/playwright.config.ts"), "utf8");
|
||||
assert.equal(INSTALL_BUDGET_MS, 120_000);
|
||||
assert.match(config, /timeout: 300_000/);
|
||||
assert.match(config, /start-published-canary\.mjs/);
|
||||
assert.match(config, /reuseExistingServer: false/);
|
||||
});
|
||||
|
||||
|
||||
test("nested ETARGET output does not retry a final authentication or network failure", async () => {
|
||||
for (const mode of ["mixed", "network"]) {
|
||||
const result = await canaryStartup({ mode });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.length, 1);
|
||||
}
|
||||
});
|
||||
|
||||
const exactCanary = "2026.1009.0-canary.1";
|
||||
const canaryTarball = `https://registry.npmjs.org/@paperclipai/server/-/server-${exactCanary}.tgz`;
|
||||
const tarballFailure = (url = canaryTarball, prefix = "npm error") => `${prefix} code E404\n${prefix} 404 Not Found - GET ${url} - Not found\n${prefix} 404\n${prefix} 404 The requested resource '@paperclipai/server@${url}' could not be found or you do not have permission to access it.\n`;
|
||||
|
||||
function tarballProof({ config = {}, manifest = {}, status = 200, redirected = false, responseUrl, rawBody } = {}) {
|
||||
const calls = [];
|
||||
const dist = { tarball: canaryTarball, integrity: `sha512-${Buffer.alloc(64).toString("base64")}`, ...manifest.dist };
|
||||
const data = { name: "@paperclipai/server", version: exactCanary, ...manifest, dist };
|
||||
return { calls, options: {
|
||||
signal: new AbortController().signal, prefix: "/fixture/install", env: {},
|
||||
runConfig: async (command, args, options) => {
|
||||
calls.push({ kind: "config", command, args, options });
|
||||
return { code: 0, stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=undefined\n", ...config };
|
||||
},
|
||||
fetchMetadata: async (url, options) => {
|
||||
calls.push({ kind: "metadata", url, options });
|
||||
return { status, redirected, url: responseUrl ?? url, body: new Response(rawBody ?? JSON.stringify(data)).body };
|
||||
},
|
||||
} };
|
||||
}
|
||||
|
||||
test("exact published tarball proof uses quiet effective config and fresh public metadata without credentials", async () => {
|
||||
for (const prefix of ["npm error", "npm ERR!"]) {
|
||||
const fixture = tarballProof();
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball, prefix), exactCanary, fixture.options), true);
|
||||
const [config, metadata] = fixture.calls;
|
||||
assert.deepEqual(config.args, ["config", "get", "registry", "@paperclipai:registry", "--prefix", "/fixture/install"]);
|
||||
assert.equal(config.options.env, fixture.options.env);
|
||||
assert.equal(config.options.quiet, true);
|
||||
assert.equal(config.options.captureStdout, true);
|
||||
assert.equal(config.options.captureStderr, true);
|
||||
assert.equal(config.options.outputLimit, 4096);
|
||||
assert.equal(metadata.url, `https://registry.npmjs.org/%40paperclipai%2Fserver/${exactCanary}`);
|
||||
assert.equal(metadata.options.signal, config.options.signal);
|
||||
assert.equal(metadata.options.redirect, "error");
|
||||
assert.equal(metadata.options.credentials, "omit");
|
||||
assert.deepEqual(metadata.options.headers, { accept: "application/json", "cache-control": "no-cache" });
|
||||
}
|
||||
});
|
||||
|
||||
test("mixed npm errors and noncanonical or unreleased tarballs never read config or metadata", async () => {
|
||||
const errors = [
|
||||
"npm error code ETARGET\n", "npm error code E401\n", "npm error code ECONNRESET\n",
|
||||
`${tarballFailure()}npm error code E401\n`, `${tarballFailure()}npm error code 1\n`,
|
||||
`npm error code ENEEDAUTH\n${tarballFailure()}`, `${tarballFailure()}${tarballFailure()}`,
|
||||
`${tarballFailure()}npm error 404 Not Found - GET https://registry.example.invalid/a.tgz - Not found\n`,
|
||||
`${tarballFailure()}npm error 404 The requested resource 'other@${canaryTarball}' could not be found\n`,
|
||||
tarballFailure().replace(" - GET ", " - PUT "),
|
||||
...[
|
||||
canaryTarball + "?token=fixture", canaryTarball + "#fragment", canaryTarball.replace("npmjs.org/", "npmjs.org:443/"),
|
||||
canaryTarball.replace("npmjs.org/", "npmjs.org.example.invalid/"), canaryTarball.replace("npmjs.org/", "fixture@npmjs.org/"),
|
||||
canaryTarball.replace("https:", "http:"), canaryTarball.replace("@paperclipai/server", "%40paperclipai%2Fserver"),
|
||||
canaryTarball.replace("@paperclipai/server", "@paperclipai/../server"), canaryTarball.replace(exactCanary, "2026.1009.0-canary.2"),
|
||||
"https://registry.npmjs.org/@paperclipai/not-released/-/not-released-2026.1009.0-canary.1.tgz",
|
||||
"https://registry.npmjs.org/@paperclipai/plugin-createos/-/plugin-createos-2026.1009.0-canary.1.tgz",
|
||||
"https://registry.npmjs.org/%40paperclipai%2Fserver/2026.1009.0-canary.1",
|
||||
].map((url) => tarballFailure(url)),
|
||||
];
|
||||
for (const error of errors) {
|
||||
const fixture = tarballProof();
|
||||
assert.equal(await canRetryPublishedTarball404(error, exactCanary, fixture.options), false, error);
|
||||
assert.deepEqual(fixture.calls, [], error);
|
||||
}
|
||||
for (const version of ["2026.1009.0", "2026.1009.0-nightly.1", "2026.1009.0-beta.1", "2026.1009.0-canary.1+build"]) {
|
||||
const fixture = tarballProof();
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball.replace(exactCanary, version)), version, fixture.options), false);
|
||||
assert.deepEqual(fixture.calls, []);
|
||||
}
|
||||
});
|
||||
|
||||
test("effective registry authority respects scope precedence and rejects unknown or private config", async () => {
|
||||
const configs = [
|
||||
[{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true],
|
||||
[{ stdout: "registry=undefined\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true],
|
||||
[{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=https://registry.example.invalid/\n" }, false],
|
||||
[{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=undefined\n" }, false],
|
||||
[{ stdout: "registry=undefined\n@paperclipai:registry=undefined\n" }, false],
|
||||
[{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=\n" }, false],
|
||||
[{ stdout: "registry=https://fixture:secret@registry.npmjs.org/\n@paperclipai:registry=undefined\n" }, false],
|
||||
[{ stdout: "unexpected output\n" }, false], [{ code: 1 }, false], [{ signal: "SIGTERM" }, false], [{ outputTruncated: true }, false],
|
||||
];
|
||||
for (const [config, expected] of configs) {
|
||||
const fixture = tarballProof({ config });
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), expected);
|
||||
assert.equal(fixture.calls.filter((call) => call.kind === "metadata").length, expected ? 1 : 0);
|
||||
}
|
||||
});
|
||||
|
||||
test("metadata must identify the exact published artifact with valid SHA512 syntax, not merely exist", async () => {
|
||||
const fixtures = [
|
||||
{ status: 404 }, { status: 401 }, { status: 403 }, { status: 302 }, { redirected: true },
|
||||
{ responseUrl: "https://registry.example.invalid/metadata" }, { rawBody: "not json" },
|
||||
{ manifest: { name: "@paperclipai/shared" } }, { manifest: { version: "2026.1009.0-canary.2" } },
|
||||
{ manifest: { dist: { tarball: canaryTarball + "?query=1" } } },
|
||||
...[undefined, "", "sha512-A===", `sha256-${Buffer.alloc(32).toString("base64")}`, `sha512-${Buffer.alloc(64).toString("base64")} extra`]
|
||||
.map((integrity) => ({ manifest: { dist: { integrity } } })),
|
||||
{ rawBody: JSON.stringify({ padding: "x".repeat(65 * 1024) }) },
|
||||
];
|
||||
for (const input of fixtures) {
|
||||
const fixture = tarballProof(input);
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false, JSON.stringify(input).slice(0, 200));
|
||||
}
|
||||
});
|
||||
|
||||
test("rejected and oversized native response bodies cancel without waiting for slow or rejected cleanup", async () => {
|
||||
for (const [status, pending] of [[200, true], [200, false], [404, true], [404, false]]) {
|
||||
let cancelled = false;
|
||||
const fixture = tarballProof();
|
||||
fixture.options.fetchMetadata = async (url) => ({ status, redirected: false, url, body: new Response(new ReadableStream({
|
||||
start(controller) { controller.enqueue(new Uint8Array(65 * 1024)); },
|
||||
cancel() { cancelled = true; return pending ? new Promise(() => {}) : Promise.reject(new Error("fixture rejected cancellation")); },
|
||||
})).body });
|
||||
const started = Date.now();
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false);
|
||||
assert.equal(cancelled, true);
|
||||
assert.ok(Date.now() - started < 1000, "cancellation acknowledgement must not extend a rejection");
|
||||
}
|
||||
});
|
||||
|
||||
test("aborting a stalled metadata read cancels its reader without waiting for acknowledgement", { timeout: 2000 }, async () => {
|
||||
let cancelled = false;
|
||||
const fixture = tarballProof();
|
||||
fixture.options.fetchMetadata = async (url) => ({ status: 200, redirected: false, url,
|
||||
body: new Response(new ReadableStream({
|
||||
cancel() { cancelled = true; return new Promise(() => {}); },
|
||||
})).body });
|
||||
const budget = new AbortController();
|
||||
const timer = setTimeout(() => budget.abort(new Error("fixture body budget expired")), 100);
|
||||
const started = Date.now();
|
||||
try {
|
||||
await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, {
|
||||
...fixture.options, signal: budget.signal,
|
||||
}), /fixture body budget expired/);
|
||||
assert.equal(cancelled, true);
|
||||
assert.ok(Date.now() - started < 1000);
|
||||
} finally { clearTimeout(timer); }
|
||||
});
|
||||
|
||||
test("metadata headers cannot outlive the budget and late response bodies are cancelled", { timeout: 2000 }, async () => {
|
||||
let headersArrive;
|
||||
let cancelled = false;
|
||||
const fixture = tarballProof();
|
||||
fixture.options.fetchMetadata = (url) => new Promise((resolve) => {
|
||||
headersArrive = () => resolve({ status: 200, redirected: false, url,
|
||||
body: new Response(new ReadableStream({ cancel() { cancelled = true; } })).body });
|
||||
});
|
||||
const budget = new AbortController();
|
||||
const timer = setTimeout(() => budget.abort(new Error("fixture headers budget expired")), 100);
|
||||
const started = Date.now();
|
||||
try {
|
||||
await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, {
|
||||
...fixture.options, signal: budget.signal,
|
||||
}), /fixture headers budget expired/);
|
||||
assert.ok(Date.now() - started < 1000);
|
||||
headersArrive();
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
assert.equal(cancelled, true);
|
||||
} finally { clearTimeout(timer); }
|
||||
});
|
||||
|
||||
test("real npm effective registry reads isolated prefix, user, and environment configuration", async () => {
|
||||
const root = mkdtempSync(path.join(os.tmpdir(), "canary-config-proof-"));
|
||||
mkdirSync(path.join(root, "prefix"));
|
||||
writeFileSync(path.join(root, "global.npmrc"), "");
|
||||
const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toLowerCase().startsWith("npm_config_")));
|
||||
env.npm_config_userconfig = path.join(root, "user.npmrc");
|
||||
env.npm_config_globalconfig = path.join(root, "global.npmrc");
|
||||
try {
|
||||
for (const [user, project, override, expected] of [
|
||||
["registry=https://registry.npmjs.org/\n", "", undefined, true],
|
||||
["registry=https://registry.example.invalid/\n", "@paperclipai:registry=https://registry.npmjs.org/\n", undefined, true],
|
||||
["registry=https://registry.npmjs.org/\n", "@paperclipai:registry=https://registry.example.invalid/\n", undefined, false],
|
||||
["registry=https://registry.npmjs.org/\n", "", "https://registry.example.invalid/", false],
|
||||
]) {
|
||||
writeFileSync(path.join(root, "user.npmrc"), user);
|
||||
writeFileSync(path.join(root, "prefix", ".npmrc"), project);
|
||||
const fixture = tarballProof();
|
||||
delete fixture.options.runConfig;
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, {
|
||||
...fixture.options, prefix: path.join(root, "prefix"), env: { ...env, ...(override ? { npm_config_registry: override } : {}) },
|
||||
}), expected);
|
||||
assert.equal(fixture.calls.length, expected ? 1 : 0);
|
||||
}
|
||||
} finally { rmSync(root, { recursive: true, force: true }); }
|
||||
});
|
||||
|
||||
async function stalledMetadata(run) {
|
||||
let closed = false;
|
||||
let closing = false;
|
||||
let observeClose;
|
||||
const connectionClosed = new Promise((resolve) => { observeClose = resolve; });
|
||||
const server = createServer((_request, response) => {
|
||||
response.on("close", () => { closed = true; observeClose(); });
|
||||
response.writeHead(200, { "content-type": "application/json" });
|
||||
response.write('{"name":');
|
||||
});
|
||||
// An HTTP client can replace an aborted pooled connection while shutdown
|
||||
// starts. Own late connections too, so server.close cannot wait on them.
|
||||
server.on("connection", (socket) => { if (closing) socket.destroy(); });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const fixture = tarballProof();
|
||||
fixture.options.fetchMetadata = async (url, options) => {
|
||||
const response = await fetch(`http://127.0.0.1:${server.address().port}/metadata`, options);
|
||||
Object.defineProperty(response, "url", { value: url });
|
||||
return response;
|
||||
};
|
||||
let closeDeadline;
|
||||
try {
|
||||
await run(fixture.options);
|
||||
await Promise.race([connectionClosed, new Promise((_, reject) => {
|
||||
closeDeadline = setTimeout(() => reject(new Error("metadata connection outlived cancellation")), 1000);
|
||||
})]);
|
||||
assert.equal(closed, true, "metadata connection must close before fixture-forced cleanup");
|
||||
}
|
||||
finally {
|
||||
clearTimeout(closeDeadline);
|
||||
closing = true;
|
||||
await new Promise((resolve) => {
|
||||
server.close(resolve);
|
||||
server.closeAllConnections();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
test("shared acquisition cancellation aborts the metadata body before the lookup deadline", async () => {
|
||||
await stalledMetadata(async (options) => {
|
||||
const budget = new AbortController();
|
||||
const timer = setTimeout(() => budget.abort(new Error("fixture shared budget expired")), 250);
|
||||
try {
|
||||
await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, { ...options, signal: budget.signal }), /fixture shared budget expired/);
|
||||
} finally { clearTimeout(timer); }
|
||||
});
|
||||
});
|
||||
|
||||
test("the short config and metadata deadline also covers a stalled real HTTP body", { timeout: 10_000 }, async () => {
|
||||
assert.equal(TARBALL_METADATA_TIMEOUT_MS, 5000);
|
||||
const started = Date.now();
|
||||
await stalledMetadata(async (options) => {
|
||||
assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, options), false);
|
||||
});
|
||||
assert.ok(Date.now() - started < 7000, "metadata must not consume the 120s acquisition budget");
|
||||
});
|
||||
|
||||
test("a verified canary tarball404 retries acquisition and runs onboarding once", async () => {
|
||||
const result = await canaryStartup({ mode: "tarball-recover" });
|
||||
assert.equal(result.code, 0, result.output);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "config").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1);
|
||||
assert.doesNotMatch(result.output, /fixture config diagnostic/);
|
||||
});
|
||||
|
||||
test("persistent promised tarball404s stop at three installs without a final lookup or onboarding", async () => {
|
||||
const result = await canaryStartup({ mode: "tarball-permanent" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 3);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "config").length, 2);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 2);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
|
||||
assert.match(result.output, /installation failed.*onboarding was not started/);
|
||||
});
|
||||
|
||||
test("private or oversized config and unpublished metadata404 fail without retry or leaking config", async () => {
|
||||
for (const input of [{ config: "private" }, { config: "large" }, { metadata: "missing" }]) {
|
||||
const result = await canaryStartup({ mode: "tarball-permanent", ...input });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
|
||||
assert.doesNotMatch(result.output, /fixture-user|fixture-password|fixture config diagnostic/);
|
||||
}
|
||||
});
|
||||
|
||||
test("truncated npm output cannot hide an earlier auth failure and qualify a tarball retry", async () => {
|
||||
const result = await canaryStartup({ mode: "tarball-large" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "config").length, 0);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
|
||||
});
|
||||
|
||||
test("tarball recovery does not replay onboarding failures or change ETARGET for other exact versions", async () => {
|
||||
const failed = await canaryStartup({ mode: "tarball-recover", onboard: "fail" });
|
||||
assert.equal(failed.code, 1);
|
||||
assert.equal(failed.calls.filter((call) => call.kind === "npm").length, 2);
|
||||
assert.equal(failed.calls.filter((call) => call.kind === "onboard").length, 1);
|
||||
assert.match(failed.output, /onboarding failed \(exit 17\)/);
|
||||
const other = await canaryStartup({ mode: "recover", version: "2026.1009.0-nightly.1" });
|
||||
assert.equal(other.code, 0, other.output);
|
||||
assert.equal(other.calls.filter((call) => call.kind === "npm").length, 2);
|
||||
assert.equal(other.calls.filter((call) => call.kind === "onboard").length, 1);
|
||||
assert.equal(other.calls.filter((call) => call.kind === "config").length, 0);
|
||||
});
|
||||
|
||||
test("existing ETARGET character-tail behavior survives long multibyte npm diagnostics", async () => {
|
||||
const result = await canaryStartup({ mode: "multibyte-recover" });
|
||||
assert.equal(result.code, 0, result.output);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "config").length, 0);
|
||||
});
|
||||
|
||||
test("metadata lookup and backoff both consume the unchanged shared acquisition budget", async () => {
|
||||
for (const input of [{ metadata: "hang" }, { delay: 5000 }]) {
|
||||
const result = await canaryStartup({ mode: "tarball-permanent", budget: 1000, ...input });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
|
||||
assert.match(result.output, /exceeded its startup budget/);
|
||||
assert.ok(result.elapsed < 4000, `shared-budget cancellation took ${result.elapsed}ms`);
|
||||
}
|
||||
});
|
||||
|
||||
test("cancellation reaps a config process and its descendant that ignore SIGTERM", async () => {
|
||||
const result = await canaryStartup({ mode: "tarball-permanent", config: "hang", cancelAtKind: "config-descendant" });
|
||||
assert.equal(result.code, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1);
|
||||
assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0);
|
||||
for (const child of result.calls.filter((call) => ["config", "config-descendant"].includes(call.kind))) {
|
||||
assert.throws(() => process.kill(child.pid, 0), { code: "ESRCH" });
|
||||
}
|
||||
assert.match(result.output, /stopped by SIGTERM/);
|
||||
});
|
||||
@@ -107,10 +107,12 @@ test("general shards retain scripts and route-named suites outside the serialize
|
||||
]) assert.ok(general.selectedGeneralServerSuites.includes(file), `missing configured server suite: ${file}`);
|
||||
});
|
||||
|
||||
test("shard flags are rejected for the workspaces-b group", () => {
|
||||
const result = dryRun(["--mode", "general", "--group", "general-workspaces-b", "--shard-index", "0", "--shard-count", "3"]);
|
||||
assert.notEqual(result.status, 0, "workspaces-b must not accept shard flags");
|
||||
});
|
||||
for (const group of ["general-workspaces-b", "general-workspaces-c"]) {
|
||||
test(`shard flags are rejected for the ${group} group`, () => {
|
||||
const result = dryRun(["--mode", "general", "--group", group, "--shard-index", "0", "--shard-count", "3"]);
|
||||
assert.notEqual(result.status, 0, `${group} must not accept shard flags`);
|
||||
});
|
||||
}
|
||||
|
||||
test("workspace lanes cover every non-server project in the root Vitest configuration", () => {
|
||||
const config = readFileSync(path.join(repoRoot, "vitest.config.ts"), "utf8");
|
||||
@@ -118,11 +120,81 @@ test("workspace lanes cover every non-server project in the root Vitest configur
|
||||
assert.ok(roots.includes("server"), "expected the explicit root Vitest project list");
|
||||
const expected = roots.filter(root => root !== "server")
|
||||
.map(root => JSON.parse(readFileSync(path.join(repoRoot, root, "package.json"), "utf8")).name).sort();
|
||||
const actual = ["general-workspaces-a", "general-workspaces-b"]
|
||||
const actual = ["general-workspaces-a", "general-workspaces-b", "general-workspaces-c"]
|
||||
.flatMap(group => dryRunJson(["--mode", "general", "--group", group]).workspaceProjects).sort();
|
||||
assert.deepEqual(actual, expected, "no configured project may be silently omitted or run twice");
|
||||
});
|
||||
|
||||
// Collects every `- group:` matrix entry in a workflow with the keys it sets.
|
||||
// Comment and blank lines are dropped first, and an entry ends only when the
|
||||
// indentation returns to the item level, so a comment between two keys cannot
|
||||
// hide a shard setting from the assertions below.
|
||||
function matrixEntries(text) {
|
||||
const entries = [];
|
||||
let current = null;
|
||||
for (const line of text.split("\n")) {
|
||||
if (/^\s*(#|$)/.test(line)) continue;
|
||||
const indent = line.match(/^ */)[0].length;
|
||||
const item = line.match(/^ *- group: (\S+)/);
|
||||
if (item) {
|
||||
current = { group: item[1], indent, keys: ["group"] };
|
||||
entries.push(current);
|
||||
continue;
|
||||
}
|
||||
if (!current) continue;
|
||||
if (indent <= current.indent) { current = null; continue; }
|
||||
const key = line.match(/^ *([A-Za-z_]+):/);
|
||||
if (key) current.keys.push(key[1]);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
test("matrixEntries sees shard keys that sit behind comments or later in the entry", () => {
|
||||
const entries = matrixEntries([
|
||||
" include:",
|
||||
" - group: general-workspaces-b",
|
||||
" group_label: workspaces-b",
|
||||
" # a comment between keys must not hide the shard setting",
|
||||
" shard_index: 0",
|
||||
"",
|
||||
" shard_count: 3",
|
||||
" - group: general-workspaces-c",
|
||||
" group_label: workspaces-c",
|
||||
" steps:",
|
||||
" - name: unrelated",
|
||||
" shard_index: 9",
|
||||
].join("\n"));
|
||||
assert.deepEqual(entries.map(entry => entry.group), ["general-workspaces-b", "general-workspaces-c"]);
|
||||
assert.deepEqual(entries[0].keys, ["group", "group_label", "shard_index", "shard_count"]);
|
||||
assert.deepEqual(entries[1].keys, ["group", "group_label"], "keys after the matrix must not leak into the last entry");
|
||||
});
|
||||
|
||||
test("workspaces-c runs the db project alone and workspaces-b no longer carries it", () => {
|
||||
// The db project was 168-250s of the 436-513s workspaces-b vitest time
|
||||
// (PR runs 38084171761, 38083084276 and 38083020834, 2026-10-10), so it
|
||||
// runs in its own lane. The PR and release matrices both carry that lane.
|
||||
const workspacesC = dryRunJson(["--mode", "general", "--group", "general-workspaces-c"]);
|
||||
assert.deepEqual(workspacesC.workspaceProjects, ["@paperclipai/db"]);
|
||||
assert.equal(workspacesC.workspacesVitestShard, null);
|
||||
const workspacesB = dryRunJson(["--mode", "general", "--group", "general-workspaces-b"]);
|
||||
assert.ok(!workspacesB.workspaceProjects.includes("@paperclipai/db"), "db must not run twice");
|
||||
assert.ok(workspacesB.workspaceProjects.length > 0, "workspaces-b must still run its other projects");
|
||||
for (const workflow of ["pr-trusted.yml", "release-verify.yml"]) {
|
||||
const entries = matrixEntries(readFileSync(path.join(repoRoot, ".github", "workflows", workflow), "utf8"));
|
||||
// The parser must demonstrably see shard keys in this workflow, otherwise
|
||||
// the unsharded assertions below could pass for the wrong reason.
|
||||
const sharded = entries.filter(entry => entry.group === "general-workspaces-a");
|
||||
assert.ok(sharded.length > 0 && sharded.every(entry => entry.keys.includes("shard_index") && entry.keys.includes("shard_count")),
|
||||
`${workflow} must carry the sharded workspaces-a entries`);
|
||||
for (const group of ["general-workspaces-b", "general-workspaces-c"]) {
|
||||
const matching = entries.filter(entry => entry.group === group);
|
||||
assert.equal(matching.length, 1, `${workflow} must run ${group} exactly once`);
|
||||
assert.ok(matching[0].keys.includes("group_label"), `${workflow} must label the ${group} job`);
|
||||
assert.deepEqual(matching[0].keys.filter(key => key.startsWith("shard_")), [], `${workflow} must run ${group} unsharded`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("workspaces-a shards map to Vitest native --shard slices over a stable project list", () => {
|
||||
const shards = [0, 1].map((index) =>
|
||||
dryRunJson([
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import test from 'node:test';
|
||||
import { stopPrefetches } from '../prefetch-child.mjs';
|
||||
import { PREPARE_BUNDLED_PACKAGE_SCRIPT, bundledInstallTimings, bundledPackageSlug, startBundledInstalls } from '../stage-bundled-packages.mjs';
|
||||
|
||||
// A small local script stands in for `prepare-bundled-package.mjs`; no test
|
||||
// here needs npm or a network.
|
||||
const withScratch = async body => {
|
||||
const root = mkdtempSync(join(tmpdir(), 'stage-bundled-packages-'));
|
||||
try { await body(root); } finally { rmSync(root, { recursive: true, force: true }); }
|
||||
};
|
||||
const entriesFor = (root, names) => names.map((name, index) => ({ name, stagedSource: join(root, `source-${index}`), target: join(root, `package-${index}`) }));
|
||||
const fakeScript = (root, body) => { const script = join(root, 'fake-prepare.mjs'); writeFileSync(script, body); return script; };
|
||||
|
||||
test('package names map to stable log and timing slugs', () => {
|
||||
assert.equal(bundledPackageSlug('@paperclipai/server'), 'server');
|
||||
assert.equal(bundledPackageSlug('@paperclipai/adapter-utils'), 'adapter_utils');
|
||||
assert.equal(bundledPackageSlug('@paperclipai/db'), 'db');
|
||||
});
|
||||
|
||||
test('the children run the CLI entry release.sh uses, never an in-process call', () => {
|
||||
assert.ok(existsSync(PREPARE_BUNDLED_PACKAGE_SCRIPT));
|
||||
const source = readFileSync(PREPARE_BUNDLED_PACKAGE_SCRIPT, 'utf8');
|
||||
assert.ok(source.includes('if (process.argv[1] === fileURLToPath(import.meta.url))'));
|
||||
assert.ok(source.includes('prepareBundledPackage(resolve(sourceDir), resolve(destinationDir))'));
|
||||
const release = readFileSync(new URL('../release.sh', import.meta.url), 'utf8');
|
||||
assert.match(release, /node "\$REPO_ROOT\/scripts\/prepare-bundled-package\.mjs" "\$REPO_ROOT\/\$pkg_dir" "\$publish_dir"/);
|
||||
});
|
||||
|
||||
test('the installs run at the same time, each with its own source and target', async () => withScratch(async root => {
|
||||
// Each child records its arguments, announces itself, and exits only once
|
||||
// the other child has announced itself too. Serial children would deadlock
|
||||
// here and the five-second guard would turn that into a failure.
|
||||
const script = fakeScript(root, `
|
||||
import { existsSync, writeFileSync } from 'node:fs';
|
||||
import { basename, join } from 'node:path';
|
||||
const [source, target] = process.argv.slice(2);
|
||||
const root = join(target, '..');
|
||||
writeFileSync(join(root, 'started-' + basename(target)), '');
|
||||
console.log('install ' + source + ' -> ' + target);
|
||||
const other = basename(target) === 'package-0' ? 'package-1' : 'package-0';
|
||||
const deadline = Date.now() + 5000;
|
||||
while (!existsSync(join(root, 'started-' + other))) {
|
||||
if (Date.now() > deadline) { console.error('the other install never started'); process.exit(3); }
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
|
||||
}
|
||||
writeFileSync(join(target, 'package.json'), '{}');
|
||||
`);
|
||||
const entries = entriesFor(root, ['@paperclipai/server', '@paperclipai/db']);
|
||||
for (const { target } of entries) mkdirSync(target);
|
||||
const installs = startBundledInstalls(entries, { logDir: root, cwd: root, script });
|
||||
assert.deepEqual(installs.map(({ slug }) => slug), ['server', 'db']);
|
||||
await Promise.all(installs.map(({ handle }) => handle.done));
|
||||
for (const [index, { stagedSource, target, slug, handle }] of installs.entries()) {
|
||||
assert.equal(handle.child.exitCode, 0);
|
||||
assert.equal(readFileSync(join(root, `bundled-${slug}.log`), 'utf8'), `install ${stagedSource} -> ${target}\n`);
|
||||
assert.equal(target, join(root, `package-${index}`));
|
||||
assert.ok(existsSync(join(target, 'package.json')));
|
||||
}
|
||||
const timings = bundledInstallTimings(installs);
|
||||
assert.deepEqual(Object.keys(timings), ['stage_bundled_server_ms', 'stage_bundled_db_ms']);
|
||||
for (const value of Object.values(timings)) assert.ok(Number.isInteger(value) && value >= 0);
|
||||
}));
|
||||
|
||||
test('stopping the installs after one fails also stops the npm process the other started', { skip: process.platform === 'win32' && 'process groups are POSIX only' }, async () => withScratch(async root => {
|
||||
// The real wrapper blocks in `execFileSync('npm', ['install', ...])`; here
|
||||
// the stand-in npm announces its pid and runs on, and the wrapper blocks on
|
||||
// it the same way. Stopping only the wrapper would leave it writing.
|
||||
const script = fakeScript(root, `
|
||||
import { execFileSync } from 'node:child_process';
|
||||
const [, target] = process.argv.slice(2);
|
||||
if (target.endsWith('package-1')) { console.error('bundled install exploded'); process.exit(2); }
|
||||
execFileSync(process.execPath, ['-e', "process.stdout.write('npm ' + process.pid + '\\\\n'); setInterval(() => {}, 1000)"], { stdio: 'inherit' });
|
||||
`);
|
||||
const installs = startBundledInstalls(entriesFor(root, ['@paperclipai/server', '@paperclipai/db']), { logDir: root, cwd: root, script });
|
||||
await assert.rejects(Promise.all(installs.map(({ handle }) => handle.done)), /bundled-db prefetch failed \(exit 2\)/);
|
||||
const log = join(root, 'bundled-server.log');
|
||||
const logHasPid = () => existsSync(log) && /npm \d+/.test(readFileSync(log, 'utf8'));
|
||||
for (let attempt = 0; attempt < 200 && !logHasPid(); attempt++) await new Promise(resolve => setTimeout(resolve, 25));
|
||||
const npmPid = Number(readFileSync(log, 'utf8').match(/npm (\d+)/)[1]);
|
||||
const alive = pid => { try { process.kill(pid, 0); return !/^\d+ \(.*\) Z /.test(readFileSync(`/proc/${pid}/stat`, 'utf8')); } catch (error) { return error.code !== 'ESRCH'; } };
|
||||
assert.ok(alive(npmPid), 'the stand-in npm runs before the stop');
|
||||
await stopPrefetches(installs.map(({ handle }) => handle));
|
||||
let stopped = false;
|
||||
for (let attempt = 0; attempt < 200 && !(stopped = !alive(npmPid)); attempt++) await new Promise(resolve => setTimeout(resolve, 25));
|
||||
assert.ok(stopped, `stand-in npm ${npmPid} outlived stopPrefetches`);
|
||||
assert.notEqual(installs[0].handle.child.signalCode ?? installs[0].handle.child.exitCode, null);
|
||||
}));
|
||||
|
||||
test('a failed install rejects with the tail of its own log and leaves the others to be stopped', async () => withScratch(async root => {
|
||||
const script = fakeScript(root, `
|
||||
const [, target] = process.argv.slice(2);
|
||||
if (target.endsWith('package-1')) { console.error('bundled install exploded'); process.exit(2); }
|
||||
setTimeout(() => {}, 30_000);
|
||||
`);
|
||||
const installs = startBundledInstalls(entriesFor(root, ['@paperclipai/server', '@paperclipai/adapter-utils']), { logDir: root, cwd: root, script });
|
||||
await assert.rejects(Promise.all(installs.map(({ handle }) => handle.done)), /bundled-adapter_utils prefetch failed \(exit 2\): bundled install exploded/);
|
||||
assert.equal(installs[0].handle.child.exitCode, null, 'the other install is still running when the failure surfaces');
|
||||
await stopPrefetches(installs.map(({ handle }) => handle));
|
||||
assert.notEqual(installs[0].handle.child.signalCode ?? installs[0].handle.child.exitCode, null);
|
||||
}));
|
||||
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
existsSync,
|
||||
cpSync,
|
||||
lstatSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
@@ -56,6 +57,33 @@ function esbuildFixture(t) {
|
||||
}
|
||||
const codexTargets = ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64", "win32-x64", "win32-arm64"];
|
||||
const codexOptional = Object.fromEntries(codexTargets.map(target => [`@openai/codex-${target}`, `npm:@openai/codex@0.160.0-${target}`]));
|
||||
const claudeTargets = [...codexTargets, "linux-x64-musl", "linux-arm64-musl"];
|
||||
const claudeOptional = Object.fromEntries(claudeTargets.map(target => [`@anthropic-ai/claude-agent-sdk-${target}`, "0.3.286"]));
|
||||
function claudeFixture(t) {
|
||||
const directory = realpathSync(mkdtempSync(join(tmpdir(), "paperclip-bundled-claude-")));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
const graph = join(directory, "node_modules");
|
||||
const writePackage = (location, metadata) => {
|
||||
mkdirSync(location, { recursive: true });
|
||||
writeFileSync(join(location, "package.json"), JSON.stringify(metadata));
|
||||
return location;
|
||||
};
|
||||
const bridge = writePackage(join(graph, "@agentclientprotocol/claude-agent-acp"), {
|
||||
name: "@agentclientprotocol/claude-agent-acp", version: "0.73.0", dependencies: { "@anthropic-ai/claude-agent-sdk": "0.3.286" },
|
||||
});
|
||||
writeFileSync(join(bridge, "index.js"), "export {};\n");
|
||||
const runtime = writePackage(join(graph, "@anthropic-ai/claude-agent-sdk"), {
|
||||
name: "@anthropic-ai/claude-agent-sdk", version: "0.3.286", exports: { ".": "./sdk.mjs" }, optionalDependencies: claudeOptional,
|
||||
});
|
||||
writeFileSync(join(runtime, "sdk.mjs"), "// official JavaScript fixture, never executed\n");
|
||||
const producer = writePackage(join(graph, "@anthropic-ai/claude-agent-sdk-linux-x64"), {
|
||||
name: "@anthropic-ai/claude-agent-sdk-linux-x64", version: "0.3.286", os: ["linux"], cpu: ["x64"],
|
||||
});
|
||||
writeFileSync(join(producer, "claude"), "producer native fixture, never executed");
|
||||
const profiles = [{ agent: "claude", ...profileData.profiles.claude }];
|
||||
const publish = { name: "paperclip-claude-pack-fixture", version: "1.0.0", dependencies: { "@agentclientprotocol/claude-agent-acp": "0.73.0" }, bundleDependencies: ["@agentclientprotocol/claude-agent-acp"] };
|
||||
return { directory, graph, bridge, runtime, producer, profiles, publish, writePackage };
|
||||
}
|
||||
function codexFixture(t) {
|
||||
const directory = realpathSync(mkdtempSync(join(tmpdir(), "paperclip-bundled-codex-")));
|
||||
t.after(() => rmSync(directory, { recursive: true, force: true }));
|
||||
@@ -176,13 +204,26 @@ test("published packages preserve the patched ACPX runtime", () => {
|
||||
assert.equal(cliEsbuildConfig.external.includes("acpx"), false);
|
||||
});
|
||||
|
||||
test("published Claude bridge pins its qualified SDK and provider dependency closure", () => {
|
||||
const published = materializePublishManifest(serverPackage);
|
||||
const { installManifest } = configureBundledProviderOverrides(
|
||||
createBundledInstallManifest(published, serverPackage.bundleDependencies),
|
||||
serverPackage.bundleDependencies, rootPackage, profileData,
|
||||
);
|
||||
assert.deepEqual(installManifest.overrides["@agentclientprotocol/claude-agent-acp@0.73.0"], {
|
||||
"@anthropic-ai/claude-agent-sdk": "0.3.286",
|
||||
"@agentclientprotocol/sdk": "1.4.0",
|
||||
zod: "4.4.3",
|
||||
});
|
||||
});
|
||||
|
||||
test("published Codex bridge uses its qualified producer closure without consumer overrides", () => {
|
||||
const published = materializePublishManifest(serverPackage);
|
||||
const { installManifest, profiles } = configureBundledProviderOverrides(
|
||||
createBundledInstallManifest(published, serverPackage.bundleDependencies),
|
||||
serverPackage.bundleDependencies, rootPackage, profileData,
|
||||
);
|
||||
assert.equal(profiles.length, 1);
|
||||
assert.deepEqual(profiles.map(profile => profile.agent), ["codex", "claude"]);
|
||||
for (const profile of profiles) {
|
||||
assert.equal(published.dependencies[profile.agentServerPackage], profile.agentServerVersion);
|
||||
assert.ok(published.bundleDependencies.includes(profile.agentServerPackage));
|
||||
@@ -201,6 +242,58 @@ test("published Codex bridge uses its qualified producer closure without consume
|
||||
assert.throws(() => configureBundledProviderOverrides(conflict, serverPackage.bundleDependencies, rootPackage, profileData), /conflicting producer override/);
|
||||
});
|
||||
|
||||
test("Linux-produced Claude bundles retain the SDK and install native packages for the consumer platform", (t) => {
|
||||
const { directory, bridge, runtime, producer, profiles, publish } = claudeFixture(t);
|
||||
const bridgeBefore = readFileSync(join(bridge, "package.json"), "utf8");
|
||||
const staged = stageBundledProviderOptionalDependencies(directory, publish, profiles);
|
||||
assert.deepEqual(staged.optionalDependencies, claudeOptional);
|
||||
assert.equal(existsSync(producer), false);
|
||||
assert.equal(JSON.parse(readFileSync(join(runtime, "package.json"), "utf8")).optionalDependencies, undefined);
|
||||
assert.equal(readFileSync(join(bridge, "package.json"), "utf8"), bridgeBefore);
|
||||
assert.deepEqual(stageBundledProviderOptionalDependencies(directory, staged, profiles), staged);
|
||||
writeFileSync(join(directory, "package.json"), JSON.stringify(staged));
|
||||
const packs = JSON.parse(execFileSync("npm", ["pack", "--ignore-scripts", "--offline", "--json"], {
|
||||
cwd: directory, encoding: "utf8", timeout: 30_000,
|
||||
env: { ...process.env, HOME: directory, npm_config_cache: join(directory, "cache"), npm_config_update_notifier: "false" },
|
||||
}));
|
||||
const paths = packs[0].files.map(file => file.path);
|
||||
assert.ok(paths.some(path => path.endsWith("@anthropic-ai/claude-agent-sdk/sdk.mjs")));
|
||||
assert.equal(paths.some(path => /@anthropic-ai\/claude-agent-sdk-(?:linux|darwin|win32)-/.test(path)), false);
|
||||
});
|
||||
|
||||
test("Claude bundle validation rejects unqualified declarations before removing native payloads", (t) => {
|
||||
const { directory, runtime, producer, profiles, publish } = claudeFixture(t);
|
||||
const file = join(runtime, "package.json"), original = JSON.parse(readFileSync(file, "utf8"));
|
||||
for (const metadata of [
|
||||
{ ...original, version: "0.3.285" },
|
||||
{ ...original, optionalDependencies: { ...claudeOptional, "@anthropic-ai/claude-agent-sdk-darwin-arm64": "latest" } },
|
||||
{ ...original, optionalDependencies: {} },
|
||||
]) {
|
||||
writeFileSync(file, JSON.stringify(metadata));
|
||||
assert.throws(() => stageBundledProviderOptionalDependencies(directory, publish, profiles), /version mismatch|not qualified/);
|
||||
assert.equal(existsSync(producer), true);
|
||||
}
|
||||
});
|
||||
|
||||
test("mixed provider bundles validate both runtimes before removing either producer executable", (t) => {
|
||||
const codex = codexFixture(t), claude = claudeFixture(t);
|
||||
for (const name of ["@agentclientprotocol/claude-agent-acp", "@anthropic-ai/claude-agent-sdk", "@anthropic-ai/claude-agent-sdk-linux-x64"]) {
|
||||
cpSync(join(claude.graph, name), join(codex.graph, name), { recursive: true });
|
||||
}
|
||||
const profiles = [...codex.profiles, ...claude.profiles];
|
||||
const publish = { ...codex.publish, dependencies: { ...codex.publish.dependencies, ...claude.publish.dependencies },
|
||||
bundleDependencies: [...codex.publish.bundleDependencies, ...claude.publish.bundleDependencies] };
|
||||
const file = join(codex.graph, "@anthropic-ai/claude-agent-sdk/package.json");
|
||||
const metadata = JSON.parse(readFileSync(file, "utf8"));
|
||||
writeFileSync(file, JSON.stringify({ ...metadata, version: "0.3.285" }));
|
||||
assert.throws(() => stageBundledProviderOptionalDependencies(codex.directory, publish, profiles), /Claude runtime version mismatch/);
|
||||
assert.equal(existsSync(codex.producer), true);
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(codex.runtime, "package.json"), "utf8")).optionalDependencies, codexOptional);
|
||||
writeFileSync(file, JSON.stringify(metadata));
|
||||
const staged = stageBundledProviderOptionalDependencies(codex.directory, publish, profiles);
|
||||
assert.deepEqual(staged.optionalDependencies, { ...codexOptional, ...claudeOptional });
|
||||
});
|
||||
|
||||
test("Codex bundles retain pinned JavaScript and delegate official consumer platform aliases without native payloads", (t) => {
|
||||
const { directory, graph, bridge, runtime, producer, nested, profiles, publish } = codexFixture(t);
|
||||
const before = [bridge, runtime].map(path => readFileSync(join(path, "package.json"), "utf8"));
|
||||
@@ -530,7 +623,7 @@ mkdir -p "$destination/node_modules/.pnpm"
|
||||
set -euo pipefail
|
||||
printf 'npm %s\\n' "$*" >> "$FAKE_CALL_LOG"
|
||||
[ "$*" = "install --omit=dev --ignore-scripts --no-audit --no-fund" ]
|
||||
node -e 'const fs = require("node:fs"); const pkg = require("./package.json"); if ("devDependencies" in pkg) process.exit(1); for (const [name, version] of Object.entries(pkg.dependencies)) { const dir = "node_modules/" + name; fs.mkdirSync(dir + "/dist", { recursive: true }); fs.writeFileSync(dir + "/package.json", JSON.stringify({ name, version })); } for (const [bridge, runtimes] of Object.entries(pkg.overrides ?? {})) { for (const [name, version] of Object.entries(runtimes)) { const dir = "node_modules/" + name; fs.mkdirSync(dir, { recursive: true }); const native = name === "@openai/codex"; const optionalDependencies = native ? Object.fromEntries(["linux-x64","linux-arm64","darwin-x64","darwin-arm64","win32-x64","win32-arm64"].map(target => [name+"-"+target,"npm:"+name+"@"+version+"-"+target])) : undefined; fs.writeFileSync(dir + "/package.json", JSON.stringify({ name, version, optionalDependencies })); } }'
|
||||
node -e 'const fs = require("node:fs"); const pkg = require("./package.json"); if ("devDependencies" in pkg) process.exit(1); for (const [name, version] of Object.entries(pkg.dependencies)) { const dir = "node_modules/" + name; fs.mkdirSync(dir + "/dist", { recursive: true }); fs.writeFileSync(dir + "/package.json", JSON.stringify({ name, version })); } for (const [bridge, runtimes] of Object.entries(pkg.overrides ?? {})) { for (const [name, version] of Object.entries(runtimes)) { const dir = "node_modules/" + name; fs.mkdirSync(dir, { recursive: true }); const native = name === "@openai/codex"; const targets = ["linux-x64","linux-arm64","darwin-x64","darwin-arm64","win32-x64","win32-arm64"]; const optionalDependencies = native ? Object.fromEntries(targets.map(target => [name+"-"+target,"npm:"+name+"@"+version+"-"+target])) : name === "@anthropic-ai/claude-agent-sdk" ? Object.fromEntries([...targets,"linux-x64-musl","linux-arm64-musl"].map(target => [name+"-"+target,version])) : undefined; fs.writeFileSync(dir + "/package.json", JSON.stringify({ name, version, optionalDependencies })); } }'
|
||||
node -e 'const fs=require("node:fs");fs.cpSync(process.env.FAKE_NATIVE_ARTIFACTS+"/@openai/codex-linux-x64","node_modules/@openai/codex-linux-x64",{recursive:true});fs.mkdirSync("node_modules/@openai/codex/vendor",{recursive:true});fs.writeFileSync("node_modules/@openai/codex/vendor/codex","producer fallback fixture");'
|
||||
mkdir -p node_modules/acpx/dist
|
||||
node -e 'const fs=require("node:fs");fs.mkdirSync("node_modules/esbuild",{recursive:true});fs.writeFileSync("node_modules/esbuild/package.json",JSON.stringify({name:"esbuild",version:"0.28.2",scripts:{postinstall:"node install.js"},optionalDependencies:Object.fromEntries(["linux-x64","darwin-arm64","darwin-x64"].map(target=>["@esbuild/"+target,"0.28.2"]))}));fs.mkdirSync("node_modules/@esbuild/linux-x64",{recursive:true});fs.writeFileSync("node_modules/@esbuild/linux-x64/package.json",JSON.stringify({name:"@esbuild/linux-x64",version:"0.28.2",os:["linux"],cpu:["x64"]}));'
|
||||
@@ -590,7 +683,7 @@ printf 'patched spawnEnvironment runtime\\n' > "$target/dist/runtime.js"
|
||||
assert.equal(lstatSync(stagedAcpxDir).isSymbolicLink(), false);
|
||||
assert.equal(existsSync(join(destinationDir, "node_modules/.pnpm")), false);
|
||||
const stagedManifest = JSON.parse(readFileSync(join(destinationDir, "package.json"), "utf8"));
|
||||
for (const [name, version] of Object.entries(codexOptional)) {
|
||||
for (const [name, version] of Object.entries({ ...codexOptional, ...claudeOptional })) {
|
||||
assert.equal(stagedManifest.optionalDependencies[name], version);
|
||||
assert.equal(existsSync(join(destinationDir, "node_modules", name)), false);
|
||||
assert.equal(stagedManifest.bundleDependencies.includes(name), false);
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "bitly",
|
||||
"name": "Bitly",
|
||||
"description": "Connect Bitly's provider-hosted MCP server.",
|
||||
"categories": [
|
||||
"analytics"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/bitly.svg"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://api-ssl.bitly.com/*"
|
||||
],
|
||||
"docsUrl": "https://dev.bitly.com/bitly-mcp/overview/quickstart/",
|
||||
"redirectConstraints": "https-or-loopback-http",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp-oauth",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "oauth",
|
||||
"ownershipModes": [
|
||||
"dcr"
|
||||
],
|
||||
"whenToUse": "Use browser sign-in for the provider-hosted MCP server.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://api-ssl.bitly.com/v4/mcp",
|
||||
"scopesHint": [
|
||||
"mcp.all"
|
||||
]
|
||||
},
|
||||
"guidanceMd": "Connect Bitly in the browser. Choose only the intended workspace. Review the discovered actions and keep writes and destructive operations disabled until explicitly needed.",
|
||||
"riskTier": "S4",
|
||||
"label": "Sign in with Bitly",
|
||||
"consoleLinks": {
|
||||
"docs": "https://dev.bitly.com/bitly-mcp/using-the-bitly-mcp-server/configuration-details/"
|
||||
},
|
||||
"warnings": [
|
||||
"The mcp.all scope covers Bitly's broad MCP catalog, including link and QR management, analytics, groups, custom domains, and Bitly Sites.",
|
||||
"The catalog includes publishing and deletion operations, including permanent short-link deletion, site deletion, and custom-domain changes. Review the full catalog and keep write and destructive actions Off during read-only qualification."
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "mcp-api-key",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "api_key",
|
||||
"ownershipModes": [
|
||||
"customer"
|
||||
],
|
||||
"whenToUse": "Use a customer-created Bitly API token when browser sign-in is not suitable.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://api-ssl.bitly.com/v4/mcp"
|
||||
},
|
||||
"guidanceMd": "Generate a Bitly API token under Settings → API. The token may authorize the MCP's full catalog under your Bitly account; Paperclip cannot narrow the token. Review discovered actions and keep writes and destructive operations disabled until explicitly needed.",
|
||||
"riskTier": "S4",
|
||||
"label": "Use an API key",
|
||||
"credentialFields": [
|
||||
{
|
||||
"key": "authorization",
|
||||
"label": "Bitly API key",
|
||||
"type": "password",
|
||||
"required": true,
|
||||
"placeholder": "Paste your Bitly API token",
|
||||
"secret": true,
|
||||
"helperMd": "Bitly's API token may authorize the MCP's broad catalog under your account. Paperclip cannot narrow the token; keep write and destructive actions Off unless you explicitly need them."
|
||||
}
|
||||
],
|
||||
"keyPlacement": {
|
||||
"location": "header",
|
||||
"name": "Authorization",
|
||||
"prefix": "Bearer "
|
||||
},
|
||||
"consoleLinks": {
|
||||
"keys": "https://bitly.com/settings/api",
|
||||
"docs": "https://dev.bitly.com/bitly-mcp/using-the-bitly-mcp-server/configuration-details/"
|
||||
},
|
||||
"warnings": [
|
||||
"This API token is not documented as a restricted MCP token. It may authorize Bitly's broad MCP catalog, including link creation/update/deletion, QR codes, Sites publishing/deletion, and custom-domain operations.",
|
||||
"Review the discovered catalog and keep write and destructive actions Off during read-only qualification."
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "clickup",
|
||||
"name": "ClickUp",
|
||||
"description": "Search ClickUp workspaces and manage tasks, docs, and conversations through the official MCP server.",
|
||||
"categories": [
|
||||
"productivity"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/clickup.png"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://mcp.clickup.com/*"
|
||||
],
|
||||
"docsUrl": "https://developer.clickup.com/docs/connect-an-ai-assistant-to-clickups-mcp-server",
|
||||
"redirectConstraints": "https-or-loopback-http",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp-oauth",
|
||||
"label": "Sign in with ClickUp",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "oauth",
|
||||
"ownershipModes": [
|
||||
"dcr"
|
||||
],
|
||||
"grantKinds": [
|
||||
"user"
|
||||
],
|
||||
"whenToUse": "Connect the provider-hosted ClickUp MCP server with browser sign-in.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://mcp.clickup.com/mcp",
|
||||
"scopesHint": [
|
||||
"read",
|
||||
"write"
|
||||
]
|
||||
},
|
||||
"guidanceMd": "Choose only the ClickUp workspace needed. Confirm the target list and task before changing content, starting a timer, posting a chat message, or deleting anything.",
|
||||
"warnings": [
|
||||
"ClickUp's MCP may expose task, document, and chat writes, time tracking, and task deletion under the signed-in user's permissions. Review the catalog and keep write and destructive actions disabled until explicitly needed.",
|
||||
"Workspace usage may be subject to plan-based rolling daily MCP limits."
|
||||
],
|
||||
"consoleLinks": {
|
||||
"docs": "https://developer.clickup.com/docs/connect-an-ai-assistant-to-clickups-mcp-server"
|
||||
},
|
||||
"riskTier": "S4",
|
||||
"requiredResourceFilters": [
|
||||
"workspace",
|
||||
"list"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "context7",
|
||||
"name": "Context7",
|
||||
"description": "Search public library documentation and retrieve current code examples.",
|
||||
"categories": [
|
||||
"developer"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/context7.svg",
|
||||
"darkLogoUrl": "/brands/apps/context7-dark.svg"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://mcp.context7.com/*"
|
||||
],
|
||||
"docsUrl": "https://context7.com/docs/api-guide",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "none",
|
||||
"ownershipModes": [
|
||||
"customer"
|
||||
],
|
||||
"whenToUse": "Use the public hosted MCP server for read-only library documentation lookups.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://mcp.context7.com/mcp"
|
||||
},
|
||||
"guidanceMd": "Connect Context7 to resolve public library IDs and retrieve current documentation. Anonymous access is supported with low rate limits; this method does not configure an API key. Keep queries limited to public documentation questions and do not include credentials, personal data, or proprietary code.",
|
||||
"riskTier": "S1",
|
||||
"consoleLinks": {
|
||||
"docs": "https://context7.com/docs/api-guide"
|
||||
},
|
||||
"warnings": [
|
||||
"Context7 processes the query text. Do not include secrets, personal information, or proprietary code.",
|
||||
"Anonymous requests have low rate limits. Provider documentation says an API key raises limits and supports private repositories; this public method does not configure a key."
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "hugging-face",
|
||||
"name": "Hugging Face",
|
||||
"description": "Connect Hugging Face's provider-hosted MCP server.",
|
||||
"categories": [
|
||||
"ai"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/hugging-face.svg"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://huggingface.co/*"
|
||||
],
|
||||
"docsUrl": "https://huggingface.co/docs/hub/agents-mcp",
|
||||
"redirectConstraints": "https-or-loopback-http",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp-oauth",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "oauth",
|
||||
"ownershipModes": [
|
||||
"dcr"
|
||||
],
|
||||
"whenToUse": "Use browser sign-in for the provider-hosted MCP server.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://huggingface.co/mcp?login&gradio=none",
|
||||
"scopesHint": [
|
||||
"read-mcp",
|
||||
"read-repos",
|
||||
"contribute-repos",
|
||||
"jobs"
|
||||
]
|
||||
},
|
||||
"guidanceMd": "Connect Hugging Face in the browser. A Hugging Face account.",
|
||||
"riskTier": "S2",
|
||||
"label": "Sign in with Hugging Face",
|
||||
"consoleLinks": {
|
||||
"docs": "https://huggingface.co/docs/hub/agents-mcp"
|
||||
},
|
||||
"warnings": [
|
||||
"A Hugging Face account."
|
||||
]
|
||||
},
|
||||
{
|
||||
"key": "public-mcp",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "none",
|
||||
"ownershipModes": [
|
||||
"customer"
|
||||
],
|
||||
"whenToUse": "Use the anonymous hosted MCP endpoint for public Hugging Face Hub lookups.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://huggingface.co/mcp"
|
||||
},
|
||||
"guidanceMd": "This anonymous method exposes the public Hugging Face Hub read tools for repository search, public repository details, datasets, and documentation. It does not require an account or API key and is rate-limited. Keep queries to public information; do not include secrets, personal data, private repository names, or proprietary content.",
|
||||
"riskTier": "S1",
|
||||
"label": "Use public Hugging Face Hub",
|
||||
"consoleLinks": {
|
||||
"docs": "https://huggingface.co/docs/hub/agents-mcp"
|
||||
},
|
||||
"warnings": [
|
||||
"Anonymous requests are rate-limited by Hugging Face.",
|
||||
"This method only advertises read tools for the public Hub; do not include secrets, personal data, private repository names, or proprietary content in queries."
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "klaviyo",
|
||||
"name": "Klaviyo",
|
||||
"description": "Review Klaviyo campaign performance and prepare campaign changes for review.",
|
||||
"categories": [
|
||||
"commerce"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/klaviyo.png"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://mcp.klaviyo.com/*"
|
||||
],
|
||||
"docsUrl": "https://developers.klaviyo.com/en/docs/connect_to_the_klaviyo_mcp_server",
|
||||
"redirectConstraints": "https-or-loopback-http",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp-oauth",
|
||||
"label": "Sign in with Klaviyo",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "oauth",
|
||||
"oauthClientRegistration": "dcr",
|
||||
"ownershipModes": [
|
||||
"dcr"
|
||||
],
|
||||
"whenToUse": "Connect to the provider-hosted Klaviyo MCP server with browser sign-in.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://mcp.klaviyo.com/mcp?toolsets=campaigns:read,campaigns:write,metrics:read,segments:read"
|
||||
},
|
||||
"guidanceMd": "Review campaign and metric data, then prepare campaign changes for human review. Campaign write access may expose sends and other state changes; never send or schedule a campaign without explicit authorization and an allowed Paperclip action.",
|
||||
"riskTier": "S4",
|
||||
"tenantFields": [
|
||||
{
|
||||
"key": "readOnly",
|
||||
"label": "Read-only mode",
|
||||
"type": "checkbox",
|
||||
"advanced": true,
|
||||
"defaultValue": false,
|
||||
"helperMd": "Turn on to remove Klaviyo MCP tools that can modify account data. This provider-side restriction is separate from Paperclip action permissions.",
|
||||
"transport": {
|
||||
"location": "query",
|
||||
"name": "read-only",
|
||||
"format": "boolean",
|
||||
"omitFalse": true
|
||||
}
|
||||
}
|
||||
],
|
||||
"consoleLinks": {
|
||||
"docs": "https://developers.klaviyo.com/en/docs/connect_to_the_klaviyo_mcp_server"
|
||||
},
|
||||
"warnings": [
|
||||
"Klaviyo's remote MCP requires an Owner, Admin, or Manager account role.",
|
||||
"The campaigns:write toolset may expose campaign sending and other writes. Keep write actions Off until reviewed; the provider's Advanced read-only mode is optional and does not replace Paperclip action permissions."
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "monday",
|
||||
"name": "monday.com",
|
||||
"description": "Read and manage boards and work items in monday.com.",
|
||||
"categories": [
|
||||
"productivity"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/monday.png",
|
||||
"darkLogoUrl": "/brands/apps/monday-dark.png"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://mcp.monday.com/*"
|
||||
],
|
||||
"docsUrl": "https://developer.monday.com/api-reference/docs/integrate-with-monday-mcp",
|
||||
"redirectConstraints": "https-or-loopback-http",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp-oauth",
|
||||
"label": "Sign in with monday.com",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "oauth",
|
||||
"ownershipModes": [
|
||||
"dcr"
|
||||
],
|
||||
"whenToUse": "Connect the hosted monday.com MCP server using OAuth and dynamic client registration.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://mcp.monday.com/mcp"
|
||||
},
|
||||
"guidanceMd": "Use the selected monday.com account only. Read the target board and its columns before acting, and verify the exact board and item before any change. Keep write and destructive actions Off unless the user explicitly asks for a specific change. Never send messages, publish, or trigger automations through a board action without separate authorization.",
|
||||
"warnings": [
|
||||
"The hosted MCP server acts as the authorizing monday.com user and inherits that user's monday.com permissions. An account admin may also restrict MCP access to selected workspaces.",
|
||||
"An additional OAuth/DCR connection can require account-admin approval for Paperclip and its exact callback URL before user authorization completes. If approval is pending, reconnect after the account admin approves it.",
|
||||
"monday.com personal API tokens are documented for personal use, prototyping, and testing, and the provider says they are not intended for integrations used by other people. This connector uses the OAuth/DCR route for additional MCP clients; account-level approval may be required.",
|
||||
"The tool catalog includes actions that can send notifications (optionally email), delete or activate automations, publish workflows, and create or run monday.com platform agents; a low-level tool can execute GraphQL mutations. Keep these actions Off unless separately authorized, and never run agents or change automations during qualification.",
|
||||
"MCP calls count toward the account's monday.com API call limits. Keep tests bounded and avoid repeated discovery or broad queries."
|
||||
],
|
||||
"consoleLinks": {
|
||||
"docs": "https://developer.monday.com/api-reference/docs/integrate-with-monday-mcp"
|
||||
},
|
||||
"riskTier": "S4"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
{
|
||||
"schemaVersion": 1,
|
||||
"slug": "resend",
|
||||
"name": "Resend",
|
||||
"description": "Connect Resend's provider-hosted MCP server.",
|
||||
"categories": [
|
||||
"communication"
|
||||
],
|
||||
"featured": false,
|
||||
"branding": {
|
||||
"logoUrl": "/brands/apps/resend.svg",
|
||||
"darkLogoUrl": "/brands/apps/resend-dark.svg"
|
||||
},
|
||||
"urlPatterns": [
|
||||
"https://mcp.resend.com/*"
|
||||
],
|
||||
"docsUrl": "https://resend.com/changelog/remote-mcp-server",
|
||||
"redirectConstraints": "https-or-loopback-http",
|
||||
"methods": [
|
||||
{
|
||||
"key": "mcp-oauth",
|
||||
"transport": "mcp_remote",
|
||||
"auth": "oauth",
|
||||
"ownershipModes": [
|
||||
"dcr"
|
||||
],
|
||||
"whenToUse": "Use browser sign-in for the provider-hosted MCP server.",
|
||||
"defaults": {
|
||||
"serverUrl": "https://mcp.resend.com/mcp",
|
||||
"scopesHint": [
|
||||
"full_access"
|
||||
]
|
||||
},
|
||||
"guidanceMd": "Sign in to the intended Resend team. The full_access grant covers reading and managing account resources, including sending email and deletion. Review the discovered actions in Paperclip before using them.",
|
||||
"riskTier": "S4",
|
||||
"label": "Sign in with Resend",
|
||||
"consoleLinks": {
|
||||
"docs": "https://resend.com/changelog/remote-mcp-server"
|
||||
},
|
||||
"warnings": [
|
||||
"Resend full_access grants broad account read/write/delete access; Paperclip action policies govern execution separately."
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -17,7 +17,7 @@ const repoRoot = path.resolve(scriptDir, "..");
|
||||
const workspacePath = path.join(repoRoot, "pnpm-workspace.yaml");
|
||||
const releasePackageMapPath = path.join(repoRoot, "scripts", "release-package-map.mjs");
|
||||
|
||||
function parseWorkspaceEntries(workspaceText) {
|
||||
export function parseWorkspaceEntries(workspaceText) {
|
||||
// Keep this aligned with the repo's block-sequence `packages:` format in
|
||||
// pnpm-workspace.yaml. If that file moves to a more complex YAML shape,
|
||||
// switch this parser to a real YAML parser instead of line matching.
|
||||
@@ -61,7 +61,7 @@ function globToRegExp(pattern) {
|
||||
return new RegExp(`^${regex}$`);
|
||||
}
|
||||
|
||||
function isWorkspacePackage(pkgDir, workspaceEntries) {
|
||||
export function isWorkspacePackage(pkgDir, workspaceEntries) {
|
||||
let included = false;
|
||||
|
||||
for (const entry of workspaceEntries) {
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -783,7 +783,7 @@ const apps = [
|
||||
{
|
||||
serverUrl: "https://mcp.sentry.dev/mcp",
|
||||
discoveryUrl:
|
||||
"https://sentry.io/.well-known/oauth-authorization-server",
|
||||
"https://mcp.sentry.dev/.well-known/oauth-protected-resource/mcp",
|
||||
},
|
||||
"S2",
|
||||
"Connect the Sentry organization and projects agents need for incident work.",
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
// The public-install verifier packs every staged public package with
|
||||
// `npm pack`. Each pack reads only its own `package-<index>` directory and
|
||||
// writes one tarball, and the work is a Node start-up plus CPU-bound gzip per
|
||||
// package, so the packs run as concurrent children through the prefetch
|
||||
// helper, at most one per core, instead of one after another. Each child
|
||||
// packs into its own destination directory, so the tarball a package produced
|
||||
// is read from there rather than guessed from a shared directory, and the
|
||||
// results come back in the order the entries were given.
|
||||
import { mkdirSync, readdirSync } from 'node:fs';
|
||||
import { availableParallelism } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { startPrefetch } from './prefetch-child.mjs';
|
||||
|
||||
export const PACK_ARGS = ['pack', '--ignore-scripts', '--pack-destination'];
|
||||
|
||||
export const packConcurrency = () => Math.max(1, availableParallelism());
|
||||
|
||||
// Packs `entries` (`{ name, target, destination }`) with at most `concurrency`
|
||||
// children at once, launched in the order given, and resolves with each entry
|
||||
// extended by its `tarball` path and `durationMs`, in the same order. Every
|
||||
// child is passed to `register` as it starts, so the caller can stop the ones
|
||||
// still running when something fails. The first failure rejects with the tail
|
||||
// of that child's log, and nothing further is launched after it.
|
||||
export function packStagedPackages(entries, { logDir, env = process.env, concurrency = packConcurrency(), register = () => {}, command = ['npm'] }) {
|
||||
const [cmd, ...commandArgs] = command;
|
||||
const results = new Array(entries.length);
|
||||
const queue = entries.map((entry, index) => ({ entry, index }));
|
||||
let active = 0;
|
||||
let failed = false;
|
||||
return new Promise((resolve, reject) => {
|
||||
const fail = error => { failed = true; reject(error); };
|
||||
const launch = () => {
|
||||
if (!failed && active === 0 && queue.length === 0) return resolve(results);
|
||||
while (!failed && active < concurrency && queue.length > 0) {
|
||||
const { entry, index } = queue.shift();
|
||||
active += 1;
|
||||
let handle;
|
||||
try {
|
||||
mkdirSync(entry.destination, { recursive: true });
|
||||
handle = startPrefetch({ name: `pack-${index}`, cmd, args: [...commandArgs, ...PACK_ARGS, entry.destination], cwd: entry.target, env, logPath: join(logDir, `pack-${index}.log`) });
|
||||
register(handle);
|
||||
} catch (error) {
|
||||
return fail(error);
|
||||
}
|
||||
handle.done.then(() => {
|
||||
const packed = readdirSync(entry.destination).filter(file => file.endsWith('.tgz'));
|
||||
if (packed.length !== 1) throw new Error(`${entry.name} pack produced ${packed.length} tarballs in ${entry.destination}`);
|
||||
results[index] = { ...entry, tarball: join(entry.destination, packed[0]), durationMs: handle.durationMs };
|
||||
active -= 1;
|
||||
launch();
|
||||
}).catch(fail);
|
||||
}
|
||||
};
|
||||
launch();
|
||||
});
|
||||
}
|
||||
|
||||
// Timing-line keys: the concurrency used, the sum of the per-pack times (what
|
||||
// the packs would have cost one after another) and the longest single pack.
|
||||
export const packTimings = (packs, concurrency) => ({
|
||||
pack_concurrency: concurrency,
|
||||
pack_serial_ms: packs.reduce((total, { durationMs }) => total + durationMs, 0),
|
||||
pack_longest_ms: packs.reduce((longest, { durationMs }) => Math.max(longest, durationMs), 0),
|
||||
});
|
||||
@@ -0,0 +1,118 @@
|
||||
// Background downloads for verification scripts: start a child early with its
|
||||
// output in a log file, join it where the result is needed, and stop it before
|
||||
// the scratch directory it writes into is removed. No shell, no inherited
|
||||
// terminal, and a failure is reported where the caller awaits it.
|
||||
//
|
||||
// A child may start processes of its own: the bundled-dependency installs are
|
||||
// a Node wrapper blocked in `npm install`. Signalling only the wrapper would
|
||||
// leave npm writing into the scratch root while it is removed, so on POSIX
|
||||
// each command runs under `prefetch-supervisor.mjs` as the leader of its own
|
||||
// process group, and stopping signals and waits for the whole group. A group
|
||||
// of its own no longer receives the terminal's Ctrl-C, and the parent cannot
|
||||
// forward it: the verifier spends most of its time blocked in `execFileSync`,
|
||||
// where no JavaScript signal handler can run, and installing one would also
|
||||
// replace the parent's immediate default termination. Instead the supervisor
|
||||
// holds a pipe from the parent and stops its group when the pipe closes, which
|
||||
// the kernel does when the parent exits for any reason. The parent installs
|
||||
// no signal handlers.
|
||||
//
|
||||
// Windows has no process groups and `kill` there ends a process outright, so a
|
||||
// signalled supervisor could pass nothing on and its command would run on
|
||||
// after cleanup returned. There a deliberate stop goes through the same pipe
|
||||
// as a written request, which the supervisor answers by ending its command,
|
||||
// and only the SIGKILL escalation targets the supervisor itself.
|
||||
import { spawn } from 'node:child_process';
|
||||
import { closeSync, openSync, readFileSync, statSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
export const PREFETCH_SUPERVISOR_SCRIPT = fileURLToPath(new URL('./prefetch-supervisor.mjs', import.meta.url));
|
||||
|
||||
const GROUPS = process.platform !== 'win32';
|
||||
const running = child => child.pid !== undefined && child.exitCode === null && child.signalCode === null;
|
||||
const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
|
||||
// Signals the child's whole process group (POSIX). Without groups the
|
||||
// supervisor is asked to stop its command through the pipe it watches, since
|
||||
// `kill` would end the supervisor at once and leave the command running; only
|
||||
// SIGKILL is sent to the supervisor itself, after it has had its chance.
|
||||
const signalTree = (child, signal) => {
|
||||
if (child.pid === undefined) return;
|
||||
if (!GROUPS) {
|
||||
if (!running(child)) return;
|
||||
if (signal === 'SIGKILL') child.kill(signal);
|
||||
else if (child.stdin?.writable) child.stdin.end('stop\n');
|
||||
return;
|
||||
}
|
||||
try { process.kill(-child.pid, signal); } catch (error) { if (error.code !== 'ESRCH') throw error; }
|
||||
};
|
||||
// True while any member of the child's group exists. On a platform without
|
||||
// groups only the child itself is known.
|
||||
const treeAlive = child => {
|
||||
if (child.pid === undefined) return false;
|
||||
if (!GROUPS) return running(child);
|
||||
try { process.kill(-child.pid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; }
|
||||
};
|
||||
|
||||
// Returns { name, child, done, durationMs }. `child` is the supervisor, whose
|
||||
// exit code or signal is the command's. `done` resolves on exit code 0 and
|
||||
// otherwise rejects with the tail of the log; the rejection is observed here
|
||||
// so an unawaited failure never surfaces as an unhandled rejection.
|
||||
export function startPrefetch({ name, cmd, args, cwd, env, logPath }) {
|
||||
const startedAt = Date.now();
|
||||
const log = openSync(logPath, 'w');
|
||||
let child;
|
||||
try {
|
||||
// stdin is the pipe the supervisor watches; nothing is ever written to it.
|
||||
child = spawn(process.execPath, [PREFETCH_SUPERVISOR_SCRIPT, cmd, ...args], { cwd, env, stdio: ['pipe', log, log], detached: GROUPS });
|
||||
} finally {
|
||||
closeSync(log); // the child holds its own descriptor
|
||||
}
|
||||
child.stdin?.on('error', () => {});
|
||||
const handle = { name, child, done: null, durationMs: null };
|
||||
let settled = false;
|
||||
handle.done = new Promise((settle, reject) => {
|
||||
const finish = failure => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
// The caller may be blocked in synchronous work when the child exits, so
|
||||
// this event can arrive long after the fact. When the child wrote
|
||||
// anything, its last write is the end time; both downloads this serves
|
||||
// print their final status line as they complete. A silent child falls
|
||||
// back to the time the exit was observed.
|
||||
let finishedAt = Date.now();
|
||||
try {
|
||||
const { mtimeMs, size } = statSync(logPath);
|
||||
if (size > 0) finishedAt = Math.min(finishedAt, mtimeMs);
|
||||
} catch {}
|
||||
handle.durationMs = Math.max(0, Math.round(finishedAt - startedAt));
|
||||
if (failure === null) return settle();
|
||||
let output = '';
|
||||
try { output = readFileSync(logPath, 'utf8').trim().slice(-2000); } catch {}
|
||||
reject(new Error(`${name} prefetch failed (${failure}): ${output}`));
|
||||
};
|
||||
child.once('error', error => finish(error.message));
|
||||
child.once('exit', (code, signal) => finish(code === 0 ? null : (signal ?? `exit ${code}`)));
|
||||
});
|
||||
handle.done.catch(() => {});
|
||||
return handle;
|
||||
}
|
||||
|
||||
// Stops every prefetch that is still running, together with any process it
|
||||
// started, and resolves only once all of them have exited, so the caller can
|
||||
// remove their working directory without racing a write. A tree that ignores
|
||||
// SIGTERM is killed after `graceMs`; one that still exists `graceMs` after
|
||||
// that is given up on so cleanup cannot hang.
|
||||
export async function stopPrefetches(prefetches, { graceMs = 5000 } = {}) {
|
||||
await Promise.all(prefetches.map(async ({ child, done }) => {
|
||||
const started = Date.now();
|
||||
let termed = false, killed = false;
|
||||
while (treeAlive(child)) {
|
||||
const elapsed = Date.now() - started;
|
||||
if (!termed) { signalTree(child, 'SIGTERM'); termed = true; }
|
||||
else if (!killed && elapsed >= graceMs) { signalTree(child, 'SIGKILL'); killed = true; }
|
||||
else if (killed && elapsed >= 2 * graceMs) break;
|
||||
await sleep(25);
|
||||
}
|
||||
try { await done; } catch {}
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
// Child side of prefetch-child.mjs: runs one prefetch command as the leader of
|
||||
// its own process group and stops the whole group when the parent that started
|
||||
// it is gone.
|
||||
//
|
||||
// The parent holds the write end of this process's stdin. The kernel closes
|
||||
// that end when the parent exits for any reason: a Ctrl-C delivered while the
|
||||
// parent is blocked in synchronous work, a plain `kill`, or SIGKILL. Reading
|
||||
// EOF here is therefore the only parent-death signal needed, and the parent
|
||||
// runs no signal handler of its own, so its default termination stays
|
||||
// immediate. The parent writes to the pipe for one reason only: to ask for a
|
||||
// deliberate stop where it cannot signal the group, which is Windows, where
|
||||
// `kill` ends a process outright and a signalled supervisor could not pass
|
||||
// anything on. Group-wide signals the parent sends through `stopPrefetches`
|
||||
// on POSIX reach this process too; it passes them on to the command and stays
|
||||
// alive until the command exits, so its own exit reports the command's
|
||||
// outcome (exit code, or the same signal).
|
||||
import { spawn } from 'node:child_process';
|
||||
import { fstatSync } from 'node:fs';
|
||||
import { constants } from 'node:os';
|
||||
|
||||
const GROUPS = process.platform !== 'win32';
|
||||
const GRACE_MS = Number(process.env.PAPERCLIP_PREFETCH_GRACE_MS) || 5000;
|
||||
const [cmd, ...args] = process.argv.slice(2);
|
||||
if (!cmd) {
|
||||
process.stderr.write('prefetch supervisor: usage: prefetch-supervisor.mjs <command> [args...]\n');
|
||||
process.exit(64);
|
||||
}
|
||||
|
||||
const child = spawn(cmd, args, { stdio: ['ignore', 'inherit', 'inherit'] });
|
||||
const signalChild = signal => { try { child.kill(signal); } catch {} };
|
||||
// On POSIX this process leads the group (the parent starts it detached), so
|
||||
// the group id is its own pid. Without groups only the command is known.
|
||||
const signalGroup = signal => {
|
||||
if (!GROUPS) return signalChild(signal);
|
||||
try { process.kill(-process.pid, signal); } catch (error) { if (error.code === 'ESRCH') signalChild(signal); }
|
||||
};
|
||||
|
||||
// A signal to the group arrives here as well as at the command; passing it on
|
||||
// is harmless when the command already has it, and covers a signal sent to
|
||||
// this process alone. The parent owns the SIGKILL escalation in that case.
|
||||
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => signalChild(signal));
|
||||
|
||||
let stopping = false;
|
||||
const stopGroup = reason => {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
process.stderr.write(`prefetch supervisor: ${reason}; stopping ${cmd}\n`);
|
||||
signalGroup('SIGTERM');
|
||||
setTimeout(() => signalGroup('SIGKILL'), GRACE_MS).unref();
|
||||
};
|
||||
// Only a pipe can carry a stop request or report the parent's exit (Node
|
||||
// creates its stdio pipes as socket pairs). Run by hand with a terminal or
|
||||
// /dev/null on stdin, there is no parent to watch and nothing is read.
|
||||
let stdinIsPipe = false;
|
||||
try { const stdin = fstatSync(0); stdinIsPipe = stdin.isFIFO() || stdin.isSocket(); } catch {}
|
||||
if (stdinIsPipe) {
|
||||
process.stdin.on('data', () => stopGroup('stop requested'));
|
||||
const parentGone = () => stopGroup('parent exited');
|
||||
process.stdin.on('end', parentGone);
|
||||
process.stdin.on('close', parentGone);
|
||||
process.stdin.on('error', parentGone);
|
||||
process.stdin.resume();
|
||||
}
|
||||
|
||||
child.on('error', error => {
|
||||
process.stderr.write(`prefetch supervisor: ${error.message}\n`);
|
||||
process.exit(127);
|
||||
});
|
||||
child.on('exit', (code, signal) => {
|
||||
if (signal !== null) {
|
||||
// Die of the same signal, so the parent sees the command's fate on this
|
||||
// process. Dropping the listener restores the default disposition first.
|
||||
process.removeAllListeners(signal);
|
||||
try { process.kill(process.pid, signal); } catch {}
|
||||
process.exit(128 + (constants.signals[signal] ?? 1));
|
||||
}
|
||||
process.exit(code ?? 1);
|
||||
});
|
||||
@@ -51,13 +51,14 @@ export function createBundledInstallManifest(publishManifest, bundledDependencie
|
||||
}
|
||||
|
||||
// npm consumers cannot inherit the workspace's pnpm overrides or patches.
|
||||
// Materialize the already-qualified Codex bridge closure in the tarball,
|
||||
// Materialize the already-qualified provider bridge closures in the tarball,
|
||||
// with overrides confined to the temporary, scripts-disabled producer graph.
|
||||
export function configureBundledProviderOverrides(installManifest, bundledDependencies, rootPackage, profileData) {
|
||||
const result = structuredClone(installManifest);
|
||||
const selected = [];
|
||||
for (const [agent, serverPackage, runtimePackage] of [
|
||||
["codex", "@agentclientprotocol/codex-acp", "@openai/codex"],
|
||||
["claude", "@agentclientprotocol/claude-agent-acp", "@anthropic-ai/claude-agent-sdk"],
|
||||
]) {
|
||||
if (!bundledDependencies.includes(serverPackage)) continue;
|
||||
const profile = profileData?.profiles?.[agent];
|
||||
@@ -75,7 +76,9 @@ export function configureBundledProviderOverrides(installManifest, bundledDepend
|
||||
if (result.overrides?.[npmSelector] !== undefined) {
|
||||
throw new Error(`Bundled ${agent} runtime has a conflicting producer override`);
|
||||
}
|
||||
const overrides = { [runtimePackage]: profile.agentRuntimeVersion };
|
||||
const overrides = { [runtimePackage]: profile.agentRuntimeVersion,
|
||||
...(agent === "claude" ? { "@agentclientprotocol/sdk": "1.4.0", zod: "4.4.3" } : {}),
|
||||
};
|
||||
result.overrides = { ...result.overrides, [npmSelector]: overrides };
|
||||
selected.push({ agent, ...profile });
|
||||
}
|
||||
@@ -87,88 +90,105 @@ const inside = (root, candidate) => {
|
||||
return value !== "" && value !== ".." && !value.startsWith(`..${sep}`) && !isAbsolute(value);
|
||||
};
|
||||
|
||||
// Bundle the pinned JavaScript closure, never the producer's Codex executable.
|
||||
// Bundle the pinned JavaScript closure, never the producer's executable.
|
||||
// npm installs the official optional package for the consumer's own platform.
|
||||
export function stageBundledProviderOptionalDependencies(destinationDir, publishManifest, profiles) {
|
||||
const graphRoot = resolve(destinationDir, "node_modules");
|
||||
if (realpathSync(graphRoot) !== graphRoot) throw new Error("Bundled Codex graph must be a canonical owned directory");
|
||||
const result = structuredClone(publishManifest), remove = new Set(), optional = {}, normalize = new Map();
|
||||
const profile = profiles.find(value => value.agent === "codex");
|
||||
if (profiles.length !== 1 || !profile) throw new Error("Bundled provider graph must contain its qualified Codex profile");
|
||||
const targets = ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64", "win32-x64", "win32-arm64"];
|
||||
const declarations = Object.fromEntries(targets.map(target => [`${profile.agentRuntimePackage}-${target}`,
|
||||
`npm:${profile.agentRuntimePackage}@${profile.agentRuntimeVersion}-${target}`]));
|
||||
const ownedDirectory = directory => {
|
||||
const stat = lstatSync(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink() || !inside(graphRoot, realpathSync(directory))) {
|
||||
throw new Error("Bundled Codex dependency escapes its producer graph");
|
||||
}
|
||||
};
|
||||
const manifestAt = directory => {
|
||||
const path = resolve(directory, "package.json"), stat = lstatSync(path);
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 256 * 1024) {
|
||||
throw new Error("Bundled Codex dependency manifest must be a bounded regular file");
|
||||
}
|
||||
return JSON.parse(readFileSync(path, "utf8"));
|
||||
};
|
||||
const bridgeDirectory = resolve(graphRoot, profile.agentServerPackage);
|
||||
ownedDirectory(bridgeDirectory);
|
||||
const bridge = manifestAt(bridgeDirectory);
|
||||
if (bridge.name !== profile.agentServerPackage || bridge.version !== profile.agentServerVersion) throw new Error("Bundled Codex bridge version mismatch");
|
||||
const issuer = createRequire(resolve(bridgeDirectory, "package.json"));
|
||||
const runtimeManifest = realpathSync(issuer.resolve(`${profile.agentRuntimePackage}/package.json`));
|
||||
if (!inside(graphRoot, runtimeManifest)) throw new Error("Bundled Codex runtime escapes its producer graph");
|
||||
let runtimeCount = 0, packageCount = 0;
|
||||
const pending = [{ directory: graphRoot, depth: 0 }];
|
||||
while (pending.length) {
|
||||
const { directory, depth } = pending.pop();
|
||||
if (depth > 64) throw new Error("Bundled Codex graph exceeds its depth limit");
|
||||
const packages = [];
|
||||
for (const entry of readdirSync(directory)) {
|
||||
if (entry.startsWith(".")) continue;
|
||||
const candidate = resolve(directory, entry); ownedDirectory(candidate);
|
||||
if (entry.startsWith("@")) {
|
||||
for (const child of readdirSync(candidate)) {
|
||||
const scoped = resolve(candidate, child); ownedDirectory(scoped); packages.push(scoped);
|
||||
}
|
||||
} else packages.push(candidate);
|
||||
}
|
||||
for (const packageDirectory of packages) {
|
||||
if (++packageCount > 20_000) throw new Error("Bundled Codex graph exceeds its package limit");
|
||||
const metadata = manifestAt(packageDirectory);
|
||||
const binding = basename(dirname(packageDirectory)) === "@openai" ? `@openai/${basename(packageDirectory)}` : null;
|
||||
if (binding?.startsWith("@openai/codex-")) {
|
||||
const target = binding.slice("@openai/codex-".length), [os, cpu] = target.split("-");
|
||||
if (!Object.hasOwn(declarations, binding) || ![binding, profile.agentRuntimePackage].includes(metadata.name)
|
||||
|| metadata.version !== `${profile.agentRuntimeVersion}-${target}` || !metadata.os?.includes(os) || !metadata.cpu?.includes(cpu)) {
|
||||
throw new Error("Bundled Codex installed platform identity mismatch");
|
||||
}
|
||||
remove.add(packageDirectory);
|
||||
continue;
|
||||
}
|
||||
const nested = resolve(packageDirectory, "node_modules");
|
||||
if (lstatExists(nested)) { ownedDirectory(nested); pending.push({ directory: nested, depth: depth + 1 }); }
|
||||
if (binding !== profile.agentRuntimePackage) continue;
|
||||
if (metadata.name !== profile.agentRuntimePackage || metadata.version !== profile.agentRuntimeVersion) throw new Error("Bundled Codex runtime version mismatch");
|
||||
runtimeCount++;
|
||||
const delegated = metadata.optionalDependencies === undefined
|
||||
&& Object.entries(declarations).every(([name, specifier]) => result.optionalDependencies?.[name] === specifier);
|
||||
if (!delegated && (Object.keys(metadata.optionalDependencies ?? {}).length !== targets.length
|
||||
|| targets.some(target => metadata.optionalDependencies?.[`${profile.agentRuntimePackage}-${target}`] !== declarations[`${profile.agentRuntimePackage}-${target}`]))) {
|
||||
throw new Error("Bundled Codex platform declaration is not qualified");
|
||||
}
|
||||
if (!delegated) {
|
||||
const normalized = { ...metadata }; delete normalized.optionalDependencies;
|
||||
normalize.set(resolve(packageDirectory, "package.json"), normalized);
|
||||
}
|
||||
const vendor = resolve(packageDirectory, "vendor");
|
||||
if (lstatExists(vendor)) { ownedDirectory(vendor); remove.add(vendor); }
|
||||
}
|
||||
if (!profiles.length || new Set(profiles.map(profile => profile.agent)).size !== profiles.length) {
|
||||
throw new Error("Bundled provider graph must contain distinct qualified profiles");
|
||||
}
|
||||
if (!runtimeCount) throw new Error("Bundled Codex runtime omitted its platform declarations");
|
||||
for (const [name, specifier] of Object.entries(declarations)) {
|
||||
if (result.optionalDependencies?.[name] !== undefined && result.optionalDependencies[name] !== specifier) throw new Error(`Bundled Codex platform conflicts with published dependency: ${name}`);
|
||||
optional[name] = specifier;
|
||||
for (const profile of profiles) {
|
||||
const expected = profile.agent === "codex" ? ["@agentclientprotocol/codex-acp", "@openai/codex"]
|
||||
: profile.agent === "claude" ? ["@agentclientprotocol/claude-agent-acp", "@anthropic-ai/claude-agent-sdk"] : null;
|
||||
if (!expected || profile.agentServerPackage !== expected[0] || profile.agentRuntimePackage !== expected[1]) {
|
||||
throw new Error("Bundled provider graph contains an unknown provider profile");
|
||||
}
|
||||
const label = profile.agent === "codex" ? "Codex" : "Claude";
|
||||
const targets = ["linux-x64", "linux-arm64", "darwin-x64", "darwin-arm64", "win32-x64", "win32-arm64",
|
||||
...(profile.agent === "claude" ? ["linux-x64-musl", "linux-arm64-musl"] : [])];
|
||||
const declarations = Object.fromEntries(targets.map(target => [`${profile.agentRuntimePackage}-${target}`,
|
||||
profile.agent === "codex" ? `npm:${profile.agentRuntimePackage}@${profile.agentRuntimeVersion}-${target}` : profile.agentRuntimeVersion]));
|
||||
const ownedDirectory = directory => {
|
||||
const stat = lstatSync(directory);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink() || !inside(graphRoot, realpathSync(directory))) {
|
||||
throw new Error(`Bundled ${label} dependency escapes its producer graph`);
|
||||
}
|
||||
};
|
||||
const manifestAt = directory => {
|
||||
const path = resolve(directory, "package.json"), stat = lstatSync(path);
|
||||
if (!stat.isFile() || stat.isSymbolicLink() || stat.nlink !== 1 || stat.size > 256 * 1024) {
|
||||
throw new Error(`Bundled ${label} dependency manifest must be a bounded regular file`);
|
||||
}
|
||||
return JSON.parse(readFileSync(path, "utf8"));
|
||||
};
|
||||
const bridgeDirectory = resolve(graphRoot, profile.agentServerPackage);
|
||||
ownedDirectory(bridgeDirectory);
|
||||
const bridge = manifestAt(bridgeDirectory);
|
||||
if (bridge.name !== profile.agentServerPackage || bridge.version !== profile.agentServerVersion) throw new Error(`Bundled ${label} bridge version mismatch`);
|
||||
const issuer = createRequire(resolve(bridgeDirectory, "package.json"));
|
||||
let runtimeManifest;
|
||||
try { runtimeManifest = issuer.resolve(`${profile.agentRuntimePackage}/package.json`); }
|
||||
catch (error) {
|
||||
if (error.code !== "ERR_PACKAGE_PATH_NOT_EXPORTED") throw error;
|
||||
runtimeManifest = resolve(dirname(issuer.resolve(profile.agentRuntimePackage)), "package.json");
|
||||
}
|
||||
runtimeManifest = realpathSync(runtimeManifest);
|
||||
if (!inside(graphRoot, runtimeManifest)) throw new Error(`Bundled ${label} runtime escapes its producer graph`);
|
||||
let runtimeCount = 0, packageCount = 0;
|
||||
const pending = [{ directory: graphRoot, depth: 0 }];
|
||||
while (pending.length) {
|
||||
const { directory, depth } = pending.pop();
|
||||
if (depth > 64) throw new Error(`Bundled ${label} graph exceeds its depth limit`);
|
||||
const packages = [];
|
||||
for (const entry of readdirSync(directory)) {
|
||||
if (entry.startsWith(".")) continue;
|
||||
const candidate = resolve(directory, entry); ownedDirectory(candidate);
|
||||
if (entry.startsWith("@")) {
|
||||
for (const child of readdirSync(candidate)) {
|
||||
const scoped = resolve(candidate, child); ownedDirectory(scoped); packages.push(scoped);
|
||||
}
|
||||
} else packages.push(candidate);
|
||||
}
|
||||
for (const packageDirectory of packages) {
|
||||
if (++packageCount > 20_000) throw new Error(`Bundled ${label} graph exceeds its package limit`);
|
||||
const metadata = manifestAt(packageDirectory);
|
||||
const scope = basename(dirname(packageDirectory));
|
||||
const binding = scope.startsWith("@") ? `${scope}/${basename(packageDirectory)}` : basename(packageDirectory);
|
||||
if (binding.startsWith(`${profile.agentRuntimePackage}-`)) {
|
||||
const target = binding.slice(profile.agentRuntimePackage.length + 1), [os, cpu] = target.split("-");
|
||||
if (!Object.hasOwn(declarations, binding) || !(metadata.name === binding || (profile.agent === "codex" && metadata.name === profile.agentRuntimePackage))
|
||||
|| metadata.version !== (profile.agent === "codex" ? `${profile.agentRuntimeVersion}-${target}` : profile.agentRuntimeVersion) || !metadata.os?.includes(os) || !metadata.cpu?.includes(cpu)) {
|
||||
throw new Error(`Bundled ${label} installed platform identity mismatch`);
|
||||
}
|
||||
remove.add(packageDirectory);
|
||||
continue;
|
||||
}
|
||||
const nested = resolve(packageDirectory, "node_modules");
|
||||
if (lstatExists(nested)) { ownedDirectory(nested); pending.push({ directory: nested, depth: depth + 1 }); }
|
||||
if (binding !== profile.agentRuntimePackage) continue;
|
||||
if (metadata.name !== profile.agentRuntimePackage || metadata.version !== profile.agentRuntimeVersion) throw new Error(`Bundled ${label} runtime version mismatch`);
|
||||
runtimeCount++;
|
||||
const delegated = metadata.optionalDependencies === undefined
|
||||
&& Object.entries(declarations).every(([name, specifier]) => result.optionalDependencies?.[name] === specifier);
|
||||
if (!delegated && (Object.keys(metadata.optionalDependencies ?? {}).length !== targets.length
|
||||
|| targets.some(target => metadata.optionalDependencies?.[`${profile.agentRuntimePackage}-${target}`] !== declarations[`${profile.agentRuntimePackage}-${target}`]))) {
|
||||
throw new Error(`Bundled ${label} platform declaration is not qualified`);
|
||||
}
|
||||
if (!delegated) {
|
||||
const normalized = { ...metadata }; delete normalized.optionalDependencies;
|
||||
normalize.set(resolve(packageDirectory, "package.json"), normalized);
|
||||
}
|
||||
const vendor = resolve(packageDirectory, "vendor");
|
||||
if (profile.agent === "codex" && lstatExists(vendor)) { ownedDirectory(vendor); remove.add(vendor); }
|
||||
}
|
||||
}
|
||||
if (!runtimeCount) throw new Error(`Bundled ${label} runtime omitted its platform declarations`);
|
||||
for (const [name, specifier] of Object.entries(declarations)) {
|
||||
if (result.optionalDependencies?.[name] !== undefined && result.optionalDependencies[name] !== specifier) throw new Error(`Bundled ${label} platform conflicts with published dependency: ${name}`);
|
||||
optional[name] = specifier;
|
||||
}
|
||||
}
|
||||
// A bundled wrapper's optional edges make npm treat missing platform slots
|
||||
// as bundled too. Delegate those exact declarations to the unbundled server
|
||||
@@ -178,7 +198,7 @@ export function stageBundledProviderOptionalDependencies(destinationDir, publish
|
||||
for (const directory of remove) rmSync(directory, { recursive: true, force: true });
|
||||
result.optionalDependencies = { ...result.optionalDependencies, ...optional };
|
||||
for (const field of ["bundleDependencies", "bundledDependencies"]) {
|
||||
if (Array.isArray(result[field])) result[field] = result[field].filter(name => !Object.hasOwn(declarations, name));
|
||||
if (Array.isArray(result[field])) result[field] = result[field].filter(name => !Object.hasOwn(optional, name));
|
||||
}
|
||||
delete result.paperclipProviderArtifacts;
|
||||
return result;
|
||||
|
||||
@@ -117,9 +117,27 @@ function excludedServerSuites(groupName) {
|
||||
}
|
||||
const generalWorkspacesAGroupName = "general-workspaces-a";
|
||||
const generalWorkspacesBGroupName = "general-workspaces-b";
|
||||
const generalWorkspacesCGroupName = "general-workspaces-c";
|
||||
const generalWorkspacesAProjects = ["@paperclipai/ui", "paperclipai"];
|
||||
const generalWorkspacesBProjects = nonServerProjects.filter((project) => !generalWorkspacesAProjects.includes(project));
|
||||
const generalGroupNames = [generalServerGroupName, generalWorkspacesAGroupName, generalWorkspacesBGroupName];
|
||||
// The db project runs in its own lane. Its vitest invocation took 168-250s
|
||||
// of a 436-513s workspaces-b job (PR runs 38084171761, 38083084276 and
|
||||
// 38083020834, 2026-10-10), which made workspaces-b the slowest check once
|
||||
// the server shards were rebalanced. Vitest's native --shard would not help
|
||||
// here: the project is already file-parallel and client.test.ts alone is
|
||||
// 117-189s of its wall time, so a hash-ordered slice can land both of its
|
||||
// slow suites on one runner. Splitting by project is deterministic and
|
||||
// leaves the remaining projects at ~265-275s of vitest time, roughly level
|
||||
// with this lane.
|
||||
const generalWorkspacesCProjects = ["@paperclipai/db"];
|
||||
const generalWorkspacesBProjects = nonServerProjects.filter(
|
||||
(project) => !generalWorkspacesAProjects.includes(project) && !generalWorkspacesCProjects.includes(project),
|
||||
);
|
||||
const generalGroupNames = [
|
||||
generalServerGroupName,
|
||||
generalWorkspacesAGroupName,
|
||||
generalWorkspacesBGroupName,
|
||||
generalWorkspacesCGroupName,
|
||||
];
|
||||
const allowedGeneralGroupNames = [
|
||||
...generalGroupNames,
|
||||
generalServerWithoutChatGroupName,
|
||||
@@ -520,6 +538,11 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount =
|
||||
return;
|
||||
}
|
||||
|
||||
if (groupName === generalWorkspacesCGroupName) {
|
||||
runProjectGroup(generalWorkspacesCProjects, groupName);
|
||||
return;
|
||||
}
|
||||
|
||||
fail(`Unknown group "${groupName}".`);
|
||||
}
|
||||
|
||||
@@ -604,7 +627,9 @@ if (options.dryRun) {
|
||||
? generalWorkspacesAProjects
|
||||
: options.group === generalWorkspacesBGroupName
|
||||
? generalWorkspacesBProjects
|
||||
: null,
|
||||
: options.group === generalWorkspacesCGroupName
|
||||
? generalWorkspacesCProjects
|
||||
: null,
|
||||
workspacesVitestShard:
|
||||
options.group === generalWorkspacesAGroupName &&
|
||||
options.shardCount !== null &&
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
// The public-install verifier stages three packages that bundle dependencies
|
||||
// (`@paperclipai/server`, `@paperclipai/adapter-utils`, `@paperclipai/db`).
|
||||
// Each one is a host-side `npm install` of that package's bundled closure into
|
||||
// its own pack directory, and none reads what another produces, so they run as
|
||||
// concurrent children instead of one after another inside the staging loop.
|
||||
// Each child is the CLI entry of `prepare-bundled-package.mjs`, the same path
|
||||
// release.sh runs, with its output in a log file that a failure reports.
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { startPrefetch } from './prefetch-child.mjs';
|
||||
|
||||
export const PREPARE_BUNDLED_PACKAGE_SCRIPT = fileURLToPath(new URL('./prepare-bundled-package.mjs', import.meta.url));
|
||||
|
||||
// `@paperclipai/adapter-utils` -> `adapter_utils`, for log names and timing keys.
|
||||
export const bundledPackageSlug = name => name.replace(/^@paperclipai\//, '').replace(/[^a-z0-9]+/gi, '_');
|
||||
|
||||
// Starts one child per entry (`{ name, stagedSource, target }`) at once and
|
||||
// returns the entries with their `slug` and prefetch `handle`. The caller joins
|
||||
// `handle.done` where the packed output is needed and registers the handles
|
||||
// for cleanup, so an earlier failure stops every install still running.
|
||||
export function startBundledInstalls(entries, { logDir, cwd, env = process.env, script = PREPARE_BUNDLED_PACKAGE_SCRIPT }) {
|
||||
return entries.map(entry => {
|
||||
const slug = bundledPackageSlug(entry.name);
|
||||
const handle = startPrefetch({ name: `bundled-${slug}`, cmd: process.execPath, args: [script, entry.stagedSource, entry.target], cwd, env, logPath: join(logDir, `bundled-${slug}.log`) });
|
||||
return { ...entry, slug, handle };
|
||||
});
|
||||
}
|
||||
|
||||
// Per-package wall time for the timing line. The helper measures from the
|
||||
// start to the child's last output; the parent is idle while it waits for the
|
||||
// installs, so the exit is observed as it happens.
|
||||
export const bundledInstallTimings = installs => Object.fromEntries(installs.map(({ slug, handle }) => [`stage_bundled_${slug}_ms`, handle.durationMs]));
|
||||
@@ -4,15 +4,18 @@
|
||||
// explicit and separate from npm installation, and is removed in finally.
|
||||
import assert from 'node:assert/strict';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
||||
import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, renameSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { createServer } from 'node:net';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { basename, dirname, join, resolve } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { materializePublishManifest, prepareBundledPackage } from './prepare-bundled-package.mjs';
|
||||
import { packConcurrency, packStagedPackages, packTimings } from './pack-staged-packages.mjs';
|
||||
import { materializePublishManifest } from './prepare-bundled-package.mjs';
|
||||
import { GROK_PUBLIC_INSTALL_IMAGE, GROK_PUBLIC_INSTALL_LIFECYCLE, assertNoBundledCodexPayloads, grokConsumerDockerArgs, installedCodexProbeSource, installedDaemonProbeSource, macPublicInstallLifecycleProfile, macInstalledStartupProbeSource } from './grok-public-install-sandbox.mjs';
|
||||
import { startPrefetch, stopPrefetches } from './prefetch-child.mjs';
|
||||
import { retainRunnerQualificationPackages } from './retain-runner-qualification-packages.mjs';
|
||||
import { bundledInstallTimings, startBundledInstalls } from './stage-bundled-packages.mjs';
|
||||
const repo = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
||||
async function verifyMacConsumer(packageDirectory, sourceRevision, receiptPath) {
|
||||
assert.equal(process.env.GITHUB_ACTIONS, 'true', 'Mac clean consumers run only on disposable hosted CI');
|
||||
@@ -100,7 +103,36 @@ const env = { ...process.env, NODE_PATH: '', PAPERCLIP_RELEASE_REUSE_UI_DIST: '1
|
||||
const run = (cmd, args, cwd = root) => execFileSync(cmd, args, { cwd, env, stdio: 'pipe', maxBuffer: 32 * 1024 * 1024 });
|
||||
const sourceRevision = run('git', ['rev-parse', 'HEAD'], repo).toString().trim();
|
||||
const releaseVersion = `0.0.0-grok-verify.${sourceRevision.slice(0, 12)}`;
|
||||
// The two network downloads the probes need do not depend on anything packed
|
||||
// below, so they overlap the CPU-bound staging instead of running after it.
|
||||
// The consumer image is pinned by digest and the Grok binary is checked against
|
||||
// its pinned sha256 by the provisioning script, so an early start changes no
|
||||
// verified property. Neither touches the consumer, the assets, nor the
|
||||
// `native/grok` path that the post-install assertion inspects.
|
||||
const prefetched = join(root, 'prefetch'); mkdirSync(prefetched);
|
||||
const stagedGrok = join(prefetched, 'grok');
|
||||
// Every background child, downloads and bundled-dependency installs alike, is
|
||||
// stopped in `finally`; `downloads` is the subset the timing line reports as
|
||||
// prefetches.
|
||||
const prefetches = [];
|
||||
const downloads = [];
|
||||
const timing = {};
|
||||
// A prefetch failure surfaces where its result is awaited, never as an
|
||||
// unhandled rejection that would mask an earlier staging error.
|
||||
const prefetch = (name, cmd, args) => {
|
||||
const handle = startPrefetch({ name, cmd, args, cwd: root, env, logPath: join(prefetched, `${name}.log`) });
|
||||
prefetches.push(handle);
|
||||
downloads.push(handle);
|
||||
return handle.done;
|
||||
};
|
||||
let phaseStarted = performance.now();
|
||||
const phase = name => { const now = performance.now(); timing[`${name}_ms`] = Math.round(now - phaseStarted); phaseStarted = now; };
|
||||
const awaited = async (name, promise) => { const started = performance.now(); await promise; timing[`${name}_wait_ms`] = Math.round(performance.now() - started); };
|
||||
try {
|
||||
const imageReady = prefetch('image', 'docker', ['pull', '--platform', 'linux/amd64', GROK_PUBLIC_INSTALL_IMAGE]);
|
||||
// Provision as the unprivileged verification user, never into the host's /opt.
|
||||
// Only the positive probe sees this file at the canonical sandbox path.
|
||||
const grokReady = prefetch('grok', process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), stagedGrok]);
|
||||
const listing = run(process.execPath, [join(repo, 'scripts/release-package-map.mjs'), 'list'], repo).toString().trim().split('\n').map(line => line.split('\t'));
|
||||
const packages = new Map(listing.map(([dir, name]) => [name, { dir, manifest: JSON.parse(readFileSync(join(repo, dir, 'package.json'), 'utf8')) }]));
|
||||
const needed = new Set();
|
||||
@@ -113,15 +145,47 @@ try {
|
||||
}
|
||||
visit('@paperclipai/server');
|
||||
visit('paperclipai');
|
||||
// release.sh also builds the standalone public packages (the sandbox-provider
|
||||
// plugins outside the pnpm workspace) before it stages anything, because it
|
||||
// publishes them. Nothing staged here reaches them: `needed` is closed under
|
||||
// `workspace:` specs, so every package staged below is a workspace package
|
||||
// that `pnpm build` already built, and that build would only cost one
|
||||
// network install and compile per plugin whose output no tarball reads.
|
||||
// Canary Dry Run still runs it through release.sh. The one path that does
|
||||
// read a standalone build is qualification-package retention at the end:
|
||||
// it packs the Daytona plugin from its `dist`, which only this build
|
||||
// produces, so the build runs when retention is requested and nowhere else.
|
||||
const retainQualificationPackages = Boolean(process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR);
|
||||
if (retainQualificationPackages) {
|
||||
const standaloneStarted = performance.now();
|
||||
run(process.execPath, [join(repo, 'scripts/build-standalone-public-packages.mjs')], repo);
|
||||
timing.stage_standalone_build_ms = Math.round(performance.now() - standaloneStarted);
|
||||
}
|
||||
// The server's `ui-dist` payload is prepared here; with the reuse flag set
|
||||
// above, that copies the `ui/dist` that `pnpm build` produced instead of
|
||||
// building the UI again.
|
||||
const uiDistStarted = performance.now();
|
||||
run('bash', [join(repo, 'scripts/prepare-server-ui-dist.sh')], repo);
|
||||
timing.stage_ui_dist_ms = Math.round(performance.now() - uiDistStarted);
|
||||
// Match release.sh's unified versioning in temporary staging directories.
|
||||
// Source manifests remain untouched, including independently versioned SDKs.
|
||||
run(process.execPath, [join(repo, 'scripts/build-standalone-public-packages.mjs')], repo);
|
||||
run('bash', [join(repo, 'scripts/prepare-server-ui-dist.sh')], repo);
|
||||
const tarballs = [];
|
||||
for (const [index, name] of [...needed].entries()) {
|
||||
// Each package's published files are staged into `source-<index>` and its
|
||||
// pack input into `package-<index>`, with <index> in dependency order so the
|
||||
// tarball list keeps matching `needed`. The packages that bundle dependencies
|
||||
// each need a host-side `npm install` of their bundled closure; those installs
|
||||
// write only their own `package-<index>` and read nothing another package
|
||||
// produces, so they start first and run together while the other packages
|
||||
// are copied, instead of one after another inside one loop. Packing is a
|
||||
// Node start-up plus CPU-bound gzip per package that reads only the package's
|
||||
// own directory, so the packs also run together, each into its own
|
||||
// `pack-<index>`, and the tarballs are checked in order once all are made.
|
||||
const entries = [...needed].map((name, index) => {
|
||||
const { dir, manifest } = packages.get(name);
|
||||
const target = join(root, `package-${index}`); mkdirSync(target);
|
||||
const stagedSource = join(root, `source-${index}`); mkdirSync(stagedSource);
|
||||
const bundled = (manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length > 0;
|
||||
return { name, index, dir, manifest, bundled, target: join(root, `package-${index}`), stagedSource: join(root, `source-${index}`), destination: join(root, `pack-${index}`) };
|
||||
});
|
||||
const stageSource = ({ dir, manifest, stagedSource }) => {
|
||||
mkdirSync(stagedSource);
|
||||
for (const file of manifest.files ?? ['dist']) {
|
||||
// release.sh stages runtime skills into these public packages before pack.
|
||||
const releaseSkills = file === 'skills' && ['server', 'packages/adapters/claude-local', 'packages/adapters/codex-local'].includes(dir);
|
||||
@@ -132,19 +196,38 @@ try {
|
||||
}
|
||||
const releaseManifest = { ...manifest, version: releaseVersion };
|
||||
writeFileSync(join(stagedSource, 'package.json'), JSON.stringify(releaseManifest));
|
||||
if ((manifest.bundleDependencies ?? manifest.bundledDependencies ?? []).length) {
|
||||
prepareBundledPackage(stagedSource, target);
|
||||
} else {
|
||||
cpSync(stagedSource, target, { recursive: true });
|
||||
writeFileSync(join(target, 'package.json'), JSON.stringify(materializePublishManifest(releaseManifest)));
|
||||
}
|
||||
run('npm', ['pack', '--ignore-scripts', '--pack-destination', root], target);
|
||||
const packed = readdirSync(root).filter(f => f.endsWith('.tgz') && !tarballs.includes(join(root, f)));
|
||||
assert.equal(packed.length, 1);
|
||||
const tarball = join(root, packed[0]);
|
||||
assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).toString().trim().split('\n'));
|
||||
tarballs.push(tarball);
|
||||
return releaseManifest;
|
||||
};
|
||||
const sourcesStarted = performance.now();
|
||||
for (const entry of entries.filter(entry => entry.bundled)) stageSource(entry);
|
||||
const installsStarted = performance.now();
|
||||
const installs = startBundledInstalls(entries.filter(entry => entry.bundled), { logDir: prefetched, cwd: repo });
|
||||
prefetches.push(...installs.map(({ handle }) => handle));
|
||||
for (const entry of entries.filter(entry => !entry.bundled)) {
|
||||
const releaseManifest = stageSource(entry);
|
||||
mkdirSync(entry.target);
|
||||
cpSync(entry.stagedSource, entry.target, { recursive: true });
|
||||
writeFileSync(join(entry.target, 'package.json'), JSON.stringify(materializePublishManifest(releaseManifest)));
|
||||
}
|
||||
timing.stage_sources_ms = Math.round(performance.now() - sourcesStarted);
|
||||
await awaited('bundled', Promise.all(installs.map(({ handle }) => handle.done)));
|
||||
timing.stage_bundled_ms = Math.round(performance.now() - installsStarted);
|
||||
const packStarted = performance.now();
|
||||
// The bundled packages make the largest tarballs, so they are launched first
|
||||
// and the short packs fill the remaining slots behind them. Each pack is
|
||||
// registered for cleanup as it starts, so a failure stops the ones still
|
||||
// running before the scratch root is removed. The results are put back into
|
||||
// dependency order before the tarballs are checked and copied.
|
||||
const concurrency = packConcurrency();
|
||||
const packs = await packStagedPackages([...entries].sort((a, b) => Number(b.bundled) - Number(a.bundled)), { logDir: prefetched, env, concurrency, register: handle => prefetches.push(handle) });
|
||||
packs.sort((a, b) => a.index - b.index);
|
||||
const checksStarted = performance.now();
|
||||
const tarballs = packs.map(({ tarball }) => tarball);
|
||||
for (const tarball of tarballs) assertNoBundledCodexPayloads(run('tar', ['-tzf', tarball]).toString().trim().split('\n'));
|
||||
timing.stage_pack_check_ms = Math.round(performance.now() - checksStarted);
|
||||
timing.stage_pack_ms = Math.round(performance.now() - packStarted);
|
||||
Object.assign(timing, packTimings(packs, concurrency));
|
||||
|
||||
// Preserve the exact Linux-produced public tarballs for independent Mac
|
||||
// consumers even if a later Linux runtime probe fails. This is a packaging
|
||||
// receipt, never an installation or provider-admission pass.
|
||||
@@ -153,7 +236,6 @@ try {
|
||||
assert.equal(JSON.parse(readFileSync(join(repo, 'server/dist/build-info.json'))).commit, sourceRevision);
|
||||
assert.ok(!existsSync(output), 'Never replace retained public packages');
|
||||
mkdirSync(output, { recursive: true });
|
||||
const { createHash } = await import('node:crypto');
|
||||
const archives = [...needed].map((name, index) => {
|
||||
const file = basename(tarballs[index]); cpSync(tarballs[index], join(output, file));
|
||||
return { name, file, sha256: createHash('sha256').update(readFileSync(tarballs[index])).digest('hex') };
|
||||
@@ -180,9 +262,12 @@ try {
|
||||
const sentinel = join(consumer, 'node_modules/paperclip-verification-lifecycle-sentinel/lifecycle-ran');
|
||||
const consumerUid = process.getuid();
|
||||
const isolated = (command, options = {}) => run('docker', grokConsumerDockerArgs({ assets, consumer, cache, uid: consumerUid, gid: process.getgid(), command, ...options }));
|
||||
phase('stage_packages');
|
||||
await awaited('image', imageReady);
|
||||
// npm resolution is intentionally the public consumer graph, not the pnpm
|
||||
// workspace graph. Freeze that result before any lifecycle script can run.
|
||||
isolated(['npm', 'install', '--ignore-scripts', '--omit=dev', ...readdirSync(assets).filter(file => file.endsWith('.tgz')).map(file => `/packages/${file}`)], { download: true });
|
||||
phase('consumer_install');
|
||||
assert.equal(existsSync(sentinel), false, 'Dependency download must not run lifecycle scripts');
|
||||
const consumerLock = readFileSync(join(consumer, 'package-lock.json'), 'utf8');
|
||||
// npm ci rejects bundled optional platform dependencies absent from its own
|
||||
@@ -191,6 +276,7 @@ try {
|
||||
isolated(GROK_PUBLIC_INSTALL_LIFECYCLE);
|
||||
assert.equal(readFileSync(sentinel, 'utf8'), 'ok', 'Offline lifecycle scripts must actually execute');
|
||||
assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Lifecycle execution must preserve the resolved consumer lock');
|
||||
phase('lifecycle_rebuild');
|
||||
for (const name of needed) {
|
||||
const installedManifest = JSON.parse(readFileSync(join(consumer, 'node_modules', name, 'package.json'), 'utf8'));
|
||||
assert.equal(installedManifest.version, releaseVersion, `Installed release version for ${name}`);
|
||||
@@ -203,6 +289,7 @@ try {
|
||||
writeFileSync(join(assets, 'codex-probe.mjs'), installedCodexProbeSource(
|
||||
'/consumer/node_modules/@paperclipai/server/dist/vendor/paperclip-runner/index.js', '/consumer', codexVersion), { mode: 0o644 });
|
||||
const codex = JSON.parse(isolated(['node', '/packages/codex-probe.mjs']).toString().trim());
|
||||
phase('codex_probe');
|
||||
assert.equal(codex.codexConsumerHostOnly, true);
|
||||
assert.equal(codex.codexPlatformPackageSource, 'official npm');
|
||||
assert.ok(existsSync(join(installed, 'providers/grok/launcher.cjs')));
|
||||
@@ -221,10 +308,14 @@ try {
|
||||
`;
|
||||
writeFileSync(join(assets, 'probe.mjs'), probe, { mode: 0o644 });
|
||||
isolated(['node', '/packages/probe.mjs', 'missing']);
|
||||
// Provision as the unprivileged verification user, never into the host's /opt.
|
||||
// Only the positive probe sees this file at the canonical sandbox path.
|
||||
run(process.execPath, [join(repo, 'packages/paperclip-runner/scripts/provision-grok.mjs'), prerequisite]);
|
||||
// The binary was provisioned outside `native/grok` while the packages were
|
||||
// staged, so the assertion above still proves npm put nothing there. It moves
|
||||
// into place only now, so only the positive probe sees it.
|
||||
await awaited('grok', grokReady);
|
||||
mkdirSync(dirname(prerequisite), { recursive: true });
|
||||
renameSync(stagedGrok, prerequisite);
|
||||
isolated(['node', '/packages/probe.mjs', 'present'], { prerequisite });
|
||||
phase('grok_probes');
|
||||
console.log(JSON.stringify({ schema: 'paperclip.grok.public-npm-install.v1', sourceRevision, releaseVersion, lifecycleScriptsEnabled: true, lifecycleSentinelVerified: true, lifecycleNetwork: 'none', consumerImage: GROK_PUBLIC_INSTALL_IMAGE, consumerUid, consumerLockPreserved: true, cleanNpmInstall: true, packageCount: needed.size, codexPackedPayloadsOmitted: true, codexPackedPackagesChecked: tarballs.length, ...codex, builtinLauncherPresent: true, separateGrokPackage: false, npmProvisionedBinary: false, missingPrerequisiteRejected: true, provisionedBinaryVerified: true, commandLeaseVerified: true, providerCalls: 0 }));
|
||||
// Exercise Pi's public CLI and installed server, never a private workspace
|
||||
// package or binary override. Public dependency downloads are explicit and
|
||||
@@ -236,6 +327,7 @@ try {
|
||||
// temporary capacity than the smaller offline lifecycle probes.
|
||||
const setup = isolated(['node', '/consumer/node_modules/paperclipai/dist/index.js', 'runtime', 'setup', 'pi'], { download: true, temporarySizeMiB: 2048 }).toString();
|
||||
const receipt = JSON.parse(setup.trim());
|
||||
phase('pi_setup');
|
||||
assert.equal(receipt.status, 'installed_verified');
|
||||
assert.equal(receipt.target, 'linux-x64');
|
||||
assert.equal(readFileSync(join(consumer, 'package-lock.json'), 'utf8'), consumerLock, 'Pi setup must preserve the consumer dependency graph');
|
||||
@@ -243,13 +335,27 @@ try {
|
||||
// Docker defaults tmpfs to noexec. The offline admission probe must execute
|
||||
// its verified private snapshot while keeping lifecycle/download scratch noexec.
|
||||
console.log(isolated(['node', '/packages/pi-public-install-probe.mjs', '/consumer/node_modules/@paperclipai/server'], { temporarySizeMiB: 2048, temporaryExecutable: true }).toString().trim());
|
||||
if (process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR) {
|
||||
phase('pi_probe');
|
||||
// Wall time per phase, so the slow part of this step is visible in the CI log
|
||||
// without re-running it. `*_prefetch_ms` runs from a download's start to its
|
||||
// last output, and `*_wait_ms` is how long the main flow waited for a
|
||||
// prefetch that had not finished yet; zero means the overlap hid it entirely.
|
||||
// `stage_ui_dist_ms`, `stage_sources_ms`, `stage_bundled_ms` and
|
||||
// `stage_pack_ms` split the staging phase, and `stage_bundled_<package>_ms`
|
||||
// is each concurrent install. `stage_standalone_build_ms` appears only when
|
||||
// qualification-package retention made the standalone build necessary.
|
||||
const downloaded = Object.fromEntries(downloads.map(({ name, durationMs }) => [`${name}_prefetch_ms`, durationMs]));
|
||||
console.log(JSON.stringify({ schema: 'paperclip.public-npm-install.timing.v1', ...timing, ...downloaded, ...bundledInstallTimings(installs) }));
|
||||
if (retainQualificationPackages) {
|
||||
const retained = retainRunnerQualificationPackages({ repo, output: process.env.PAPERCLIP_RUNNER_QUALIFICATION_PACKAGES_DIR, sourceRevision, releaseVersion, env,
|
||||
publicArchives: [...needed].map((name, index) => ({ name, file: tarballs[index] })),
|
||||
});
|
||||
console.log(JSON.stringify(retained));
|
||||
}
|
||||
} finally {
|
||||
// An earlier failure must not leave a prefetch running, and scratch is
|
||||
// removed only once both children have stopped writing into it.
|
||||
await stopPrefetches(prefetches);
|
||||
rmSync(root, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user