diff --git a/doc/RELEASING.md b/doc/RELEASING.md index 771d3acb5b..dbf40956a5 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -104,7 +104,7 @@ It: - publishes the user-facing `paperclipai` package last, so `paperclipai@canary` does not advance before the full package set exists - verifies that `canary` resolves to the just-published version and that published internal dependencies exist on npm - installs `paperclipai@canary` into a clean temporary prefix as the final npm gate -- starts the exact published canary through a separate fresh npm install for the onboarding smoke; only npm `ETARGET` install failures retry, up to three attempts within two minutes, before onboarding runs once under the existing five-minute startup deadline +- starts the exact published canary through a separate fresh npm install for the onboarding smoke; npm `ETARGET` and verified canary tarball availability failures retry, up to three attempts within two minutes, before onboarding runs once under the existing five-minute startup deadline - fails by default if npm leaves `latest` pointing at a canary; use `--allow-canary-latest` only when that state is intentional - creates a git tag `canary/vYYYY.MDD.P-canary.N` @@ -116,6 +116,23 @@ current patches and manifests while the separate lockfile-refresh PR is still pending, without resolving new dependencies in the smoke job. The smoke still installs and tests the exact published canary version, not a workspace build. +The tarball retry applies only to an unambiguous npm `E404` for a canonical +public-registry tarball of a CI-published Paperclip package at that exact canary +version. The helper queries npm's effective global and applicable scoped registry +in the failed install's prefix, working directory, and environment. A custom or +unknown effective registry fails. Configuration output stays private in memory. +Fresh primary-registry metadata must match the package, version, and tarball URL +and announce a valid SHA512 integrity value. This establishes the announced +artifact identity; it does not download the tarball or independently verify its +contents. The actual npm install still checks tarball integrity. + +Configuration and metadata checks share a five-second deadline, including the +response body, within the same two-minute acquisition budget. Metadata is limited +to 64 KiB. Redirects, authentication errors, missing metadata, conflicting npm +errors, and other artifacts fail. A persistent promised-tarball failure also +fails after three installs. Onboarding errors never restart acquisition or +repeat onboarding. + The lockfile artifact is retained for 14 days. A publisher-job rerun replaces its source-named artifact; a smoke-only rerun uses the existing artifact. A missing artifact fails the job rather than falling back to a different dependency set. diff --git a/scripts/__tests__/canary-startup-fixture.mjs b/scripts/__tests__/canary-startup-fixture.mjs index d75d1fa944..4a51ce621b 100644 --- a/scripts/__tests__/canary-startup-fixture.mjs +++ b/scripts/__tests__/canary-startup-fixture.mjs @@ -10,7 +10,8 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".." // Exercise the real startup helper and child-process boundary without registry // traffic. The npm fixture installs a real executable into its isolated prefix. -export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false }) { +export async function canaryStartup({ mode, onboard = "success", budget = 5000, delay = 5, cancelAtKind, playwright = false, + config = "public", metadata = "valid", version = "2026.1009.0-canary.1" }) { const root = mkdtempSync(path.join(os.tmpdir(), "canary-startup-test-")); const calls = path.join(root, "calls.jsonl"); const fixtureBin = path.join(root, "bin"); @@ -39,15 +40,35 @@ if (process.env.FIXTURE_ONBOARD === "serve") { const fs = require("node:fs"); const path = require("node:path"); const args = process.argv.slice(2); +if (args[0] === "config") { + fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config", pid:process.pid, args}) + "\\n"); + if (process.env.FIXTURE_CONFIG === "hang") { + const { spawn } = require("node:child_process"); + const child = spawn(process.execPath, ["-e", "process.on('SIGTERM',()=>{}); setTimeout(()=>process.exit(0),10000)"], {stdio:"ignore"}); + fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"config-descendant", pid:child.pid}) + "\\n"); + process.on("SIGTERM", () => {}); + setTimeout(() => process.exit(0), 10000); + } else { + if (process.env.FIXTURE_CONFIG === "large") console.log("x".repeat(8192)); + console.log("registry=" + (process.env.FIXTURE_CONFIG === "private" ? "https://fixture-user:fixture-password@registry.example.invalid/" : "https://registry.npmjs.org/") + "\\n@paperclipai:registry=undefined"); + console.error("fixture config diagnostic must stay private"); + } +} else { const entries = fs.existsSync(process.env.FIXTURE_CALLS) ? fs.readFileSync(process.env.FIXTURE_CALLS, "utf8").trim().split("\\n").filter(Boolean).map(JSON.parse) : []; const attempt = entries.filter(e => e.kind === "npm").length + 1; fs.appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"npm", pid:process.pid, args, cache:process.env.npm_config_cache}) + "\\n"); const mode = process.env.FIXTURE_MODE; if (mode === "hang") { setTimeout(() => {}, 10000); } else if (mode === "permanent" || (mode === "recover" && attempt === 1)) { console.error("npm error code ETARGET"); process.exit(1); } +else if (mode === "multibyte-recover" && attempt === 1) { console.error("npm error code ETARGET\\n" + "界".repeat(30 * 1024)); process.exit(1); } else if (mode === "auth") { console.error("npm error code E401"); process.exit(1); } else if (mode === "mixed") { console.error("npm error code ETARGET\\nnpm error code E401"); process.exit(1); } else if (mode === "network") { console.error("npm error code ECONNRESET"); process.exit(1); } +else if (mode === "tarball-permanent" || mode === "tarball-large" || (mode === "tarball-recover" && attempt === 1)) { + const url = "https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz"; + if (mode === "tarball-large") console.error("npm error code E401\\n" + "x".repeat(70 * 1024)); + console.error("npm error code E404\\nnpm error 404 Not Found - GET " + url + " - Not found"); process.exit(1); +} else { const prefix = args[args.indexOf("--prefix") + 1]; const bin = path.join(prefix, "node_modules", ".bin"); @@ -55,12 +76,24 @@ else { fs.writeFileSync(path.join(bin, "paperclipai"), ${JSON.stringify(cli).replaceAll('$','\$')}); fs.chmodSync(path.join(bin, "paperclipai"), 0o755); } +} `); chmodSync(npmFixture, 0o755); const helper = new URL("../../tests/canary-onboarding/start-published-canary.mjs", import.meta.url).href; const driver = path.join(root, "driver.mjs"); writeFileSync(driver, `import { startPublishedCanary } from ${JSON.stringify(helper)}; -try { await startPublishedCanary({version:"2026.1009.0-canary.1", workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); } +import { appendFileSync } from "node:fs"; +globalThis.fetch = async (url, options) => { + appendFileSync(process.env.FIXTURE_CALLS, JSON.stringify({kind:"metadata", url, redirect:options.redirect, credentials:options.credentials, headers:options.headers}) + "\\n"); + const manifest = {name:"@paperclipai/server",version:process.env.PAPERCLIPAI_VERSION,dist:{tarball:"https://registry.npmjs.org/@paperclipai/server/-/server-" + process.env.PAPERCLIPAI_VERSION + ".tgz",integrity:"sha512-" + Buffer.alloc(64).toString("base64")}}; + const body = process.env.FIXTURE_METADATA === "hang" ? new ReadableStream({ + start(controller) { options.signal.addEventListener("abort", () => controller.error(options.signal.reason), {once:true}); }, + }) : JSON.stringify(manifest); + const response = new Response(body, {status:process.env.FIXTURE_METADATA === "missing" ? 404 : 200}); + Object.defineProperty(response,"url",{value:url}); + return response; +}; +try { await startPublishedCanary({version:${JSON.stringify(version)}, workspace:${JSON.stringify(root)}, dataDir:${JSON.stringify(path.join(root,"data"))}, installBudgetMs:${budget}, retryDelayMs:${delay}}); } catch (error) { console.error(error.message); process.exitCode = 1; } `); const playwrightConfig = path.join(root, "playwright.config.mjs"); @@ -80,8 +113,8 @@ export default {...original, testDir:${JSON.stringify(root)}, testMatch:"fixture ? [fileURLToPath(helper), "2026.1009.0-canary.1", root, path.join(root, "data")] : [driver], { cwd: repoRoot, env: { ...process.env, PATH: fixtureBin + path.delimiter + process.env.PATH, - FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port), - PAPERCLIPAI_VERSION: "2026.1009.0-canary.1", PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"), + FIXTURE_MODE: mode, FIXTURE_ONBOARD: onboard, FIXTURE_CALLS: calls, FIXTURE_PORT: String(port), FIXTURE_CONFIG: config, FIXTURE_METADATA: metadata, + PAPERCLIPAI_VERSION: version, PAPERCLIP_CANARY_SMOKE_BASE_URL: `http://127.0.0.1:${port}`, PAPERCLIP_CANARY_SMOKE_SERVER_LOG: path.join(root,"server.log"), npm_config_cache: path.join(root, "fresh-cache"), }, stdio: ["ignore", "pipe", "pipe"] }); let output = ""; diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 9e5364c869..043636edb9 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -5,7 +5,9 @@ import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; +import { createServer } from "node:http"; import { canaryStartup } from "./canary-startup-fixture.mjs"; +import { canRetryPublishedTarball404, TARBALL_METADATA_TIMEOUT_MS } from "../../tests/canary-onboarding/start-published-canary.mjs"; const repoRoot = path.resolve( path.dirname(fileURLToPath(import.meta.url)), @@ -830,3 +832,327 @@ test("nested ETARGET output does not retry a final authentication or network fai assert.equal(result.calls.length, 1); } }); + +const exactCanary = "2026.1009.0-canary.1"; +const canaryTarball = `https://registry.npmjs.org/@paperclipai/server/-/server-${exactCanary}.tgz`; +const tarballFailure = (url = canaryTarball, prefix = "npm error") => `${prefix} code E404\n${prefix} 404 Not Found - GET ${url} - Not found\n${prefix} 404\n${prefix} 404 The requested resource '@paperclipai/server@${url}' could not be found or you do not have permission to access it.\n`; + +function tarballProof({ config = {}, manifest = {}, status = 200, redirected = false, responseUrl, rawBody } = {}) { + const calls = []; + const dist = { tarball: canaryTarball, integrity: `sha512-${Buffer.alloc(64).toString("base64")}`, ...manifest.dist }; + const data = { name: "@paperclipai/server", version: exactCanary, ...manifest, dist }; + return { calls, options: { + signal: new AbortController().signal, prefix: "/fixture/install", env: {}, + runConfig: async (command, args, options) => { + calls.push({ kind: "config", command, args, options }); + return { code: 0, stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=undefined\n", ...config }; + }, + fetchMetadata: async (url, options) => { + calls.push({ kind: "metadata", url, options }); + return { status, redirected, url: responseUrl ?? url, body: new Response(rawBody ?? JSON.stringify(data)).body }; + }, + } }; +} + +test("exact published tarball proof uses quiet effective config and fresh public metadata without credentials", async () => { + for (const prefix of ["npm error", "npm ERR!"]) { + const fixture = tarballProof(); + assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball, prefix), exactCanary, fixture.options), true); + const [config, metadata] = fixture.calls; + assert.deepEqual(config.args, ["config", "get", "registry", "@paperclipai:registry", "--prefix", "/fixture/install"]); + assert.equal(config.options.env, fixture.options.env); + assert.equal(config.options.quiet, true); + assert.equal(config.options.captureStdout, true); + assert.equal(config.options.captureStderr, true); + assert.equal(config.options.outputLimit, 4096); + assert.equal(metadata.url, `https://registry.npmjs.org/%40paperclipai%2Fserver/${exactCanary}`); + assert.equal(metadata.options.signal, config.options.signal); + assert.equal(metadata.options.redirect, "error"); + assert.equal(metadata.options.credentials, "omit"); + assert.deepEqual(metadata.options.headers, { accept: "application/json", "cache-control": "no-cache" }); + } +}); + +test("mixed npm errors and noncanonical or unreleased tarballs never read config or metadata", async () => { + const errors = [ + "npm error code ETARGET\n", "npm error code E401\n", "npm error code ECONNRESET\n", + `${tarballFailure()}npm error code E401\n`, `${tarballFailure()}npm error code 1\n`, + `npm error code ENEEDAUTH\n${tarballFailure()}`, `${tarballFailure()}${tarballFailure()}`, + `${tarballFailure()}npm error 404 Not Found - GET https://registry.example.invalid/a.tgz - Not found\n`, + `${tarballFailure()}npm error 404 The requested resource 'other@${canaryTarball}' could not be found\n`, + tarballFailure().replace(" - GET ", " - PUT "), + ...[ + canaryTarball + "?token=fixture", canaryTarball + "#fragment", canaryTarball.replace("npmjs.org/", "npmjs.org:443/"), + canaryTarball.replace("npmjs.org/", "npmjs.org.example.invalid/"), canaryTarball.replace("npmjs.org/", "fixture@npmjs.org/"), + canaryTarball.replace("https:", "http:"), canaryTarball.replace("@paperclipai/server", "%40paperclipai%2Fserver"), + canaryTarball.replace("@paperclipai/server", "@paperclipai/../server"), canaryTarball.replace(exactCanary, "2026.1009.0-canary.2"), + "https://registry.npmjs.org/@paperclipai/not-released/-/not-released-2026.1009.0-canary.1.tgz", + "https://registry.npmjs.org/@paperclipai/plugin-createos/-/plugin-createos-2026.1009.0-canary.1.tgz", + "https://registry.npmjs.org/%40paperclipai%2Fserver/2026.1009.0-canary.1", + ].map((url) => tarballFailure(url)), + ]; + for (const error of errors) { + const fixture = tarballProof(); + assert.equal(await canRetryPublishedTarball404(error, exactCanary, fixture.options), false, error); + assert.deepEqual(fixture.calls, [], error); + } + for (const version of ["2026.1009.0", "2026.1009.0-nightly.1", "2026.1009.0-beta.1", "2026.1009.0-canary.1+build"]) { + const fixture = tarballProof(); + assert.equal(await canRetryPublishedTarball404(tarballFailure(canaryTarball.replace(exactCanary, version)), version, fixture.options), false); + assert.deepEqual(fixture.calls, []); + } +}); + +test("effective registry authority respects scope precedence and rejects unknown or private config", async () => { + const configs = [ + [{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true], + [{ stdout: "registry=undefined\n@paperclipai:registry=https://registry.npmjs.org/\n" }, true], + [{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=https://registry.example.invalid/\n" }, false], + [{ stdout: "registry=https://registry.example.invalid/\n@paperclipai:registry=undefined\n" }, false], + [{ stdout: "registry=undefined\n@paperclipai:registry=undefined\n" }, false], + [{ stdout: "registry=https://registry.npmjs.org/\n@paperclipai:registry=\n" }, false], + [{ stdout: "registry=https://fixture:secret@registry.npmjs.org/\n@paperclipai:registry=undefined\n" }, false], + [{ stdout: "unexpected output\n" }, false], [{ code: 1 }, false], [{ signal: "SIGTERM" }, false], [{ outputTruncated: true }, false], + ]; + for (const [config, expected] of configs) { + const fixture = tarballProof({ config }); + assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), expected); + assert.equal(fixture.calls.filter((call) => call.kind === "metadata").length, expected ? 1 : 0); + } +}); + +test("metadata must identify the exact published artifact with valid SHA512 syntax, not merely exist", async () => { + const fixtures = [ + { status: 404 }, { status: 401 }, { status: 403 }, { status: 302 }, { redirected: true }, + { responseUrl: "https://registry.example.invalid/metadata" }, { rawBody: "not json" }, + { manifest: { name: "@paperclipai/shared" } }, { manifest: { version: "2026.1009.0-canary.2" } }, + { manifest: { dist: { tarball: canaryTarball + "?query=1" } } }, + ...[undefined, "", "sha512-A===", `sha256-${Buffer.alloc(32).toString("base64")}`, `sha512-${Buffer.alloc(64).toString("base64")} extra`] + .map((integrity) => ({ manifest: { dist: { integrity } } })), + { rawBody: JSON.stringify({ padding: "x".repeat(65 * 1024) }) }, + ]; + for (const input of fixtures) { + const fixture = tarballProof(input); + assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false, JSON.stringify(input).slice(0, 200)); + } +}); + +test("rejected and oversized native response bodies cancel without waiting for slow or rejected cleanup", async () => { + for (const [status, pending] of [[200, true], [200, false], [404, true], [404, false]]) { + let cancelled = false; + const fixture = tarballProof(); + fixture.options.fetchMetadata = async (url) => ({ status, redirected: false, url, body: new Response(new ReadableStream({ + start(controller) { controller.enqueue(new Uint8Array(65 * 1024)); }, + cancel() { cancelled = true; return pending ? new Promise(() => {}) : Promise.reject(new Error("fixture rejected cancellation")); }, + })).body }); + const started = Date.now(); + assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, fixture.options), false); + assert.equal(cancelled, true); + assert.ok(Date.now() - started < 1000, "cancellation acknowledgement must not extend a rejection"); + } +}); + +test("aborting a stalled metadata read cancels its reader without waiting for acknowledgement", { timeout: 2000 }, async () => { + let cancelled = false; + const fixture = tarballProof(); + fixture.options.fetchMetadata = async (url) => ({ status: 200, redirected: false, url, + body: new Response(new ReadableStream({ + cancel() { cancelled = true; return new Promise(() => {}); }, + })).body }); + const budget = new AbortController(); + const timer = setTimeout(() => budget.abort(new Error("fixture body budget expired")), 100); + const started = Date.now(); + try { + await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, { + ...fixture.options, signal: budget.signal, + }), /fixture body budget expired/); + assert.equal(cancelled, true); + assert.ok(Date.now() - started < 1000); + } finally { clearTimeout(timer); } +}); + +test("metadata headers cannot outlive the budget and late response bodies are cancelled", { timeout: 2000 }, async () => { + let headersArrive; + let cancelled = false; + const fixture = tarballProof(); + fixture.options.fetchMetadata = (url) => new Promise((resolve) => { + headersArrive = () => resolve({ status: 200, redirected: false, url, + body: new Response(new ReadableStream({ cancel() { cancelled = true; } })).body }); + }); + const budget = new AbortController(); + const timer = setTimeout(() => budget.abort(new Error("fixture headers budget expired")), 100); + const started = Date.now(); + try { + await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, { + ...fixture.options, signal: budget.signal, + }), /fixture headers budget expired/); + assert.ok(Date.now() - started < 1000); + headersArrive(); + await new Promise((resolve) => setImmediate(resolve)); + assert.equal(cancelled, true); + } finally { clearTimeout(timer); } +}); + +test("real npm effective registry reads isolated prefix, user, and environment configuration", async () => { + const root = mkdtempSync(path.join(os.tmpdir(), "canary-config-proof-")); + mkdirSync(path.join(root, "prefix")); + writeFileSync(path.join(root, "global.npmrc"), ""); + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.toLowerCase().startsWith("npm_config_"))); + env.npm_config_userconfig = path.join(root, "user.npmrc"); + env.npm_config_globalconfig = path.join(root, "global.npmrc"); + try { + for (const [user, project, override, expected] of [ + ["registry=https://registry.npmjs.org/\n", "", undefined, true], + ["registry=https://registry.example.invalid/\n", "@paperclipai:registry=https://registry.npmjs.org/\n", undefined, true], + ["registry=https://registry.npmjs.org/\n", "@paperclipai:registry=https://registry.example.invalid/\n", undefined, false], + ["registry=https://registry.npmjs.org/\n", "", "https://registry.example.invalid/", false], + ]) { + writeFileSync(path.join(root, "user.npmrc"), user); + writeFileSync(path.join(root, "prefix", ".npmrc"), project); + const fixture = tarballProof(); + delete fixture.options.runConfig; + assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, { + ...fixture.options, prefix: path.join(root, "prefix"), env: { ...env, ...(override ? { npm_config_registry: override } : {}) }, + }), expected); + assert.equal(fixture.calls.length, expected ? 1 : 0); + } + } finally { rmSync(root, { recursive: true, force: true }); } +}); + +async function stalledMetadata(run) { + let closed = false; + let closing = false; + let observeClose; + const connectionClosed = new Promise((resolve) => { observeClose = resolve; }); + const server = createServer((_request, response) => { + response.on("close", () => { closed = true; observeClose(); }); + response.writeHead(200, { "content-type": "application/json" }); + response.write('{"name":'); + }); + // An HTTP client can replace an aborted pooled connection while shutdown + // starts. Own late connections too, so server.close cannot wait on them. + server.on("connection", (socket) => { if (closing) socket.destroy(); }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const fixture = tarballProof(); + fixture.options.fetchMetadata = async (url, options) => { + const response = await fetch(`http://127.0.0.1:${server.address().port}/metadata`, options); + Object.defineProperty(response, "url", { value: url }); + return response; + }; + let closeDeadline; + try { + await run(fixture.options); + await Promise.race([connectionClosed, new Promise((_, reject) => { + closeDeadline = setTimeout(() => reject(new Error("metadata connection outlived cancellation")), 1000); + })]); + assert.equal(closed, true, "metadata connection must close before fixture-forced cleanup"); + } + finally { + clearTimeout(closeDeadline); + closing = true; + await new Promise((resolve) => { + server.close(resolve); + server.closeAllConnections(); + }); + } +} + +test("shared acquisition cancellation aborts the metadata body before the lookup deadline", async () => { + await stalledMetadata(async (options) => { + const budget = new AbortController(); + const timer = setTimeout(() => budget.abort(new Error("fixture shared budget expired")), 250); + try { + await assert.rejects(canRetryPublishedTarball404(tarballFailure(), exactCanary, { ...options, signal: budget.signal }), /fixture shared budget expired/); + } finally { clearTimeout(timer); } + }); +}); + +test("the short config and metadata deadline also covers a stalled real HTTP body", { timeout: 10_000 }, async () => { + assert.equal(TARBALL_METADATA_TIMEOUT_MS, 5000); + const started = Date.now(); + await stalledMetadata(async (options) => { + assert.equal(await canRetryPublishedTarball404(tarballFailure(), exactCanary, options), false); + }); + assert.ok(Date.now() - started < 7000, "metadata must not consume the 120s acquisition budget"); +}); + +test("a verified canary tarball404 retries acquisition and runs onboarding once", async () => { + const result = await canaryStartup({ mode: "tarball-recover" }); + assert.equal(result.code, 0, result.output); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2); + assert.equal(result.calls.filter((call) => call.kind === "config").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1); + assert.doesNotMatch(result.output, /fixture config diagnostic/); +}); + +test("persistent promised tarball404s stop at three installs without a final lookup or onboarding", async () => { + const result = await canaryStartup({ mode: "tarball-permanent" }); + assert.equal(result.code, 1); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 3); + assert.equal(result.calls.filter((call) => call.kind === "config").length, 2); + assert.equal(result.calls.filter((call) => call.kind === "metadata").length, 2); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0); + assert.match(result.output, /installation failed.*onboarding was not started/); +}); + +test("private or oversized config and unpublished metadata404 fail without retry or leaking config", async () => { + for (const input of [{ config: "private" }, { config: "large" }, { metadata: "missing" }]) { + const result = await canaryStartup({ mode: "tarball-permanent", ...input }); + assert.equal(result.code, 1); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0); + assert.doesNotMatch(result.output, /fixture-user|fixture-password|fixture config diagnostic/); + } +}); + +test("truncated npm output cannot hide an earlier auth failure and qualify a tarball retry", async () => { + const result = await canaryStartup({ mode: "tarball-large" }); + assert.equal(result.code, 1); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "config").length, 0); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0); +}); + +test("tarball recovery does not replay onboarding failures or change ETARGET for other exact versions", async () => { + const failed = await canaryStartup({ mode: "tarball-recover", onboard: "fail" }); + assert.equal(failed.code, 1); + assert.equal(failed.calls.filter((call) => call.kind === "npm").length, 2); + assert.equal(failed.calls.filter((call) => call.kind === "onboard").length, 1); + assert.match(failed.output, /onboarding failed \(exit 17\)/); + const other = await canaryStartup({ mode: "recover", version: "2026.1009.0-nightly.1" }); + assert.equal(other.code, 0, other.output); + assert.equal(other.calls.filter((call) => call.kind === "npm").length, 2); + assert.equal(other.calls.filter((call) => call.kind === "onboard").length, 1); + assert.equal(other.calls.filter((call) => call.kind === "config").length, 0); +}); + +test("existing ETARGET character-tail behavior survives long multibyte npm diagnostics", async () => { + const result = await canaryStartup({ mode: "multibyte-recover" }); + assert.equal(result.code, 0, result.output); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 2); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "config").length, 0); +}); + +test("metadata lookup and backoff both consume the unchanged shared acquisition budget", async () => { + for (const input of [{ metadata: "hang" }, { delay: 5000 }]) { + const result = await canaryStartup({ mode: "tarball-permanent", budget: 1000, ...input }); + assert.equal(result.code, 1); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0); + assert.match(result.output, /exceeded its startup budget/); + assert.ok(result.elapsed < 4000, `shared-budget cancellation took ${result.elapsed}ms`); + } +}); + +test("cancellation reaps a config process and its descendant that ignore SIGTERM", async () => { + const result = await canaryStartup({ mode: "tarball-permanent", config: "hang", cancelAtKind: "config-descendant" }); + assert.equal(result.code, 1); + assert.equal(result.calls.filter((call) => call.kind === "npm").length, 1); + assert.equal(result.calls.filter((call) => call.kind === "onboard").length, 0); + for (const child of result.calls.filter((call) => ["config", "config-descendant"].includes(call.kind))) { + assert.throws(() => process.kill(child.pid, 0), { code: "ESRCH" }); + } + assert.match(result.output, /stopped by SIGTERM/); +}); diff --git a/tests/canary-onboarding/start-published-canary.mjs b/tests/canary-onboarding/start-published-canary.mjs index d52d0f87da..f294d8a9f1 100644 --- a/tests/canary-onboarding/start-published-canary.mjs +++ b/tests/canary-onboarding/start-published-canary.mjs @@ -2,21 +2,148 @@ import { spawn } from "node:child_process"; import { mkdtemp, rm } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { loadReleaseManifest } from "../../scripts/release-package-map.mjs"; // Leave three minutes for onboarding inside Playwright's existing five-minute // startup deadline. This budget includes every install attempt and backoff. export const INSTALL_BUDGET_MS = 120_000; const MAX_INSTALL_ATTEMPTS = 3; const STDERR_TAIL_CHARS = 64 * 1024; +export const TARBALL_METADATA_TIMEOUT_MS = 5_000; +const METADATA_LIMIT_BYTES = 64 * 1024; +const PRIMARY_REGISTRY = "https://registry.npmjs.org/"; +const RELEASE_PACKAGE_NAMES = new Set(loadReleaseManifest() + .filter((entry) => entry.publishFromCi === true).map((entry) => entry.name)); -function run(command, args, { env, signal, captureStderr = false }) { +function withAbort(pending, signal) { + return new Promise((resolve, reject) => { + const abort = () => { + signal.removeEventListener("abort", abort); + reject(signal.reason); + }; + const finish = (settle) => (value) => { + signal.removeEventListener("abort", abort); + settle(value); + }; + // Keep handlers on the original operation even after the bounded wait + // ends. A transport need not settle immediately when its signal aborts. + Promise.resolve(pending).then(finish(resolve), finish(reject)); + signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }); +} + +export async function canRetryPublishedTarball404(stderr, version, { + signal, prefix, env = process.env, fetchMetadata = globalThis.fetch, runConfig = run, +}) { + // A package metadata 404, a third-party artifact, or another release must not + // be mistaken for availability of this canary's announced public tarball. + if (!/^\d+\.\d+\.\d+-canary\.\d+$/.test(version)) return false; + const codes = [...stderr.matchAll(/^npm (?:error|ERR!) code ([A-Z0-9_]+)\r?$/gm)]; + const final = codes.at(-1); + if (codes.length !== 1 || final?.[1] !== "E404") return false; + const block = stderr.slice(final.index); + const requests = [...block.matchAll(/^npm (?:error|ERR!) 404 Not Found - GET (\S+) - Not found\r?$/gm)]; + const httpLines = stderr.match(/^npm (?:error|ERR!).* - (?:GET|HEAD|POST|PUT|DELETE|PATCH) /gm) ?? []; + if (requests.length !== 1 || httpLines.length !== 1) return false; + const tarball = requests[0][1]; + const packageName = [...RELEASE_PACKAGE_NAMES].find((name) => { + const basename = name.split("/").at(-1); + return tarball === `${PRIMARY_REGISTRY}${name}/-/${basename}-${version}.tgz`; + }); + if (!packageName) return false; + const resources = [...block.matchAll(/^npm (?:error|ERR!) 404\s+(?:The requested resource )?'([^']+)'(?: could not be found| is not in this registry)/gm)]; + if (resources.some((resource) => resource[1] !== `${packageName}@${tarball}`)) return false; + signal.throwIfAborted(); + const timeout = new AbortController(); + const timer = setTimeout(() => timeout.abort(new Error("Canary tarball metadata lookup timed out")), TARBALL_METADATA_TIMEOUT_MS); + const lookupSignal = AbortSignal.any([signal, timeout.signal]); + const metadataUrl = `${PRIMARY_REGISTRY}${encodeURIComponent(packageName)}/${version}`; + let response; + try { + // Query only registry values in the failed install's effective prefix, + // inherited cwd and environment. Keep even warnings out of the logs: + // configuration values can contain credentials. Config and HTTP share + // this short deadline, inside the unchanged installation budget. + const config = await runConfig(process.platform === "win32" ? "npm.cmd" : "npm", [ + "config", "get", "registry", "@paperclipai:registry", "--prefix", prefix, + ], { env, signal: lookupSignal, captureStdout: true, captureStderr: true, quiet: true, outputLimit: 4096 }); + if (config.code !== 0 || config.signal || config.outputTruncated) return false; + const lines = config.stdout.trim().split(/\r?\n/); + if (lines.length !== 2 || !lines[0].startsWith("registry=") || !lines[1].startsWith("@paperclipai:registry=")) return false; + const globalRegistry = lines[0].slice("registry=".length); + const scoped = lines[1].slice("@paperclipai:registry=".length); + const effectiveRegistry = packageName.startsWith("@") && scoped !== "undefined" ? scoped : globalRegistry; + if (![PRIMARY_REGISTRY, PRIMARY_REGISTRY.slice(0, -1)].includes(effectiveRegistry)) return false; + response = await withAbort(Promise.resolve(fetchMetadata(metadataUrl, { + signal: lookupSignal, redirect: "error", credentials: "omit", + headers: { accept: "application/json", "cache-control": "no-cache" }, + })).then((arrived) => { + if (lookupSignal.aborted) void arrived.body?.cancel().catch(() => {}); + return arrived; + }), lookupSignal); + if (response.status !== 200 || response.redirected || response.url !== metadataUrl || !response.body) return false; + const reader = response.body.getReader(); + const cancelReader = () => { void reader.cancel().catch(() => {}); }; + lookupSignal.addEventListener("abort", cancelReader, { once: true }); + const chunks = []; + let bytes = 0; + try { + while (true) { + lookupSignal.throwIfAborted(); + const { done, value } = await withAbort(reader.read(), lookupSignal); + if (done) break; + bytes += value.byteLength; + if (bytes > METADATA_LIMIT_BYTES) return false; + chunks.push(value); + } + } finally { + // Initiate cleanup without extending either deadline for a transport's + // pending/rejected cancellation acknowledgement. + lookupSignal.removeEventListener("abort", cancelReader); + cancelReader(); + reader.releaseLock(); + } + lookupSignal.throwIfAborted(); + const manifest = JSON.parse(Buffer.concat(chunks).toString("utf8")); + const integrity = manifest.dist?.integrity; + // This validates the integrity announced by primary metadata. No tarball + // is downloaded here, and this is not an independent content digest proof. + return manifest.name === packageName && manifest.version === version + && manifest.dist?.tarball === tarball && typeof integrity === "string" + && /^sha512-[A-Za-z0-9+/]{86}==$/.test(integrity) + && Buffer.from(integrity.slice(7), "base64").toString("base64") === integrity.slice(7); + } catch { + signal.throwIfAborted(); + return false; + } finally { + void response?.body?.cancel().catch(() => {}); + clearTimeout(timer); + } +} + +function run(command, args, { env, signal, captureStderr = false, captureStdout = false, quiet = false, outputLimit = STDERR_TAIL_CHARS }) { return new Promise((resolve, reject) => { signal.throwIfAborted(); const grouped = process.platform !== "win32"; const child = spawn(command, args, { - env, detached: grouped, stdio: ["ignore", "inherit", captureStderr ? "pipe" : "inherit"], + env, detached: grouped, stdio: ["ignore", captureStdout ? "pipe" : "inherit", captureStderr ? "pipe" : "inherit"], }); - let stderr = ""; + let stderr = quiet ? Buffer.alloc(0) : ""; + let stdout = quiet ? Buffer.alloc(0) : ""; + let outputTruncated = false; + const retain = (previous, chunk) => { + if (quiet) { + outputTruncated ||= previous.length + chunk.byteLength > outputLimit; + return Buffer.concat([previous, chunk]).subarray(-outputLimit); + } + // Preserve the existing install stderr character tail and ETARGET + // behavior, including multibyte diagnostics. Only private config capture + // uses a byte bound; any truncated E404 evidence is rejected. + const next = previous + chunk.toString(); + outputTruncated ||= next.length > outputLimit; + return next.slice(-outputLimit); + }; let stopping; const kill = (kind) => { try { @@ -34,15 +161,20 @@ function run(command, args, { env, signal, captureStderr = false }) { }; signal.addEventListener("abort", stop, { once: true }); child.stderr?.on("data", (chunk) => { - process.stderr.write(chunk); - stderr = (stderr + chunk.toString()).slice(-STDERR_TAIL_CHARS); + if (!quiet) process.stderr.write(chunk); + stderr = retain(stderr, chunk); + }); + child.stdout?.on("data", (chunk) => { + if (!quiet) process.stdout.write(chunk); + stdout = retain(stdout, chunk); }); child.on("error", reject); child.on("close", async (code, exitSignal) => { signal.removeEventListener("abort", stop); await stopping; if (signal.aborted) reject(signal.reason); - else resolve({ code, signal: exitSignal, stderr }); + else resolve({ code, signal: exitSignal, + stderr: quiet ? stderr.toString("utf8") : stderr, stdout: quiet ? stdout.toString("utf8") : stdout, outputTruncated }); }); }); } @@ -85,7 +217,10 @@ export async function startPublishedCanary({ if (result.code === 0) break; const codes = [...result.stderr.matchAll(/^npm (?:error|ERR!) code ([A-Z0-9_]+)\r?$/gm)]; const missingVersion = codes.at(-1)?.[1] === "ETARGET"; - if (!missingVersion || result.signal || attempt === MAX_INSTALL_ATTEMPTS) { + const retryable = !result.signal && attempt < MAX_INSTALL_ATTEMPTS + && (missingVersion || (!result.outputTruncated + && await canRetryPublishedTarball404(result.stderr, version, { signal: installSignal, prefix, env }))); + if (!retryable) { throw new Error(`Canary npm installation failed (exit ${result.code ?? result.signal}); onboarding was not started`); } await rm(prefix, { recursive: true, force: true });