diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index 21700f2388f..013021e7246 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -85,7 +85,7 @@ "typecheck:browser": "tsc -p tsconfig.browser.json --noEmit", "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", - "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/grok-profile-identity.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", + "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node ../../scripts/ci-profile/node-test.mjs test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs test/acpx-model-selection-package-contract.test.mjs test/acpx-message-boundaries-package-contract.test.mjs test/acpx-rich-extensions-package-contract.test.mjs test/acpx-pi-receipt-package-contract.test.mjs test/acpx-response-delivery-package-contract.test.mjs scripts/candidate-provider-pack.test.mjs scripts/materialize-pi-distribution.test.mjs scripts/cursor-provisioner-layout.test.mjs scripts/release-provider-pack.test.mjs scripts/grok-profile-identity.test.mjs scripts/provider-pack-runtime-tree.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs scripts/installed-package-pack.test.mjs scripts/runner-runtime-dependency-roots.test.mjs scripts/runner-package-executable.test.mjs scripts/stage-runner-binary.test.mjs scripts/build-inputs.test.mjs scripts/provider-pack-executable-shims.test.mjs scripts/controlled-evals/controller.test.mjs scripts/controlled-evals/portable.test.mjs && node --test scripts/build-copilot-distribution.test.mjs scripts/copilot-inner-distribution.test.mjs scripts/copilot-smoke-resources.test.mjs scripts/discover-copilot-acp.test.mjs scripts/materialize-copilot-binary.test.mjs scripts/install-copilot-assets.test.mjs scripts/probe-copilot-acp.test.mjs scripts/qualify-copilot-acp.test.mjs test/acpx-message-boundaries-package-contract.test.mjs", "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", diff --git a/packages/paperclip-runner/scripts/materialize-pi-distribution.mjs b/packages/paperclip-runner/scripts/materialize-pi-distribution.mjs index 72fd11c3bd4..5ccba52a288 100644 --- a/packages/paperclip-runner/scripts/materialize-pi-distribution.mjs +++ b/packages/paperclip-runner/scripts/materialize-pi-distribution.mjs @@ -3,6 +3,7 @@ import { createHash } from "node:crypto"; import { chmod, copyFile, lstat, mkdir, mkdtemp, readFile, readdir, realpath, rename, rm, unlink, writeFile } from "node:fs/promises"; import { stripTypeScriptTypes } from "node:module"; import { dirname, isAbsolute, join, resolve } from "node:path"; +import { setTimeout as sleep } from "node:timers/promises"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { PI_DISTRIBUTION_CLOSURE_SHA256 } from "../src/drivers/acpx/pi-closure-pins.ts"; @@ -132,6 +133,54 @@ export async function resolvePiBundledNpm(nodeRoot) { return entry; } +class RetryablePiNodeDownloadError extends Error {} + +async function readPiNodeArchive(url, fetchImpl, timeoutMs) { + let response; + try { response = await fetchImpl(url, { redirect: "error", signal: AbortSignal.timeout(timeoutMs) }); } + catch (error) { throw error?.name === "TimeoutError" || error instanceof TypeError ? new RetryablePiNodeDownloadError(error.message, { cause: error }) : error; } + if (!response.ok) { + // Release the rejected response's connection before any retry starts. + await response.body?.cancel().catch(() => {}); + const message = `Pinned Pi Node download returned HTTP ${response.status}`; + throw response.status >= 500 || response.status === 429 ? new RetryablePiNodeDownloadError(message) : new Error(message); + } + if (!response.body) throw new Error("Pinned Pi Node download failed"); + const chunks = []; let length = 0; + try { + for await (const chunk of response.body) { + length += chunk.length; if (length > 128 * 1024 * 1024) throw new Error("Pi Node archive exceeds its bound"); + chunks.push(Buffer.from(chunk)); + } + } catch (error) { + // The timeout also bounds the body; a stalled or reset stream is transport loss. + throw error?.name === "TimeoutError" || error instanceof TypeError ? new RetryablePiNodeDownloadError(error.message, { cause: error }) : error; + } + return Buffer.concat(chunks); +} + +/** + * nodejs.org transfers occasionally stall. Retry only transport failures; the + * release hash still gates every byte, and a mismatch or 4xx fails closed at once. + */ +export async function downloadPinnedPiNodeArchive({ url, archiveSha256, fetchImpl = fetch, attempts = 3, timeoutMs = 60_000, retryDelayMs = 5_000, warn = (message) => console.warn(message), checkCancelled = () => {} }) { + for (let attempt = 1; ; attempt++) { + checkCancelled(); + let bytes; + try { bytes = await readPiNodeArchive(url, fetchImpl, timeoutMs); } + catch (error) { + if (!(error instanceof RetryablePiNodeDownloadError)) throw error; + if (attempt >= attempts) throw new Error(`Pinned Pi Node download failed after ${attempt} attempts: ${error.message}`, { cause: error }); + // Keep each transient failure in the build log even when a retry succeeds. + warn(`Pinned Pi Node download attempt ${attempt}/${attempts} failed: ${error.message}; retrying`); + await sleep(retryDelayMs * attempt); + continue; + } + if (hash(bytes) !== archiveSha256) throw new Error("Pi Node archive does not match its release pin"); + return bytes; + } +} + /** Build on the target platform. No lifecycle scripts, model requests, or auth. */ export async function materializePiDistribution({ outputRoot, nodeExecutable, npmExecutable = "npm", inputs, checkCancelled = () => {} }) { // Cancellation is observed between bounded subprocesses. A setup signal must @@ -159,15 +208,7 @@ export async function materializePiDistribution({ outputRoot, nodeExecutable, np if (nodeExecutable) node = await realpath(nodeExecutable); else { const archiveName = `node-v${PI_NODE_VERSION}-${target}.tar.gz`; - const response = await fetch(`https://nodejs.org/dist/v${PI_NODE_VERSION}/${archiveName}`, { redirect: "error", signal: AbortSignal.timeout(60_000) }); - if (!response.ok || !response.body) throw new Error("Pinned Pi Node download failed"); - const chunks = []; let length = 0; - for await (const chunk of response.body) { - length += chunk.length; if (length > 128 * 1024 * 1024) throw new Error("Pi Node archive exceeds its bound"); - chunks.push(Buffer.from(chunk)); - } - const bytes = Buffer.concat(chunks); - if (hash(bytes) !== nodePin.archiveSha256) throw new Error("Pi Node archive does not match its release pin"); + const bytes = await downloadPinnedPiNodeArchive({ url: `https://nodejs.org/dist/v${PI_NODE_VERSION}/${archiveName}`, archiveSha256: nodePin.archiveSha256, checkCancelled }); const archivePath = join(staging, archiveName); await writeFile(archivePath, bytes); const nodeRoot = join(staging, "node-extract"); await mkdir(nodeRoot); await runOwned("tar", ["-xzf", archivePath, "-C", nodeRoot, "--strip-components=1", `node-v${PI_NODE_VERSION}-${target}/bin/node`, `node-v${PI_NODE_VERSION}-${target}/lib/node_modules/npm`], { timeout: 30_000 }); diff --git a/packages/paperclip-runner/scripts/materialize-pi-distribution.test.mjs b/packages/paperclip-runner/scripts/materialize-pi-distribution.test.mjs index 3c65cf25d99..196fe157269 100644 --- a/packages/paperclip-runner/scripts/materialize-pi-distribution.test.mjs +++ b/packages/paperclip-runner/scripts/materialize-pi-distribution.test.mjs @@ -5,7 +5,7 @@ import { mkdtemp, mkdir, readFile, readdir, realpath, rm, symlink, writeFile } f import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; -import { assertPiNodeSystemDependencies, PI_BRACE_SECURITY_PATCH_SHA256, PI_DISTRIBUTION_PINS, materializePiDistribution, resolvePiBundledNpm, piDistributionInstallCommand, verifyLockedPiPackageGraph, writePiDistributionManifest } from "./materialize-pi-distribution.mjs"; +import { assertPiNodeSystemDependencies, downloadPinnedPiNodeArchive, PI_BRACE_SECURITY_PATCH_SHA256, PI_DISTRIBUTION_PINS, materializePiDistribution, resolvePiBundledNpm, piDistributionInstallCommand, verifyLockedPiPackageGraph, writePiDistributionManifest } from "./materialize-pi-distribution.mjs"; import { verifyPiRuntimeManifest } from "../src/drivers/acpx/pi-verified-runtime.ts"; async function fixture(t) { @@ -137,3 +137,95 @@ test("setup selects the exact archive npm and rejects wrong versions or linked e await symlink(join(root, prefix, "package.json"), join(root, prefix, "bin/npm-cli.js")); await assert.rejects(resolvePiBundledNpm(root), /invalid archive entry/); }); + +test("Node archive download retries transport stalls and records each failed attempt", async () => { + const archive = Buffer.from("pinned node archive"); + const archiveSha256 = createHash("sha256").update(archive).digest("hex"); + const stalledBody = new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array([1])); controller.error(new DOMException("The operation was aborted due to timeout", "TimeoutError")); } }); + const replies = [ + () => Promise.reject(new DOMException("The operation was aborted due to timeout", "TimeoutError")), + () => Promise.resolve(new Response(stalledBody)), + () => Promise.resolve(new Response(archive)), + ]; + const signals = []; const warnings = []; + const bytes = await downloadPinnedPiNodeArchive({ + url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256, retryDelayMs: 0, warn: (message) => warnings.push(message), + fetchImpl: (url, init) => { signals.push(init.signal); assert.equal(init.redirect, "error"); return replies.shift()(); }, + }); + assert.deepEqual(bytes, archive); + assert.equal(signals.length, 3); + assert.equal(new Set(signals).size, 3, "each attempt needs a fresh timeout"); + assert.equal(warnings.length, 2); + assert.match(warnings[0], /attempt 1\/3 failed: The operation was aborted due to timeout; retrying/); +}); + +test("Node archive download deadline aborts stalled headers and stalled bodies", async () => { + const archive = Buffer.from("pinned node archive"); + const archiveSha256 = createHash("sha256").update(archive).digest("hex"); + const abortWith = (signal, fail) => signal.addEventListener("abort", () => fail(signal.reason), { once: true }); + const replies = [ + // Headers never arrive; only the supplied deadline can end this attempt. + (signal) => new Promise((_, reject) => abortWith(signal, reject)), + // Headers arrive, then the body stalls after one chunk until the same deadline fires. + (signal) => Promise.resolve(new Response(new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array([1])); abortWith(signal, (reason) => controller.error(reason)); } }))), + () => Promise.resolve(new Response(archive)), + ]; + const warnings = []; + const bytes = await downloadPinnedPiNodeArchive({ + url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256, timeoutMs: 20, retryDelayMs: 0, warn: (message) => warnings.push(message), + fetchImpl: (url, init) => replies.shift()(init.signal), + }); + assert.deepEqual(bytes, archive); + assert.equal(warnings.length, 2); + for (const warning of warnings) assert.match(warning, /The operation was aborted due to timeout; retrying/); +}); + +test("Node archive download releases a rejected response before the next attempt", async () => { + const archive = Buffer.from("pinned node archive"); + const archiveSha256 = createHash("sha256").update(archive).digest("hex"); + const events = []; + const busy = () => new Response(new ReadableStream({ cancel() { events.push("cancel"); } }), { status: 503 }); + const replies = [busy, () => new Response(archive)]; + await downloadPinnedPiNodeArchive({ + url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256, retryDelayMs: 0, warn: () => {}, + fetchImpl: () => { events.push("fetch"); return Promise.resolve(replies.shift()()); }, + }); + assert.deepEqual(events, ["fetch", "cancel", "fetch"]); + events.length = 0; + await assert.rejects(downloadPinnedPiNodeArchive({ + url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256, retryDelayMs: 0, warn: () => assert.fail("must not retry"), + fetchImpl: () => { events.push("fetch"); return Promise.resolve(new Response(new ReadableStream({ cancel() { events.push("cancel"); } }), { status: 404 })); }, + }), /returned HTTP 404/); + assert.deepEqual(events, ["fetch", "cancel"]); +}); + +test("Node archive download names the exhausted request instead of an anonymous timeout", async () => { + let calls = 0; + await assert.rejects(downloadPinnedPiNodeArchive({ + url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256: "0".repeat(64), retryDelayMs: 0, warn: () => {}, + fetchImpl: () => { calls++; return Promise.reject(new TypeError("fetch failed")); }, + }), /^Error: Pinned Pi Node download failed after 3 attempts: fetch failed$/); + assert.equal(calls, 3); + calls = 0; + await assert.rejects(downloadPinnedPiNodeArchive({ + url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256: "0".repeat(64), retryDelayMs: 0, warn: () => {}, + fetchImpl: () => { calls++; return Promise.resolve(new Response("busy", { status: 503 })); }, + }), /failed after 3 attempts: Pinned Pi Node download returned HTTP 503/); + assert.equal(calls, 3); +}); + +test("Node archive download fails closed without retrying pin, bound, client or cancellation failures", async () => { + const options = { url: "https://nodejs.org/dist/fixture.tar.gz", archiveSha256: "0".repeat(64), retryDelayMs: 0, warn: () => assert.fail("must not retry") }; + let calls = 0; + const once = (response) => () => { calls++; return Promise.resolve(response()); }; + await assert.rejects(downloadPinnedPiNodeArchive({ ...options, fetchImpl: once(() => new Response("tampered")) }), /does not match its release pin/); + await assert.rejects(downloadPinnedPiNodeArchive({ ...options, fetchImpl: once(() => new Response("missing", { status: 404 })) }), /returned HTTP 404/); + const oversized = new ReadableStream({ start(controller) { controller.enqueue(new Uint8Array(128 * 1024 * 1024 + 1)); controller.close(); } }); + await assert.rejects(downloadPinnedPiNodeArchive({ ...options, fetchImpl: once(() => new Response(oversized)) }), /exceeds its bound/); + assert.equal(calls, 3); + let checks = 0; + await assert.rejects(downloadPinnedPiNodeArchive({ + ...options, warn: () => {}, checkCancelled: () => { if (++checks > 1) throw new Error("cancelled"); }, + fetchImpl: () => Promise.reject(new DOMException("The operation was aborted due to timeout", "TimeoutError")), + }), /cancelled/); +});