diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 00000000000..7b6ea7115cf --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,16 @@ +# Cargo defaults abort a crate download when the transfer rate stays +# below 10 bytes per second for 30 seconds, then retry 3 times. On a +# shared CI runner a brief network stall can cross that 30-second window +# on every attempt, so the whole download fails even though the network +# recovers soon after. A wider timeout and one more retry give a stall +# more time to pass before cargo gives up. +# +# Worst case for one dead crate: 6 attempts (1 try plus 5 retries) times +# a 60-second timeout equals 360 seconds. The job that runs this build +# has a 20-minute (1200-second) timeout, so this worst case uses 30% of +# the budget and leaves 840 seconds for every other step. +[net] +retry = 5 + +[http] +timeout = 60 diff --git a/package.json b/package.json index 0a7fb82bab4..22af6b6d780 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs", + "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/check-cargo-network-config.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/scripts/cargo-network-stall-harness/.gitignore b/scripts/cargo-network-stall-harness/.gitignore new file mode 100644 index 00000000000..00930dff382 --- /dev/null +++ b/scripts/cargo-network-stall-harness/.gitignore @@ -0,0 +1 @@ +.scratch/ diff --git a/scripts/cargo-network-stall-harness/run-differential.sh b/scripts/cargo-network-stall-harness/run-differential.sh new file mode 100755 index 00000000000..f1f82d5e890 --- /dev/null +++ b/scripts/cargo-network-stall-harness/run-differential.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# Reproduce the CI network stall on purpose, then prove the new cargo +# config survives it. +# +# The script builds one synthetic crate and serves it from a local HTTP +# server. The server holds the download body empty for a fixed stall +# period, then sends it in full. Two arms fetch that crate: +# +# Arm A (the control) forces the old cargo defaults with environment +# variables. The fetch must fail with the exact error text seen in CI. +# +# Arm B uses the repository's own .cargo/config.toml, found by cargo's +# normal upward directory search from the repository root. The same +# stall must not fail the fetch. +# +# Run this script from anywhere; it resolves paths from its own location. +set -euo pipefail + +HARNESS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$HARNESS_DIR/../.." && pwd)" +STALL_SECONDS="${STALL_SECONDS:-40}" +CRATE_NAME="stallharnesscrate" +CRATE_VERSION="0.1.0" +SCRATCH="$HARNESS_DIR/.scratch" + +export PATH="$HOME/.cargo/bin:$PATH" + +if ! command -v cargo >/dev/null 2>&1; then + echo "cargo is not on PATH; cannot run the stall differential" >&2 + exit 1 +fi + +rm -rf "$SCRATCH" +mkdir -p "$SCRATCH/crate-src/${CRATE_NAME}-${CRATE_VERSION}/src" + +cat > "$SCRATCH/crate-src/${CRATE_NAME}-${CRATE_VERSION}/Cargo.toml" < "$SCRATCH/crate-src/${CRATE_NAME}-${CRATE_VERSION}/src/lib.rs" + +TARBALL="$SCRATCH/${CRATE_NAME}-${CRATE_VERSION}.crate" +tar --sort=name --owner=0 --group=0 --numeric-owner \ + -czf "$TARBALL" -C "$SCRATCH/crate-src" "${CRATE_NAME}-${CRATE_VERSION}" +CKSUM="$(sha256sum "$TARBALL" | awk '{print $1}')" + +mkdir -p "$SCRATCH/consumer/src" +cat > "$SCRATCH/consumer/Cargo.toml" < "$SCRATCH/consumer/src/main.rs" + +node "$HARNESS_DIR/stall-server.mjs" \ + --port 0 \ + --crate-name "$CRATE_NAME" \ + --crate-version "$CRATE_VERSION" \ + --cksum "$CKSUM" \ + --stall-seconds "$STALL_SECONDS" \ + --tarball "$TARBALL" \ + > "$SCRATCH/server.log" 2>&1 & +SERVER_PID=$! +trap 'kill "$SERVER_PID" 2>/dev/null || true' EXIT + +for _ in $(seq 1 50); do + if grep -q "^LISTENING " "$SCRATCH/server.log" 2>/dev/null; then + break + fi + sleep 0.1 +done +PORT="$(grep "^LISTENING " "$SCRATCH/server.log" | awk '{print $2}')" +if [ -z "$PORT" ]; then + echo "stall server did not report a port" >&2 + cat "$SCRATCH/server.log" >&2 + exit 1 +fi +echo "stall server listening on 127.0.0.1:$PORT (stall period ${STALL_SECONDS}s)" + +export CARGO_REGISTRIES_STALL_HARNESS_INDEX="sparse+http://127.0.0.1:${PORT}/index/" + +run_arm() { + local label="$1" + shift + rm -rf "$SCRATCH/cargo-home" "$SCRATCH/consumer/Cargo.lock" + mkdir -p "$SCRATCH/cargo-home" + echo "--- Arm $label ---" + ( + cd "$REPO_ROOT" + CARGO_HOME="$SCRATCH/cargo-home" "$@" cargo fetch --manifest-path "$SCRATCH/consumer/Cargo.toml" + ) +} + +set +e +ARM_A_OUTPUT="$(run_arm A env CARGO_NET_RETRY=3 CARGO_HTTP_TIMEOUT=30 2>&1)" +ARM_A_STATUS=$? +set -e +echo "$ARM_A_OUTPUT" +echo "Arm A exit status: $ARM_A_STATUS" + +set +e +ARM_B_OUTPUT="$(run_arm B env 2>&1)" +ARM_B_STATUS=$? +set -e +echo "$ARM_B_OUTPUT" +echo "Arm B exit status: $ARM_B_STATUS" + +echo "--- verdict ---" +FAIL=0 + +if [ "$ARM_A_STATUS" -eq 0 ]; then + echo "FAIL: Arm A (old defaults) was expected to fail but the fetch succeeded" + FAIL=1 +elif ! grep -q "transfer too slow: failed to transfer more than 10 bytes in 30s" <<<"$ARM_A_OUTPUT"; then + echo "FAIL: Arm A did not report the expected stall error text" + FAIL=1 +else + echo "PASS: Arm A failed with the expected stall error text" +fi + +if [ "$ARM_B_STATUS" -ne 0 ]; then + echo "FAIL: Arm B (new config) was expected to succeed but the fetch failed" + FAIL=1 +else + echo "PASS: Arm B succeeded through the same stall" +fi + +exit $FAIL diff --git a/scripts/cargo-network-stall-harness/stall-server.mjs b/scripts/cargo-network-stall-harness/stall-server.mjs new file mode 100644 index 00000000000..b42b674802f --- /dev/null +++ b/scripts/cargo-network-stall-harness/stall-server.mjs @@ -0,0 +1,97 @@ +#!/usr/bin/env node +// A local stand-in for a cargo sparse registry. It serves one synthetic +// crate and holds the download response body empty for a fixed period +// before it sends the full body. This copies a real network stall: the +// transfer rate stays at 0 bytes per second until the stall period ends. +import http from "node:http"; +import { readFileSync } from "node:fs"; + +function parseArgs(argv) { + const args = {}; + for (let i = 0; i < argv.length; i += 2) { + const key = argv[i].replace(/^--/, ""); + args[key] = argv[i + 1]; + } + return args; +} + +const args = parseArgs(process.argv.slice(2)); +const requestedPort = Number(args.port ?? 0); +const crateName = args["crate-name"]; +const crateVersion = args["crate-version"]; +const cksum = args.cksum; +const stallSeconds = Number(args["stall-seconds"]); +const tarball = readFileSync(args.tarball); + +if (!crateName || !crateVersion || !cksum || !Number.isFinite(stallSeconds) || !args.tarball) { + console.error( + "usage: stall-server.mjs --port --crate-name --crate-version --cksum --stall-seconds --tarball ", + ); + process.exit(1); +} + +// Cargo's sparse registry protocol picks the index path from the crate +// name length. See the cargo book, "Index Format", for this rule. +function indexPathSegment(name) { + if (name.length <= 2) return `${name.length}/${name}`; + if (name.length === 3) return `3/${name[0]}/${name}`; + return `${name.slice(0, 2)}/${name.slice(2, 4)}/${name}`; +} + +const indexSegment = indexPathSegment(crateName); + +const server = http.createServer((req, res) => { + const url = new URL(req.url, "http://127.0.0.1"); + const port = server.address().port; + + if (url.pathname === "/index/config.json") { + const body = JSON.stringify({ + dl: `http://127.0.0.1:${port}/dl`, + api: `http://127.0.0.1:${port}`, + }); + res.writeHead(200, { + "content-type": "application/json", + "content-length": Buffer.byteLength(body), + }); + res.end(body); + return; + } + + if (url.pathname === `/index/${indexSegment}`) { + const body = `${JSON.stringify({ + name: crateName, + vers: crateVersion, + deps: [], + cksum, + features: {}, + yanked: false, + })}\n`; + res.writeHead(200, { + "content-type": "text/plain", + "content-length": Buffer.byteLength(body), + }); + res.end(body); + return; + } + + if (url.pathname === `/dl/${crateName}/${crateVersion}/download`) { + res.writeHead(200, { + "content-type": "application/octet-stream", + "content-length": tarball.length, + }); + res.flushHeaders(); + // Hold the body empty for the stall period, then send it all at once. + // This puts the transfer rate at 0 bytes/sec until the stall ends. + setTimeout(() => { + res.end(tarball); + }, stallSeconds * 1000); + return; + } + + res.writeHead(404); + res.end(); +}); + +server.listen(requestedPort, "127.0.0.1", () => { + console.log(`LISTENING ${server.address().port}`); +}); diff --git a/scripts/check-cargo-network-config.test.mjs b/scripts/check-cargo-network-config.test.mjs new file mode 100644 index 00000000000..5e8ffdb587d --- /dev/null +++ b/scripts/check-cargo-network-config.test.mjs @@ -0,0 +1,48 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +const scriptsDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.join(scriptsDir, ".."); +const configPath = path.join(repoRoot, ".cargo", "config.toml"); + +// A cargo build with the default network settings can fail on a short +// stall: cargo aborts a download after 30 seconds below 10 bytes per +// second, then retries only 3 times. These minimums keep the repository +// config wide enough to survive that kind of stall. +const MIN_NET_RETRY = 5; +const MIN_HTTP_TIMEOUT = 60; + +function readIntUnderSection(text, sectionName, key) { + const sectionPattern = new RegExp(`\\[${sectionName}\\]([\\s\\S]*?)(?:\\n\\[|$)`); + const sectionMatch = text.match(sectionPattern); + assert.ok(sectionMatch, `expected a [${sectionName}] section in ${configPath}`); + const keyPattern = new RegExp(`^\\s*${key}\\s*=\\s*(\\d+)\\s*$`, "m"); + const keyMatch = sectionMatch[1].match(keyPattern); + assert.ok(keyMatch, `expected ${sectionName}.${key} in ${configPath}`); + return Number(keyMatch[1]); +} + +test("the repository cargo config file exists at the repository root", () => { + assert.doesNotThrow(() => readFileSync(configPath, "utf8")); +}); + +test("the repository cargo config raises the network retry count", () => { + const text = readFileSync(configPath, "utf8"); + const retry = readIntUnderSection(text, "net", "retry"); + assert.ok( + retry >= MIN_NET_RETRY, + `net.retry is ${retry}; it must stay at ${MIN_NET_RETRY} or higher so a transient network stall does not fail the build`, + ); +}); + +test("the repository cargo config raises the http transfer timeout", () => { + const text = readFileSync(configPath, "utf8"); + const timeout = readIntUnderSection(text, "http", "timeout"); + assert.ok( + timeout >= MIN_HTTP_TIMEOUT, + `http.timeout is ${timeout}; it must stay at ${MIN_HTTP_TIMEOUT} or higher so a transient network stall does not fail the build`, + ); +});