fix(cargo): raise network retry and timeout to survive a transient stall

Cargo's default network settings abort a crate download after 30
seconds below 10 bytes per second, then retry 3 times. On a shared CI
runner, a brief network stall can cross that 30-second window on every
attempt, so a build with hundreds of uncached crate downloads can fail
even though the network recovers soon after.

Add a repository-root .cargo/config.toml that raises net.retry to 5
and http.timeout to 60 seconds. Cargo finds this file by an upward
directory search from the current working directory, independent of
any --manifest-path argument, so every cargo invocation in the
repository picks it up.

Add a deterministic local harness that serves a synthetic crate over
a stalling HTTP server, then proves the old defaults fail with the
exact CI error text and the new values survive the same stall. Add a
regression test that asserts the config values stay at or above the
chosen minimums, wired into the existing release-registry test list
that CI already runs.

Co-authored-by: Paperclip <noreply@paperclip.ing>

.cargo/config.toml

@@ -0,0 +1,16 @@
+# Cargo defaults abort a crate download when the transfer rate stays
+# below 10 bytes per second for 30 seconds, then retry 3 times. On a
+# shared CI runner a brief network stall can cross that 30-second window
+# on every attempt, so the whole download fails even though the network
+# recovers soon after. A wider timeout and one more retry give a stall
+# more time to pass before cargo gives up.
+#
+# Worst case for one dead crate: 6 attempts (1 try plus 5 retries) times
+# a 60-second timeout equals 360 seconds. The job that runs this build
+# has a 20-minute (1200-second) timeout, so this worst case uses 30% of
+# the budget and leaves 840 seconds for every other step.
+[net]
+retry = 5
+
+[http]
+timeout = 60

package.json

@@ -58,7 +58,7 @@
     "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs",
     "smoke:pipelines-tutorial": "./scripts/smoke/pipelines-tutorial-smoke.sh",
     "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs",
-    "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs",
+    "test:release-registry": "node --test scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/check-cargo-network-config.test.mjs",
     "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs",
     "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts",
     "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack",

scripts/cargo-network-stall-harness/.gitignore

@@ -0,0 +1 @@
+.scratch/

scripts/cargo-network-stall-harness/run-differential.sh

@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# Reproduce the CI network stall on purpose, then prove the new cargo
+# config survives it.
+#
+# The script builds one synthetic crate and serves it from a local HTTP
+# server. The server holds the download body empty for a fixed stall
+# period, then sends it in full. Two arms fetch that crate:
+#
+#   Arm A (the control) forces the old cargo defaults with environment
+#   variables. The fetch must fail with the exact error text seen in CI.
+#
+#   Arm B uses the repository's own .cargo/config.toml, found by cargo's
+#   normal upward directory search from the repository root. The same
+#   stall must not fail the fetch.
+#
+# Run this script from anywhere; it resolves paths from its own location.
+set -euo pipefail
+
+HARNESS_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$HARNESS_DIR/../.." && pwd)"
+STALL_SECONDS="${STALL_SECONDS:-40}"
+CRATE_NAME="stallharnesscrate"
+CRATE_VERSION="0.1.0"
+SCRATCH="$HARNESS_DIR/.scratch"
+
+export PATH="$HOME/.cargo/bin:$PATH"
+
+if ! command -v cargo >/dev/null 2>&1; then
+  echo "cargo is not on PATH; cannot run the stall differential" >&2
+  exit 1
+fi
+
+rm -rf "$SCRATCH"
+mkdir -p "$SCRATCH/crate-src/${CRATE_NAME}-${CRATE_VERSION}/src"
+
+cat > "$SCRATCH/crate-src/${CRATE_NAME}-${CRATE_VERSION}/Cargo.toml" <<EOF
+[package]
+name = "${CRATE_NAME}"
+version = "${CRATE_VERSION}"
+edition = "2021"
+EOF
+: > "$SCRATCH/crate-src/${CRATE_NAME}-${CRATE_VERSION}/src/lib.rs"
+
+TARBALL="$SCRATCH/${CRATE_NAME}-${CRATE_VERSION}.crate"
+tar --sort=name --owner=0 --group=0 --numeric-owner \
+  -czf "$TARBALL" -C "$SCRATCH/crate-src" "${CRATE_NAME}-${CRATE_VERSION}"
+CKSUM="$(sha256sum "$TARBALL" | awk '{print $1}')"
+
+mkdir -p "$SCRATCH/consumer/src"
+cat > "$SCRATCH/consumer/Cargo.toml" <<EOF
+[package]
+name = "stall-harness-consumer"
+version = "0.0.0"
+edition = "2021"
+publish = false
+
+[dependencies]
+${CRATE_NAME} = { version = "${CRATE_VERSION}", registry = "stall-harness" }
+EOF
+echo "fn main() {}" > "$SCRATCH/consumer/src/main.rs"
+
+node "$HARNESS_DIR/stall-server.mjs" \
+  --port 0 \
+  --crate-name "$CRATE_NAME" \
+  --crate-version "$CRATE_VERSION" \
+  --cksum "$CKSUM" \
+  --stall-seconds "$STALL_SECONDS" \
+  --tarball "$TARBALL" \
+  > "$SCRATCH/server.log" 2>&1 &
+SERVER_PID=$!
+trap 'kill "$SERVER_PID" 2>/dev/null || true' EXIT
+
+for _ in $(seq 1 50); do
+  if grep -q "^LISTENING " "$SCRATCH/server.log" 2>/dev/null; then
+    break
+  fi
+  sleep 0.1
+done
+PORT="$(grep "^LISTENING " "$SCRATCH/server.log" | awk '{print $2}')"
+if [ -z "$PORT" ]; then
+  echo "stall server did not report a port" >&2
+  cat "$SCRATCH/server.log" >&2
+  exit 1
+fi
+echo "stall server listening on 127.0.0.1:$PORT (stall period ${STALL_SECONDS}s)"
+
+export CARGO_REGISTRIES_STALL_HARNESS_INDEX="sparse+http://127.0.0.1:${PORT}/index/"
+
+run_arm() {
+  local label="$1"
+  shift
+  rm -rf "$SCRATCH/cargo-home" "$SCRATCH/consumer/Cargo.lock"
+  mkdir -p "$SCRATCH/cargo-home"
+  echo "--- Arm $label ---"
+  (
+    cd "$REPO_ROOT"
+    CARGO_HOME="$SCRATCH/cargo-home" "$@" cargo fetch --manifest-path "$SCRATCH/consumer/Cargo.toml"
+  )
+}
+
+set +e
+ARM_A_OUTPUT="$(run_arm A env CARGO_NET_RETRY=3 CARGO_HTTP_TIMEOUT=30 2>&1)"
+ARM_A_STATUS=$?
+set -e
+echo "$ARM_A_OUTPUT"
+echo "Arm A exit status: $ARM_A_STATUS"
+
+set +e
+ARM_B_OUTPUT="$(run_arm B env 2>&1)"
+ARM_B_STATUS=$?
+set -e
+echo "$ARM_B_OUTPUT"
+echo "Arm B exit status: $ARM_B_STATUS"
+
+echo "--- verdict ---"
+FAIL=0
+
+if [ "$ARM_A_STATUS" -eq 0 ]; then
+  echo "FAIL: Arm A (old defaults) was expected to fail but the fetch succeeded"
+  FAIL=1
+elif ! grep -q "transfer too slow: failed to transfer more than 10 bytes in 30s" <<<"$ARM_A_OUTPUT"; then
+  echo "FAIL: Arm A did not report the expected stall error text"
+  FAIL=1
+else
+  echo "PASS: Arm A failed with the expected stall error text"
+fi
+
+if [ "$ARM_B_STATUS" -ne 0 ]; then
+  echo "FAIL: Arm B (new config) was expected to succeed but the fetch failed"
+  FAIL=1
+else
+  echo "PASS: Arm B succeeded through the same stall"
+fi
+
+exit $FAIL

scripts/cargo-network-stall-harness/stall-server.mjs

@@ -0,0 +1,97 @@
+#!/usr/bin/env node
+// A local stand-in for a cargo sparse registry. It serves one synthetic
+// crate and holds the download response body empty for a fixed period
+// before it sends the full body. This copies a real network stall: the
+// transfer rate stays at 0 bytes per second until the stall period ends.
+import http from "node:http";
+import { readFileSync } from "node:fs";
+
+function parseArgs(argv) {
+  const args = {};
+  for (let i = 0; i < argv.length; i += 2) {
+    const key = argv[i].replace(/^--/, "");
+    args[key] = argv[i + 1];
+  }
+  return args;
+}
+
+const args = parseArgs(process.argv.slice(2));
+const requestedPort = Number(args.port ?? 0);
+const crateName = args["crate-name"];
+const crateVersion = args["crate-version"];
+const cksum = args.cksum;
+const stallSeconds = Number(args["stall-seconds"]);
+const tarball = readFileSync(args.tarball);
+
+if (!crateName || !crateVersion || !cksum || !Number.isFinite(stallSeconds) || !args.tarball) {
+  console.error(
+    "usage: stall-server.mjs --port <n> --crate-name <name> --crate-version <ver> --cksum <hex> --stall-seconds <n> --tarball <path>",
+  );
+  process.exit(1);
+}
+
+// Cargo's sparse registry protocol picks the index path from the crate
+// name length. See the cargo book, "Index Format", for this rule.
+function indexPathSegment(name) {
+  if (name.length <= 2) return `${name.length}/${name}`;
+  if (name.length === 3) return `3/${name[0]}/${name}`;
+  return `${name.slice(0, 2)}/${name.slice(2, 4)}/${name}`;
+}
+
+const indexSegment = indexPathSegment(crateName);
+
+const server = http.createServer((req, res) => {
+  const url = new URL(req.url, "http://127.0.0.1");
+  const port = server.address().port;
+
+  if (url.pathname === "/index/config.json") {
+    const body = JSON.stringify({
+      dl: `http://127.0.0.1:${port}/dl`,
+      api: `http://127.0.0.1:${port}`,
+    });
+    res.writeHead(200, {
+      "content-type": "application/json",
+      "content-length": Buffer.byteLength(body),
+    });
+    res.end(body);
+    return;
+  }
+
+  if (url.pathname === `/index/${indexSegment}`) {
+    const body = `${JSON.stringify({
+      name: crateName,
+      vers: crateVersion,
+      deps: [],
+      cksum,
+      features: {},
+      yanked: false,
+    })}\n`;
+    res.writeHead(200, {
+      "content-type": "text/plain",
+      "content-length": Buffer.byteLength(body),
+    });
+    res.end(body);
+    return;
+  }
+
+  if (url.pathname === `/dl/${crateName}/${crateVersion}/download`) {
+    res.writeHead(200, {
+      "content-type": "application/octet-stream",
+      "content-length": tarball.length,
+    });
+    res.flushHeaders();
+    // Hold the body empty for the stall period, then send it all at once.
+    // This puts the transfer rate at 0 bytes/sec until the stall ends.
+    setTimeout(() => {
+      res.end(tarball);
+    }, stallSeconds * 1000);
+    return;
+  }
+
+  res.writeHead(404);
+  res.end();
+});
+
+server.listen(requestedPort, "127.0.0.1", () => {
+  console.log(`LISTENING ${server.address().port}`);
+});

scripts/check-cargo-network-config.test.mjs

@@ -0,0 +1,48 @@
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import path from "node:path";
+import test from "node:test";
+import { fileURLToPath } from "node:url";
+
+const scriptsDir = path.dirname(fileURLToPath(import.meta.url));
+const repoRoot = path.join(scriptsDir, "..");
+const configPath = path.join(repoRoot, ".cargo", "config.toml");
+
+// A cargo build with the default network settings can fail on a short
+// stall: cargo aborts a download after 30 seconds below 10 bytes per
+// second, then retries only 3 times. These minimums keep the repository
+// config wide enough to survive that kind of stall.
+const MIN_NET_RETRY = 5;
+const MIN_HTTP_TIMEOUT = 60;
+
+function readIntUnderSection(text, sectionName, key) {
+  const sectionPattern = new RegExp(`\\[${sectionName}\\]([\\s\\S]*?)(?:\\n\\[|$)`);
+  const sectionMatch = text.match(sectionPattern);
+  assert.ok(sectionMatch, `expected a [${sectionName}] section in ${configPath}`);
+  const keyPattern = new RegExp(`^\\s*${key}\\s*=\\s*(\\d+)\\s*$`, "m");
+  const keyMatch = sectionMatch[1].match(keyPattern);
+  assert.ok(keyMatch, `expected ${sectionName}.${key} in ${configPath}`);
+  return Number(keyMatch[1]);
+}
+
+test("the repository cargo config file exists at the repository root", () => {
+  assert.doesNotThrow(() => readFileSync(configPath, "utf8"));
+});
+
+test("the repository cargo config raises the network retry count", () => {
+  const text = readFileSync(configPath, "utf8");
+  const retry = readIntUnderSection(text, "net", "retry");
+  assert.ok(
+    retry >= MIN_NET_RETRY,
+    `net.retry is ${retry}; it must stay at ${MIN_NET_RETRY} or higher so a transient network stall does not fail the build`,
+  );
+});
+
+test("the repository cargo config raises the http transfer timeout", () => {
+  const text = readFileSync(configPath, "utf8");
+  const timeout = readIntUnderSection(text, "http", "timeout");
+  assert.ok(
+    timeout >= MIN_HTTP_TIMEOUT,
+    `http.timeout is ${timeout}; it must stay at ${MIN_HTTP_TIMEOUT} or higher so a transient network stall does not fail the build`,
+  );
+});