diff --git a/.github/workflows/canary-finalize.yml b/.github/workflows/canary-finalize.yml index 5950c076d2f..cc0b56a0142 100644 --- a/.github/workflows/canary-finalize.yml +++ b/.github/workflows/canary-finalize.yml @@ -32,7 +32,7 @@ jobs: name: Confirm canary on npm and tag it if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest - timeout-minutes: 45 + timeout-minutes: 75 permissions: contents: write actions: write diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 744eea46d2a..92c8aff2ece 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -638,7 +638,7 @@ jobs: (needs.smoke_nightly.result == 'success' || (needs.smoke_nightly.result == 'skipped' && github.event_name == 'workflow_dispatch' && inputs.dry_run)) runs-on: ubuntu-latest - timeout-minutes: 150 + timeout-minutes: 170 environment: npm-canary # The workflow-level concurrency group is per event, so a forced dispatch # nightly could otherwise overlap the scheduled one and race it to the @@ -927,7 +927,7 @@ jobs: (needs.verify_beta_candidate.result == 'success' || (needs.verify_beta_candidate.result == 'skipped' && needs.select_beta.outputs.mode == 'promote')) runs-on: ubuntu-latest - timeout-minutes: 150 + timeout-minutes: 170 environment: npm-beta # Serialize beta publishes so two dispatches cannot race to the same next # -beta.N version; release.sh additionally refuses to double-publish a @@ -1392,7 +1392,7 @@ jobs: if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && !inputs.dry_run && needs.preflight_stable.result == 'success' && needs.verify_stable.result == 'success' && needs.release_runner_source.result == 'success' && (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') }} needs: [preflight_stable, verify_stable, release_runner_source, release_runner_binaries] runs-on: ubuntu-latest - timeout-minutes: 150 + timeout-minutes: 170 environment: npm-stable permissions: contents: write diff --git a/doc/RELEASING.md b/doc/RELEASING.md index bd0317195ca..ea58a4d96e7 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -109,7 +109,7 @@ Every push to `master` runs the canary path inside [`.github/workflows/release.y - requires the source commit to be on master and the canary number to come from it (`N / 100` equals the commit's master position), so a dispatch with invented inputs cannot tag another commit - checks npm provenance ([`scripts/verify-canary-provenance.mjs`](../scripts/verify-canary-provenance.mjs)): any package whose provenance names something other than `.github/workflows/release.yml` on `refs/heads/master` at the source commit fails the check. A package with no provenance is accepted, because that is the supported retry after a duplicate transparency-log entry, only the trusted release workflow can publish, and the number check above already ties the version to the commit. Brief npm errors are retried. - verifies that `canary` resolves to this version or a newer canary, and that published internal dependencies exist on npm -- installs the exact `paperclipai@` into a clean temporary prefix +- installs the exact `paperclipai@` into a clean temporary prefix, retrying every 30 s within a 20-minute deadline that also stops a stuck install (`NPM_INSTALL_VERIFY_TIMEOUT_SECONDS`, `NPM_INSTALL_VERIFY_DELAY_SECONDS`; nightly, beta and stable use the same retry for their `paperclipai@` install): npm can expose a large package's metadata minutes before its tarball downloads (on 2026-11-16 `@paperclipai/server`, about 524 MB unpacked, still returned 404 several minutes after its recorded publish time) - fails by default if npm leaves `latest` pointing at a canary; use `--allow-canary-latest` only when that state is intentional - creates and pushes the git tag `canary/vYYYY.MDD.P-canary.N` (a tag that already points at the same commit is accepted, so a rerun is safe). Nightly still selects only tagged canaries. - dispatches `docker.yml` at the tag diff --git a/scripts/confirm-canary-publication.sh b/scripts/confirm-canary-publication.sh index 47ace380829..9ab2d6ecc1f 100755 --- a/scripts/confirm-canary-publication.sh +++ b/scripts/confirm-canary-publication.sh @@ -60,6 +60,7 @@ else fi release_info "==> Installing paperclipai@${version} into a clean prefix..." -verify_npm_installable "paperclipai@${version}" "$version" || release_fail "paperclipai@${version} did not install cleanly" +wait_for_npm_installable "paperclipai@${version}" "$version" "${NPM_INSTALL_VERIFY_TIMEOUT_SECONDS:-1200}" "${NPM_INSTALL_VERIFY_DELAY_SECONDS:-30}" \ + || release_fail "paperclipai@${version} did not install cleanly" push_release_tag_at "$PUBLISH_REMOTE" "$tag_name" "$source_sha" diff --git a/scripts/release-lib.sh b/scripts/release-lib.sh index 798e34ba586..f2db11d0973 100644 --- a/scripts/release-lib.sh +++ b/scripts/release-lib.sh @@ -572,12 +572,18 @@ wait_for_npm_package_versions() { verify_npm_installable() { local package_spec="$1" local expected_version="$2" + local install_timeout_seconds="${3:-}" local install_dir local installed_version + local install=(npm install --prefix "" "$package_spec" --no-audit --no-fund) install_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-release-install.XXXXXX")" + install[3]="$install_dir" + if [ -n "$install_timeout_seconds" ] && command -v timeout >/dev/null 2>&1; then + install=(timeout --kill-after=10 "$install_timeout_seconds" "${install[@]}") + fi - if ! npm install --prefix "$install_dir" "$package_spec" --no-audit --no-fund; then + if ! "${install[@]}"; then rm -rf "$install_dir" return 1 fi @@ -588,6 +594,29 @@ verify_npm_installable() { [ "$installed_version" = "$expected_version" ] } +wait_for_npm_installable() { + local package_spec="$1" + local expected_version="$2" + local deadline_seconds="${3:-1200}" + local delay_seconds="${4:-30}" + local deadline=$(( $(date +%s) + deadline_seconds )) + local attempt=1 + local remaining + + while :; do + remaining=$(( deadline - $(date +%s) )) + [ "$remaining" -gt 0 ] || return 1 + if verify_npm_installable "$package_spec" "$expected_version" "$remaining"; then + return 0 + fi + remaining=$(( deadline - $(date +%s) )) + [ "$remaining" -gt 0 ] || return 1 + release_warn "${package_spec} did not install yet (attempt ${attempt}, ${remaining}s left); a large tarball can lag its metadata. Retrying in $(( delay_seconds < remaining ? delay_seconds : remaining ))s." + sleep $(( delay_seconds < remaining ? delay_seconds : remaining )) + attempt=$((attempt + 1)) + done +} + wait_for_release_registry_state() { local attempts="${1:-12}" local delay_seconds="${2:-5}" diff --git a/scripts/release-lib.test.mjs b/scripts/release-lib.test.mjs index c890ffee0ac..3e78f3e8826 100644 --- a/scripts/release-lib.test.mjs +++ b/scripts/release-lib.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { execFileSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -395,15 +395,22 @@ test("the workflow budget fails closed after the last registry check", () => { assert.equal(result.calls.match(/^sleep /gm)?.length, budget.verifyAttempts - 1); }); -test("every publish job budgets for build time and four delayed packages", () => { +test("every publish job budgets for build time, four delayed packages and the install deadline", () => { const { verifyAttempts, verifyDelaySeconds } = workflowVerifyBudget(); const pollingSeconds = (verifyAttempts - 1) * verifyDelaySeconds; - const requiredSeconds = 30 * 60 + 4 * pollingSeconds; + const releaseScript = readFileSync(new URL("./release.sh", import.meta.url), "utf8"); + const installDeadlineSeconds = Number(releaseScript.match(/NPM_INSTALL_VERIFY_TIMEOUT_SECONDS:-(\d+)/)?.[1]); + assert.ok(installDeadlineSeconds > 0); - for (const job of ["publish_canary", "publish_nightly", "publish_beta", "publish_stable"]) { + for (const [job, requiredSeconds] of [ + ["publish_canary", 30 * 60 + 4 * pollingSeconds], + ["publish_nightly", 30 * 60 + 4 * pollingSeconds + installDeadlineSeconds], + ["publish_beta", 30 * 60 + 4 * pollingSeconds + installDeadlineSeconds], + ["publish_stable", 30 * 60 + 4 * pollingSeconds + installDeadlineSeconds], + ]) { const body = releaseWorkflow.split(`\n ${job}:\n`)[1]?.split(/\n [a-z_]+:\n/)[0] ?? ""; const timeoutMinutes = Number(body.match(/^ timeout-minutes: (\d+)$/m)?.[1]); - assert.ok(timeoutMinutes * 60 > requiredSeconds, `${job} must leave time beyond build and polling`); + assert.ok(timeoutMinutes * 60 > requiredSeconds, `${job} must leave time beyond build, polling and the install deadline`); } }); @@ -497,3 +504,47 @@ test("a canary version is accepted only for the master commit whose position num const side = fixture.git(fixture.repo, "rev-parse", "HEAD").trim(); assert.throws(() => confirm("2026.1116.0-canary.2600", side), /is not on origin\/master/); }); + +function installRetryFixture({ failures = 0, hangSeconds = 0 }) { + const dir = mkdtempSync(join(tmpdir(), "paperclip-install-retry-")); + const bin = join(dir, "bin"); + mkdirSync(bin); + const counter = join(dir, "attempts"); + writeFileSync(counter, "0"); + writeExecutable(join(bin, "npm"), `#!/usr/bin/env bash +n=$(( $(cat "${counter}") + 1 )); echo "$n" > "${counter}" +if [ "${hangSeconds}" -gt 0 ]; then sleep "${hangSeconds}"; fi +if [ "$n" -le "${failures}" ]; then echo "npm error code E404" >&2; exit 1; fi +prefix="$3"; mkdir -p "$prefix/node_modules/paperclipai" +printf '{"version":"1.2.3"}' > "$prefix/node_modules/paperclipai/package.json" +`); + const run = (deadline, delay = 0) => { + const started = Date.now(); + const result = spawnSync("bash", ["-c", `source "${repoRoot}/scripts/release-lib.sh"; wait_for_npm_installable paperclipai@canary 1.2.3 ${deadline} ${delay}`], + { encoding: "utf8", env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } }); + return { status: result.status, seconds: (Date.now() - started) / 1000, attempts: Number(readFileSync(counter, "utf8")), stderr: result.stderr }; + }; + return { run }; +} + +test("a release install is retried while its tarball lags the metadata", () => { + const recovered = installRetryFixture({ failures: 3 }).run(60); + assert.equal(recovered.status, 0); + assert.equal(recovered.attempts, 4); + assert.match(recovered.stderr, /attempt 3, \d+s left/); + assert.equal(installRetryFixture({}).run(60).attempts, 1); +}); + +const hasTimeoutCommand = spawnSync("bash", ["-c", "command -v timeout"], { encoding: "utf8" }).status === 0; + +test("install retries stop at the deadline", () => { + const exhausted = installRetryFixture({ failures: 1000 }).run(3, 1); + assert.notEqual(exhausted.status, 0); + assert.ok(exhausted.seconds < 8, `took ${exhausted.seconds}s`); +}); + +test("an install that hangs is stopped at the deadline", { skip: !hasTimeoutCommand && "needs the timeout command (GNU coreutils)" }, () => { + const hung = installRetryFixture({ hangSeconds: 120 }).run(3, 1); + assert.notEqual(hung.status, 0); + assert.ok(hung.seconds < 20, `a hung install outlived the deadline: ${hung.seconds}s`); +}); diff --git a/scripts/release.sh b/scripts/release.sh index 79b276844d8..6b7e71d8f87 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -485,7 +485,7 @@ else fi release_info " Installing paperclipai@$DIST_TAG into a clean prefix..." - if ! verify_npm_installable "paperclipai@$DIST_TAG" "$TARGET_PUBLISH_VERSION"; then + if ! wait_for_npm_installable "paperclipai@$DIST_TAG" "$TARGET_PUBLISH_VERSION" "${NPM_INSTALL_VERIFY_TIMEOUT_SECONDS:-1200}" "${NPM_INSTALL_VERIFY_DELAY_SECONDS:-30}"; then release_fail "paperclipai@$DIST_TAG did not install cleanly at expected version ${TARGET_PUBLISH_VERSION}" fi release_info " ✓ Clean-prefix install resolved ${TARGET_PUBLISH_VERSION}"