mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-17 13:01:05 +01:00
fix(release): retry the finalize install while a large tarball lags its npm metadata (#21925)
## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Since #21917, each canary's npm confirmation, git tag, Docker build and onboarding smoke run in `canary-finalize.yml` > - The first canary through it, `2026.1116.0-canary.814700`, passed the master-position, visibility, provenance (31 of 31) and dist-tag checks > - It then failed the clean-prefix install: the 524 MB `@paperclipai/server` tarball returned 404 while npm already listed the version > - Without a tag, nightly cannot select that canary, and no Docker image or smoke runs for it > - This pull request retries that install for a bounded time > - The benefit is that a large tarball's normal lag no longer stops a canary from finishing ## Linked Issues or Issue Description **What happened?** Canary finalize run 43927092510 for `2026.1116.0-canary.814700`: `npm error 404 Not Found - GET https://registry.npmjs.org/@paperclipai/server/-/server-2026.1116.0-canary.814700.tgz`. npm's packument records that version at 18:31:41Z. The install ran at 18:32:03Z, and the tarball still returned 404 at 18:34:39 and 18:35:15. `@paperclipai/shared` and `paperclipai` tarballs at the same version returned 200. **Expected behavior** Finalize waits, within a bound, until the published tarball downloads, then tags the canary. Refs #21917 ## What Changed - `scripts/release-lib.sh`: `wait_for_npm_installable` retries `verify_npm_installable` up to N attempts with a fixed delay. - `scripts/confirm-canary-publication.sh`: uses it with 40 attempts × 30 s (`NPM_INSTALL_VERIFY_ATTEMPTS`, `NPM_INSTALL_VERIFY_DELAY_SECONDS`). - `.github/workflows/canary-finalize.yml`: `confirm_canary` timeout 45 → 75 min, to fit the existing 30-minute visibility budget plus the 20-minute install budget. - Test and `doc/RELEASING.md`. ## Verification - `node --test scripts/release-lib.test.mjs`: 23 of 23 pass, including a new test: immediate success, recovery after 3 failures (4 attempts, warning names attempt 3/5), and failure after the bound (4 of 4 attempts). - `node --test scripts/__tests__/release-verify-workflow.test.mjs`: 73 of 73 pass. - After merge: the next canary's finalize run passes the install and pushes its tag. `canary.814700` itself is recovered by rerunning its finalize run once its tarball downloads. ## Risks - Low. A tarball that never appears still fails finalize, after 20 minutes instead of at once. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, in Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [ ] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b816c95f96
commit
0e8447fe77
7 files changed
+94
-13
No files matched your search
@@ -32,7 +32,7 @@ jobs:
|
||||
name: Confirm canary on npm and tag it
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
timeout-minutes: 75
|
||||
permissions:
|
||||
contents: write
|
||||
actions: write
|
||||
|
||||
@@ -638,7 +638,7 @@ jobs:
|
||||
(needs.smoke_nightly.result == 'success' ||
|
||||
(needs.smoke_nightly.result == 'skipped' && github.event_name == 'workflow_dispatch' && inputs.dry_run))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
timeout-minutes: 170
|
||||
environment: npm-canary
|
||||
# The workflow-level concurrency group is per event, so a forced dispatch
|
||||
# nightly could otherwise overlap the scheduled one and race it to the
|
||||
@@ -927,7 +927,7 @@ jobs:
|
||||
(needs.verify_beta_candidate.result == 'success' ||
|
||||
(needs.verify_beta_candidate.result == 'skipped' && needs.select_beta.outputs.mode == 'promote'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
timeout-minutes: 170
|
||||
environment: npm-beta
|
||||
# Serialize beta publishes so two dispatches cannot race to the same next
|
||||
# -beta.N version; release.sh additionally refuses to double-publish a
|
||||
@@ -1392,7 +1392,7 @@ jobs:
|
||||
if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && !inputs.dry_run && needs.preflight_stable.result == 'success' && needs.verify_stable.result == 'success' && needs.release_runner_source.result == 'success' && (needs.release_runner_source.outputs.required == 'false' || needs.release_runner_binaries.result == 'success') }}
|
||||
needs: [preflight_stable, verify_stable, release_runner_source, release_runner_binaries]
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 150
|
||||
timeout-minutes: 170
|
||||
environment: npm-stable
|
||||
permissions:
|
||||
contents: write
|
||||
|
||||
+1
-1
@@ -109,7 +109,7 @@ Every push to `master` runs the canary path inside [`.github/workflows/release.y
|
||||
- requires the source commit to be on master and the canary number to come from it (`N / 100` equals the commit's master position), so a dispatch with invented inputs cannot tag another commit
|
||||
- checks npm provenance ([`scripts/verify-canary-provenance.mjs`](../scripts/verify-canary-provenance.mjs)): any package whose provenance names something other than `.github/workflows/release.yml` on `refs/heads/master` at the source commit fails the check. A package with no provenance is accepted, because that is the supported retry after a duplicate transparency-log entry, only the trusted release workflow can publish, and the number check above already ties the version to the commit. Brief npm errors are retried.
|
||||
- verifies that `canary` resolves to this version or a newer canary, and that published internal dependencies exist on npm
|
||||
- installs the exact `paperclipai@<version>` into a clean temporary prefix
|
||||
- installs the exact `paperclipai@<version>` into a clean temporary prefix, retrying every 30 s within a 20-minute deadline that also stops a stuck install (`NPM_INSTALL_VERIFY_TIMEOUT_SECONDS`, `NPM_INSTALL_VERIFY_DELAY_SECONDS`; nightly, beta and stable use the same retry for their `paperclipai@<dist-tag>` install): npm can expose a large package's metadata minutes before its tarball downloads (on 2026-11-16 `@paperclipai/server`, about 524 MB unpacked, still returned 404 several minutes after its recorded publish time)
|
||||
- fails by default if npm leaves `latest` pointing at a canary; use `--allow-canary-latest` only when that state is intentional
|
||||
- creates and pushes the git tag `canary/vYYYY.MDD.P-canary.N` (a tag that already points at the same commit is accepted, so a rerun is safe). Nightly still selects only tagged canaries.
|
||||
- dispatches `docker.yml` at the tag
|
||||
|
||||
@@ -60,6 +60,7 @@ else
|
||||
fi
|
||||
|
||||
release_info "==> Installing paperclipai@${version} into a clean prefix..."
|
||||
verify_npm_installable "paperclipai@${version}" "$version" || release_fail "paperclipai@${version} did not install cleanly"
|
||||
wait_for_npm_installable "paperclipai@${version}" "$version" "${NPM_INSTALL_VERIFY_TIMEOUT_SECONDS:-1200}" "${NPM_INSTALL_VERIFY_DELAY_SECONDS:-30}" \
|
||||
|| release_fail "paperclipai@${version} did not install cleanly"
|
||||
|
||||
push_release_tag_at "$PUBLISH_REMOTE" "$tag_name" "$source_sha"
|
||||
+30
-1
@@ -572,12 +572,18 @@ wait_for_npm_package_versions() {
|
||||
verify_npm_installable() {
|
||||
local package_spec="$1"
|
||||
local expected_version="$2"
|
||||
local install_timeout_seconds="${3:-}"
|
||||
local install_dir
|
||||
local installed_version
|
||||
local install=(npm install --prefix "" "$package_spec" --no-audit --no-fund)
|
||||
|
||||
install_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-release-install.XXXXXX")"
|
||||
install[3]="$install_dir"
|
||||
if [ -n "$install_timeout_seconds" ] && command -v timeout >/dev/null 2>&1; then
|
||||
install=(timeout --kill-after=10 "$install_timeout_seconds" "${install[@]}")
|
||||
fi
|
||||
|
||||
if ! npm install --prefix "$install_dir" "$package_spec" --no-audit --no-fund; then
|
||||
if ! "${install[@]}"; then
|
||||
rm -rf "$install_dir"
|
||||
return 1
|
||||
fi
|
||||
@@ -588,6 +594,29 @@ verify_npm_installable() {
|
||||
[ "$installed_version" = "$expected_version" ]
|
||||
}
|
||||
|
||||
wait_for_npm_installable() {
|
||||
local package_spec="$1"
|
||||
local expected_version="$2"
|
||||
local deadline_seconds="${3:-1200}"
|
||||
local delay_seconds="${4:-30}"
|
||||
local deadline=$(( $(date +%s) + deadline_seconds ))
|
||||
local attempt=1
|
||||
local remaining
|
||||
|
||||
while :; do
|
||||
remaining=$(( deadline - $(date +%s) ))
|
||||
[ "$remaining" -gt 0 ] || return 1
|
||||
if verify_npm_installable "$package_spec" "$expected_version" "$remaining"; then
|
||||
return 0
|
||||
fi
|
||||
remaining=$(( deadline - $(date +%s) ))
|
||||
[ "$remaining" -gt 0 ] || return 1
|
||||
release_warn "${package_spec} did not install yet (attempt ${attempt}, ${remaining}s left); a large tarball can lag its metadata. Retrying in $(( delay_seconds < remaining ? delay_seconds : remaining ))s."
|
||||
sleep $(( delay_seconds < remaining ? delay_seconds : remaining ))
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
}
|
||||
|
||||
wait_for_release_registry_state() {
|
||||
local attempts="${1:-12}"
|
||||
local delay_seconds="${2:-5}"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
@@ -395,15 +395,22 @@ test("the workflow budget fails closed after the last registry check", () => {
|
||||
assert.equal(result.calls.match(/^sleep /gm)?.length, budget.verifyAttempts - 1);
|
||||
});
|
||||
|
||||
test("every publish job budgets for build time and four delayed packages", () => {
|
||||
test("every publish job budgets for build time, four delayed packages and the install deadline", () => {
|
||||
const { verifyAttempts, verifyDelaySeconds } = workflowVerifyBudget();
|
||||
const pollingSeconds = (verifyAttempts - 1) * verifyDelaySeconds;
|
||||
const requiredSeconds = 30 * 60 + 4 * pollingSeconds;
|
||||
const releaseScript = readFileSync(new URL("./release.sh", import.meta.url), "utf8");
|
||||
const installDeadlineSeconds = Number(releaseScript.match(/NPM_INSTALL_VERIFY_TIMEOUT_SECONDS:-(\d+)/)?.[1]);
|
||||
assert.ok(installDeadlineSeconds > 0);
|
||||
|
||||
for (const job of ["publish_canary", "publish_nightly", "publish_beta", "publish_stable"]) {
|
||||
for (const [job, requiredSeconds] of [
|
||||
["publish_canary", 30 * 60 + 4 * pollingSeconds],
|
||||
["publish_nightly", 30 * 60 + 4 * pollingSeconds + installDeadlineSeconds],
|
||||
["publish_beta", 30 * 60 + 4 * pollingSeconds + installDeadlineSeconds],
|
||||
["publish_stable", 30 * 60 + 4 * pollingSeconds + installDeadlineSeconds],
|
||||
]) {
|
||||
const body = releaseWorkflow.split(`\n ${job}:\n`)[1]?.split(/\n [a-z_]+:\n/)[0] ?? "";
|
||||
const timeoutMinutes = Number(body.match(/^ timeout-minutes: (\d+)$/m)?.[1]);
|
||||
assert.ok(timeoutMinutes * 60 > requiredSeconds, `${job} must leave time beyond build and polling`);
|
||||
assert.ok(timeoutMinutes * 60 > requiredSeconds, `${job} must leave time beyond build, polling and the install deadline`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -497,3 +504,47 @@ test("a canary version is accepted only for the master commit whose position num
|
||||
const side = fixture.git(fixture.repo, "rev-parse", "HEAD").trim();
|
||||
assert.throws(() => confirm("2026.1116.0-canary.2600", side), /is not on origin\/master/);
|
||||
});
|
||||
|
||||
function installRetryFixture({ failures = 0, hangSeconds = 0 }) {
|
||||
const dir = mkdtempSync(join(tmpdir(), "paperclip-install-retry-"));
|
||||
const bin = join(dir, "bin");
|
||||
mkdirSync(bin);
|
||||
const counter = join(dir, "attempts");
|
||||
writeFileSync(counter, "0");
|
||||
writeExecutable(join(bin, "npm"), `#!/usr/bin/env bash
|
||||
n=$(( $(cat "${counter}") + 1 )); echo "$n" > "${counter}"
|
||||
if [ "${hangSeconds}" -gt 0 ]; then sleep "${hangSeconds}"; fi
|
||||
if [ "$n" -le "${failures}" ]; then echo "npm error code E404" >&2; exit 1; fi
|
||||
prefix="$3"; mkdir -p "$prefix/node_modules/paperclipai"
|
||||
printf '{"version":"1.2.3"}' > "$prefix/node_modules/paperclipai/package.json"
|
||||
`);
|
||||
const run = (deadline, delay = 0) => {
|
||||
const started = Date.now();
|
||||
const result = spawnSync("bash", ["-c", `source "${repoRoot}/scripts/release-lib.sh"; wait_for_npm_installable paperclipai@canary 1.2.3 ${deadline} ${delay}`],
|
||||
{ encoding: "utf8", env: { ...process.env, PATH: `${bin}:${process.env.PATH}` } });
|
||||
return { status: result.status, seconds: (Date.now() - started) / 1000, attempts: Number(readFileSync(counter, "utf8")), stderr: result.stderr };
|
||||
};
|
||||
return { run };
|
||||
}
|
||||
|
||||
test("a release install is retried while its tarball lags the metadata", () => {
|
||||
const recovered = installRetryFixture({ failures: 3 }).run(60);
|
||||
assert.equal(recovered.status, 0);
|
||||
assert.equal(recovered.attempts, 4);
|
||||
assert.match(recovered.stderr, /attempt 3, \d+s left/);
|
||||
assert.equal(installRetryFixture({}).run(60).attempts, 1);
|
||||
});
|
||||
|
||||
const hasTimeoutCommand = spawnSync("bash", ["-c", "command -v timeout"], { encoding: "utf8" }).status === 0;
|
||||
|
||||
test("install retries stop at the deadline", () => {
|
||||
const exhausted = installRetryFixture({ failures: 1000 }).run(3, 1);
|
||||
assert.notEqual(exhausted.status, 0);
|
||||
assert.ok(exhausted.seconds < 8, `took ${exhausted.seconds}s`);
|
||||
});
|
||||
|
||||
test("an install that hangs is stopped at the deadline", { skip: !hasTimeoutCommand && "needs the timeout command (GNU coreutils)" }, () => {
|
||||
const hung = installRetryFixture({ hangSeconds: 120 }).run(3, 1);
|
||||
assert.notEqual(hung.status, 0);
|
||||
assert.ok(hung.seconds < 20, `a hung install outlived the deadline: ${hung.seconds}s`);
|
||||
});
|
||||
+1
-1
@@ -485,7 +485,7 @@ else
|
||||
fi
|
||||
|
||||
release_info " Installing paperclipai@$DIST_TAG into a clean prefix..."
|
||||
if ! verify_npm_installable "paperclipai@$DIST_TAG" "$TARGET_PUBLISH_VERSION"; then
|
||||
if ! wait_for_npm_installable "paperclipai@$DIST_TAG" "$TARGET_PUBLISH_VERSION" "${NPM_INSTALL_VERIFY_TIMEOUT_SECONDS:-1200}" "${NPM_INSTALL_VERIFY_DELAY_SECONDS:-30}"; then
|
||||
release_fail "paperclipai@$DIST_TAG did not install cleanly at expected version ${TARGET_PUBLISH_VERSION}"
|
||||
fi
|
||||
release_info " ✓ Clean-prefix install resolved ${TARGET_PUBLISH_VERSION}"
|
||||
|
||||
Reference in new issue
Block a user