diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a11e7967fc..878899cb24 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,21 @@ env: # 07:07:40.) The publish jobs' timeout-minutes are sized for several # laggard packages; if most of a batch lags the full budget, npm is having # a real incident and the job failing is correct. - NPM_PUBLISH_VERIFY_ATTEMPTS: "60" + # + # Raised to 30 minutes on 2026-09-14, when 10 was not enough twice in a + # row: @paperclipai/server was accepted at 17:49:09 and became visible at + # 18:04:29 — five minutes after the poll gave up. Each timeout aborts the + # release mid-batch, leaving that version half-published, and because the + # next version number is derived from what is already on npm the retry + # moves to a new number and meets the same lag. Waiting is cheap; a + # half-published release is not. + # + # Packages are published and verified one at a time, so the publish jobs' + # timeout-minutes went to 150 alongside this: a 30-minute build plus four + # packages each lagging the full budget still finishes inside the job, + # instead of the job timing out mid-batch and leaving the same + # half-published state this budget exists to avoid. + NPM_PUBLISH_VERIFY_ATTEMPTS: "180" NPM_PUBLISH_VERIFY_DELAY_SECONDS: "10" jobs: @@ -326,7 +340,7 @@ jobs: if: github.event_name == 'push' needs: verify_canary runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 150 environment: npm-canary outputs: canary_version: ${{ steps.canary_tag.outputs.version }} @@ -582,7 +596,7 @@ jobs: (needs.smoke_nightly.result == 'success' || (needs.smoke_nightly.result == 'skipped' && github.event_name == 'workflow_dispatch' && inputs.dry_run)) runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 150 environment: npm-canary # The workflow-level concurrency group is per event, so a forced dispatch # nightly could otherwise overlap the scheduled one and race it to the @@ -848,7 +862,7 @@ jobs: (needs.verify_beta_candidate.result == 'success' || (needs.verify_beta_candidate.result == 'skipped' && needs.select_beta.outputs.mode == 'promote')) runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 150 environment: npm-beta # Serialize beta publishes so two dispatches cannot race to the same next # -beta.N version; release.sh additionally refuses to double-publish a @@ -1271,7 +1285,7 @@ jobs: if: github.event_name == 'workflow_dispatch' && inputs.channel == 'stable' && !inputs.dry_run needs: [preflight_stable, verify_stable] runs-on: ubuntu-latest - timeout-minutes: 90 + timeout-minutes: 150 environment: npm-stable permissions: contents: write diff --git a/scripts/release-lib.test.mjs b/scripts/release-lib.test.mjs index cc51aa7b15..4d8da63ab2 100644 --- a/scripts/release-lib.test.mjs +++ b/scripts/release-lib.test.mjs @@ -6,6 +6,14 @@ import { join } from "node:path"; import test from "node:test"; const repoRoot = new URL("..", import.meta.url).pathname.replace(/\/$/, ""); +const releaseWorkflow = readFileSync(new URL("../.github/workflows/release.yml", import.meta.url), "utf8"); + +function workflowVerifyBudget() { + return { + verifyAttempts: Number(releaseWorkflow.match(/^ NPM_PUBLISH_VERIFY_ATTEMPTS: "(\d+)"$/m)?.[1]), + verifyDelaySeconds: Number(releaseWorkflow.match(/^ NPM_PUBLISH_VERIFY_DELAY_SECONDS: "(\d+)"$/m)?.[1]), + }; +} function writeExecutable(path, body) { writeFileSync(path, body, { mode: 0o755 }); @@ -18,6 +26,9 @@ function runPublishHelper({ callerPipefail = true, publishTool = "pnpm", waitForRegistry = false, + npmVersionExistsAfterChecks = 0, + verifyAttempts = 1, + verifyDelaySeconds = 0, }) { const fixtureDir = mkdtempSync(join(tmpdir(), "paperclip-release-lib-")); const binDir = join(fixtureDir, "bin"); @@ -74,9 +85,18 @@ exit 1 `#!/usr/bin/env bash set -euo pipefail printf 'npm %s\\n' "$*" >> "$FAKE_CALL_LOG" -if [ "$1" = "view" ] && [ "$NPM_VERSION_EXISTS" = "true" ]; then - echo "1.2.3" - exit 0 +if [ "$1" = "view" ]; then + checks=0 + if [ -f "$FAKE_STATE_DIR/view-checks" ]; then + read -r checks < "$FAKE_STATE_DIR/view-checks" + fi + checks=$((checks + 1)) + echo "$checks" > "$FAKE_STATE_DIR/view-checks" + if [ "$NPM_VERSION_EXISTS" = "true" ] || + { [ "$NPM_VERSION_EXISTS_AFTER_CHECKS" -gt 0 ] && [ "$checks" -ge "$NPM_VERSION_EXISTS_AFTER_CHECKS" ]; }; then + echo "1.2.3" + exit 0 + fi fi if [ "$1" = "publish" ]; then case "$PNPM_MODE" in @@ -130,9 +150,11 @@ exec npm "$@" const script = ` ${shellOptions} source "${repoRoot}/scripts/release-lib.sh" +# Record virtual waiting so registry-delay tests remain fast and offline. +sleep() { printf 'sleep %s\\n' "$*" >> "$FAKE_CALL_LOG"; } ${ waitForRegistry - ? `publish_package_to_npm_and_wait ${distTag} @paperclipai/example 1.2.3 ${publishTool} 1 0` + ? `publish_package_to_npm_and_wait ${distTag} @paperclipai/example 1.2.3 ${publishTool} "$VERIFY_ATTEMPTS" "$VERIFY_DELAY_SECONDS"` : `publish_package_to_npm ${distTag} @paperclipai/example 1.2.3 ${publishTool}` } `; @@ -149,6 +171,9 @@ ${ FAKE_CALL_LOG: callLog, FAKE_STATE_DIR: stateDir, NPM_VERSION_EXISTS: npmVersionExists ? "true" : "false", + NPM_VERSION_EXISTS_AFTER_CHECKS: String(npmVersionExistsAfterChecks), + VERIFY_ATTEMPTS: String(verifyAttempts), + VERIFY_DELAY_SECONDS: String(verifyDelaySeconds), PNPM_MODE: pnpmMode, REPO_ROOT: fixtureDir, }, @@ -264,3 +289,65 @@ test("publish_package_to_npm_and_wait blocks the release when registry visibilit assert.match(result.calls, /^npm view @paperclipai\/example@1\.2\.3 version$/m); assert.match(result.output, /did not become registry-visible/); }); + +test("the workflow budget tolerates the observed 15-minute 20-second registry delay", () => { + const budget = workflowVerifyBudget(); + assert.ok(budget.verifyDelaySeconds > 0); + const delayedCheck = Math.ceil((15 * 60 + 20) / budget.verifyDelaySeconds) + 1; + const result = runPublishHelper({ + pnpmMode: "success", + waitForRegistry: true, + npmVersionExistsAfterChecks: delayedCheck, + ...budget, + }); + + assert.equal(result.status, 0, result.output); + assert.equal(result.calls.match(/^pnpm publish /gm)?.length, 1); + assert.equal(result.calls.match(/^npm view /gm)?.length, delayedCheck); + assert.deepEqual( + result.calls.split("\n").filter((call) => call.startsWith("sleep ")), + Array(delayedCheck - 1).fill(`sleep ${budget.verifyDelaySeconds}`), + ); +}); + +test("the workflow budget does not delay an immediately visible publish", () => { + const result = runPublishHelper({ + pnpmMode: "success", + npmVersionExists: true, + waitForRegistry: true, + ...workflowVerifyBudget(), + }); + + assert.equal(result.status, 0, result.output); + assert.equal(result.calls.match(/^npm view /gm)?.length, 1); + assert.doesNotMatch(result.calls, /^sleep /m); +}); + +test("the workflow budget fails closed after the last registry check", () => { + const budget = workflowVerifyBudget(); + assert.ok(budget.verifyAttempts > 0); + const result = runPublishHelper({ + pnpmMode: "success", + waitForRegistry: true, + npmVersionExistsAfterChecks: budget.verifyAttempts + 1, + ...budget, + }); + + assert.notEqual(result.status, 0); + assert.match(result.output, /did not become registry-visible/); + assert.equal(result.calls.match(/^pnpm publish /gm)?.length, 1); + assert.equal(result.calls.match(/^npm view /gm)?.length, budget.verifyAttempts); + assert.equal(result.calls.match(/^sleep /gm)?.length, budget.verifyAttempts - 1); +}); + +test("every publish job budgets for build time and four delayed packages", () => { + const { verifyAttempts, verifyDelaySeconds } = workflowVerifyBudget(); + const pollingSeconds = (verifyAttempts - 1) * verifyDelaySeconds; + const requiredSeconds = 30 * 60 + 4 * pollingSeconds; + + for (const job of ["publish_canary", "publish_nightly", "publish_beta", "publish_stable"]) { + const body = releaseWorkflow.split(`\n ${job}:\n`)[1]?.split(/\n [a-z_]+:\n/)[0] ?? ""; + const timeoutMinutes = Number(body.match(/^ timeout-minutes: (\d+)$/m)?.[1]); + assert.ok(timeoutMinutes * 60 > requiredSeconds, `${job} must leave time beyond build and polling`); + } +}); diff --git a/server/src/__tests__/access-routes-permissions-upgrade.test.ts b/server/src/__tests__/access-routes-permissions-upgrade.test.ts index 7f9d440cfe..799701e108 100644 --- a/server/src/__tests__/access-routes-permissions-upgrade.test.ts +++ b/server/src/__tests__/access-routes-permissions-upgrade.test.ts @@ -88,6 +88,12 @@ describeEmbeddedPostgres("access routes permissions upgrade compatibility", () = let db!: Db; let tempDb: Awaited> | null = null; + // Load the large router graph during setup so a cold CI transform does not + // consume the first permission assertion's timeout budget. + beforeAll(async () => { + await import("../routes/access.js"); + }, 30_000); + beforeAll(async () => { tempDb = await startEmbeddedPostgresTestDatabase("paperclip-access-routes-permissions-upgrade-"); db = createDb(tempDb.connectionString);