diff --git a/.github/scripts/tests/post-merge-runner-routing.test.mjs b/.github/scripts/tests/post-merge-runner-routing.test.mjs new file mode 100644 index 0000000000..95c2a1c5ef --- /dev/null +++ b/.github/scripts/tests/post-merge-runner-routing.test.mjs @@ -0,0 +1,83 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { runInNewContext } from "node:vm"; + +const fleet = "runs-on/fleet=paperclip-post-merge-x64/env=public-ci"; +const sha = "a".repeat(40); +const base = { + repository: "paperclipai/paperclip", repository_id: "1170821064", + ref: "refs/heads/master", event_name: "push", sha, +}; +const expectedJobs = { + "cloud-readiness.yml": ["artifacts", "source_verified", "ready"], + "cloud-artifacts.yml": ["dispatch_migrator"], + "release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"], + "runner-chaos-evals.yml": ["chaos_and_recovery"], + "release.yml": ["plan_preview", "package_preview"], +}; +for (const [file, expectedNames] of Object.entries(expectedJobs)) { + const workflow = readFileSync(new URL(`../../workflows/${file}`, import.meta.url), "utf8"); + const jobs = [...workflow.matchAll(/^ ([a-z_]+):\n([\s\S]*?)(?=^ [a-z_]+:\n|(?![\s\S]))/gm)]; + const routed = jobs.filter(([, , body]) => body.includes(fleet)); + test(`${file}: all intended jobs carry the post-merge guard`, () => { + assert.deepEqual(routed.map(([, name]) => name).sort(), [...expectedNames].sort()); + }); + for (const [, job, body] of routed) { + const expression = body.match(/^ runs-on: \$\{\{ (.+) \}\}$/m)?.[1]; + assert.ok(expression, `${file}/${job} must use an explicit runner expression`); + const release = file === "release.yml"; + const checkRef = release || file === "release-verify.yml" || file === "runner-chaos-evals.yml"; + const inputs = { ref: sha, source_ref: sha, channel: "cloud-migrator" }; + const defaultContext = { ...base, event_name: release ? "workflow_dispatch" : "push" }; + const cases = [ + { name: "exact master source", expected: fleet }, + { name: "manual exact master source", github: { event_name: "workflow_dispatch" }, expected: fleet }, + { name: "switch disabled", enabled: "false" }, + { name: "switch absent", enabled: "" }, + { name: "malformed switch", enabled: "yes" }, + { name: "fork", github: { repository: "someone/paperclip", repository_id: "123" } }, + { name: "repository renamed or transferred", github: { repository_id: "123" } }, + { name: "unapproved PR", github: { event_name: "pull_request", ref: "refs/pull/1/merge" } }, + { name: "PR event even with master ref", github: { event_name: "pull_request" } }, + { name: "privileged PR event", github: { event_name: "pull_request_target" } }, + { name: "workflow completion event", github: { event_name: "workflow_run" } }, + { name: "repository dispatch", github: { event_name: "repository_dispatch" } }, + { name: "scheduled caller", github: { event_name: "schedule" } }, + { name: "branch workflow", github: { ref: "refs/heads/feature" } }, + { name: "release tag", github: { ref: "refs/tags/v2026.911.0" } }, + ]; + if (checkRef) { + const key = release ? "source_ref" : "ref"; + for (const value of ["b".repeat(40), "refs/pull/1/head", "master", "feature", "v1.0.0", ""]) { + cases.push({ name: `unverified source ${value || "(empty)"}`, inputs: { [key]: value } }); + } + cases.push({ name: "missing source identity", github: { sha: "" }, inputs: { [key]: "" } }); + } + if (release) { + cases.push({ name: "preview of master", inputs: { channel: "preview" } }); + cases.push({ name: "stable release", inputs: { channel: "stable" } }); + } + for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) { + test(`${file}/${job}: ${name}`, () => { + const context = { github: { ...defaultContext, ...github }, inputs: { ...inputs, ...overrides }, vars: { AWS_POST_MERGE_CI_ENABLED: enabled } }; + // These canonical contexts use boolean operators and string comparisons + // whose results match GitHub's expression evaluation. + assert.equal(runInNewContext(expression, context), expected); + const timeout = body.match(/^ timeout-minutes: (.+)$/m)?.[1]; + assert.ok(timeout, "AWS jobs need a timeout below the 45-minute instance lifetime"); + const minutes = timeout.startsWith("${{") ? runInNewContext(timeout.slice(3, -2), context) : Number(timeout); + if (expected === fleet) assert.ok(minutes > 0 && minutes < 45); + if (release && job === "plan_preview") assert.equal(minutes, expected === fleet ? 10 : 360); + }); + } + } + if (file === "release.yml") { + test("npm publisher always uses a GitHub-hosted runner", () => { + const publisher = jobs.find(([ , job]) => job === "publish_preview")?.[2]; + assert.match(publisher, /^ runs-on: ubuntu-latest$/m); + assert.match(publisher, /^ environment: npm-canary$/m); + assert.match(publisher, /^ id-token: write$/m); + }); + } +} diff --git a/.github/workflows/cloud-artifacts.yml b/.github/workflows/cloud-artifacts.yml index 53d8ab710b..ecc0dcadb2 100644 --- a/.github/workflows/cloud-artifacts.yml +++ b/.github/workflows/cloud-artifacts.yml @@ -11,7 +11,7 @@ jobs: dispatch_migrator: name: Start exact-source cloud migrator publication if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 permissions: actions: write diff --git a/.github/workflows/cloud-readiness.yml b/.github/workflows/cloud-readiness.yml index b84d984132..002da29614 100644 --- a/.github/workflows/cloud-readiness.yml +++ b/.github/workflows/cloud-readiness.yml @@ -32,7 +32,7 @@ jobs: artifacts: if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' name: Wait for exact-source cloud artifacts - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 35 permissions: contents: read @@ -55,7 +55,7 @@ jobs: name: Cloud source verified v1 needs: [verify] if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 permissions: contents: read @@ -81,7 +81,7 @@ jobs: name: Cloud deployable v1 needs: [verify, image, artifacts] if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 steps: - name: Record cloud readiness diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 72f91a3f8a..38faeff353 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -8,6 +8,9 @@ on: required: true type: string +# Caller-provided refs may name unmerged PR code. AWS is eligible only when +# the caller runs on canonical master and verifies that event's exact SHA. +# The organization group also restricts these workflow files to master. jobs: runner_chaos_evals: name: Pre-release Runner chaos evals @@ -17,7 +20,7 @@ jobs: typecheck: name: Typecheck - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -74,7 +77,7 @@ jobs: general_tests: name: General tests (${{ matrix.group_label }}) - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -181,7 +184,7 @@ jobs: serialized_tests: name: Serialized tests (${{ matrix.shard_label }}) - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -230,7 +233,7 @@ jobs: runner_workflow_evals: name: Runner workflow eval scorer contract - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -260,7 +263,7 @@ jobs: verify_paperclip_runner: name: Verify Paperclip Runner - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -314,7 +317,7 @@ jobs: build: name: Build - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4653a1f85c..a11e7967fc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -76,11 +76,15 @@ env: jobs: plan_preview: + # Only the current master commit can use AWS. A preview or older source + # falls back to GitHub-hosted runners, including raced merge dispatches. name: Check preview artifacts if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} permissions: contents: read + # Preserve the previous hosted default; only AWS needs the Fleet limit. + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 10 || 360 }} outputs: image: ${{ steps.plan.outputs.image }} packages: ${{ steps.plan.outputs.packages }} @@ -104,7 +108,7 @@ jobs: name: Build preview migrator needs: plan_preview if: needs.plan_preview.outputs.packages == 'true' - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 30 permissions: contents: read @@ -141,6 +145,7 @@ jobs: retention-days: 7 publish_preview: + # npm trusted publishing supports GitHub-hosted runners only. name: Publish preview migrator needs: [plan_preview, package_preview] if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success' diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index 42f8d6f307..65a453ee8c 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -20,8 +20,8 @@ concurrency: jobs: chaos_and_recovery: name: Restart, replay, trace, and recovery faults - runs-on: ubuntu-latest - timeout-minutes: 45 + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} permissions: contents: read diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index 7af553e247..5c1b421a15 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -123,6 +123,36 @@ registry checks can be rerun without deploying or changing mutable npm channels. When reverting this workflow, restore the master push trigger in `docker-cloud.yml` in the same change so master images continue to build. +## Reserved AWS verification capacity + +`AWS_POST_MERGE_CI_ENABLED=true` routes cloud source verification, artifact +waiting, readiness signals, and exact-master migrator preparation to the +`paperclip-post-merge` runner group. The separate Fleet label is +`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. Its 36 reserved slots use +the same four-vCPU, 16-GiB machines as approved PR jobs. PR capacity is reduced +to 64; image capacity stays at eight. The total ceiling remains 108 runners. +This keeps PR bursts from consuming every post-merge verification slot. + +Every selector checks the canonical repository name and ID, master ref, and a +push or manual event. Reusable verification also requires `inputs.ref` to equal +that event's `github.sha`. The migrator route requires `cloud-migrator` and +`inputs.source_ref == github.sha`. Branch/tag refs, PR events, arbitrary preview +sources, and missing or disabled switches use GitHub-hosted runners. If another +merge lands before a migrator dispatch resolves master, the older source uses +GitHub-hosted runners too. npm publication always remains GitHub-hosted to keep +its trusted-publisher identity. + +Before enabling the switch, deploy the separate Fleet and restrict its GitHub +runner group to repository ID `1170821064` and these workflows at +`refs/heads/master`: `cloud-readiness.yml`, `cloud-artifacts.yml`, +`release-verify.yml`, `runner-chaos-evals.yml`, and `release.yml`. Do not authorize +PR-controlled workflow versions. PR placement retains its independent pinned +workflow and six-account author/actor allowlist. + +Disable the switch and rerun the whole workflow to restore GitHub-hosted +placement. Assigned jobs keep their original runners. Readiness requirements, +source checks, and npm integrity checks are unchanged. + ## AWS cloud build routing `AWS_CLOUD_BUILDS_ENABLED=true` routes the Docker cloud job to the