diff --git a/.github/scripts/tests/post-merge-runner-routing.test.mjs b/.github/scripts/tests/post-merge-runner-routing.test.mjs index 7c46aa47931..0e905635214 100644 --- a/.github/scripts/tests/post-merge-runner-routing.test.mjs +++ b/.github/scripts/tests/post-merge-runner-routing.test.mjs @@ -10,7 +10,8 @@ const base = { ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1, }; const expectedJobs = { - "cloud-readiness.yml": [], + "docker.yml": ["source_verification", "publication_verification"], + "cloud-readiness.yml": ["admission", "source_verified"], "cloud-migrator-artifacts.yml": [], "release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"], "runner-chaos-evals.yml": ["chaos_and_recovery"], @@ -59,9 +60,9 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { cases.push({ name: "stable release", inputs: { channel: "stable" } }); } cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet }); - cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } }); - cases.push({ name: "later rerun", github: { run_attempt: 3 } }); - cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } }); + cases.push({ name: "admitted retry", github: { run_attempt: 2 }, expected: release ? "ubuntu-latest" : fleet }); + cases.push({ name: "later retry", github: { run_attempt: 3 }, expected: release ? "ubuntu-latest" : fleet }); + cases.push({ name: "retry attempt as a string", github: { run_attempt: "2" }, expected: release ? "ubuntu-latest" : fleet }); cases.push({ name: "missing attempt", github: { run_attempt: undefined } }); for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) { test(`${file}/${job}: ${name}`, () => { @@ -88,7 +89,7 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { } -test("Cloud readiness bookkeeping never waits for the AWS verification fleet", () => { +test("Cloud source proof keeps source-only gating on protected verification capacity", () => { const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8"); const bodies = new Map(); for (const [name, needs] of [ @@ -97,7 +98,7 @@ test("Cloud readiness bookkeeping never waits for the AWS verification fleet", ( const body = workflow.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=^ [a-z_]+:|(?![\\s\\S]))`, "m"))?.[1]; assert.ok(body, `missing ${name} job`); bodies.set(name, body); - assert.match(body, /^ runs-on: ubuntu-latest$/m); + assert.match(body, /runs-on:.*paperclip-post-merge-x64/); assert.doesNotMatch(body, /^ +continue-on-error:|^ +if:.*always\(\)/m); assert.match(body, /^ if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'$/m); assert.match(body, /^ +SOURCE_SHA: \$\{\{ github.sha \}\}$/m); diff --git a/.github/workflows/cloud-migrator-artifacts.yml b/.github/workflows/cloud-migrator-artifacts.yml index 5c698230938..e60b7a54456 100644 --- a/.github/workflows/cloud-migrator-artifacts.yml +++ b/.github/workflows/cloud-migrator-artifacts.yml @@ -12,9 +12,33 @@ concurrency: cancel-in-progress: false jobs: - build: + admission: + name: Cloud image candidate admission v1 if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + admitted: ${{ steps.admission.outputs.admitted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - id: admission + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/cloud-release-admission.mjs image-only + + build: + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} timeout-minutes: 10 permissions: contents: read @@ -52,7 +76,7 @@ jobs: publish: needs: build if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' - runs-on: ubuntu-latest + runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/cloud-readiness.yml b/.github/workflows/cloud-readiness.yml index 6c709ade0ed..b9ee017ddea 100644 --- a/.github/workflows/cloud-readiness.yml +++ b/.github/workflows/cloud-readiness.yml @@ -8,14 +8,38 @@ on: permissions: {} -# Source verification must start outside the full npm release's queue. +# Source verification stays independent of npm publication and image builds. concurrency: group: cloud-readiness-${{ github.sha }} cancel-in-progress: false jobs: + admission: + name: Cloud candidate admission v1 + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 5 + permissions: + contents: read + actions: read + outputs: + admitted: ${{ steps.admission.outputs.admitted }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + sparse-checkout: scripts + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - id: admission + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/cloud-release-admission.mjs + verify: - if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' + needs: admission + if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true' permissions: contents: read uses: ./.github/workflows/release-verify.yml @@ -29,8 +53,9 @@ jobs: name: Cloud source verified v1 needs: [verify] if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' - # Bookkeeping must not wait for the AWS builders it observes. - runs-on: ubuntu-latest + # Keep the final proof off the shared hosted queue too. The protected + # fleet has a separate job ceiling and launches Spot workers on demand. + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 permissions: contents: read diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ec7c71000e5..6048c277e41 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,29 @@ concurrency: cancel-in-progress: false jobs: + source_verification: + name: Watch exact-source verification + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 44 + permissions: + contents: read + actions: write + outputs: + verified: ${{ steps.watch.outputs.verified }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Stop images when source checks fail and report runner waits + id: watch + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs + reuse_canary_image: if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') runs-on: ubuntu-latest @@ -100,11 +123,10 @@ jobs: - platform: linux/arm64 runner: ubuntu-24.04-arm arch: arm64 - runs-on: ${{ matrix.runner }} - # Native builds land well inside this; it is a backstop, not a budget. - # (The interim fix while this PR landed raised the single QEMU job's cap - # to 120 minutes; native per-arch legs make that headroom unnecessary.) - timeout-minutes: 60 + runs-on: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && format('runs-on/fleet=paperclip-core-image-{0}/env=public-ci', matrix.arch == 'amd64' && 'x64' || 'arm64') || matrix.runner }} + # Stop before the AWS worker's 45-minute maximum lifetime. Hosted tag + # builds retain their existing limit. + timeout-minutes: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && 40 || 60 }} permissions: contents: read packages: write @@ -115,6 +137,9 @@ jobs: # Full history and tags so `git describe` below can compute the # release version to stamp into the image. fetch-depth: 0 + # Keep the version's full commit graph without downloading blobs + # from every historical revision of this large repository. + filter: blob:none # `.git` is dockerignored, so a running image cannot derive its own # version and otherwise reports the source package.json placeholder in @@ -183,6 +208,7 @@ jobs: echo "Using refreshed pnpm-lock.yaml in the Docker build context." - name: Free runner disk + if: ${{ !(vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master') }} run: | set -euo pipefail echo "Disk before cleanup:" @@ -304,16 +330,40 @@ jobs: if-no-files-found: error retention-days: 1 + # Wait for proof on ephemeral verification capacity. Publication runners do + # only registry/signing work. Each Spot observer stays below its 45m lifetime. + publication_verification: + name: Require exact-source verification + if: ${{ !cancelled() && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.build-and-push.result == 'success' && needs.source_verification.result == 'success' && needs.source_verification.outputs.verified != 'true' }} + needs: [build-and-push, source_verification] + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: 44 + permissions: + contents: read + actions: write # restart only exact-source producers deferred before admission + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Require source proof and migrator admission before publication + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs --require-proof + # Names the per-architecture digests as one manifest list under the real # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: - if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} - needs: build-and-push - runs-on: ubuntu-latest + if: ${{ !cancelled() && needs.build-and-push.result == 'success' && (needs.source_verification.result == 'success' || needs.source_verification.result == 'skipped') && (needs.publication_verification.result == 'success' || (needs.publication_verification.result == 'skipped' && (github.repository != 'paperclipai/paperclip' || github.repository_id != '1170821064' || github.ref != 'refs/heads/master' || needs.source_verification.outputs.verified == 'true'))) }} + needs: [build-and-push, source_verification, publication_verification] + runs-on: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32') || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read + actions: read packages: write id-token: write attestations: write @@ -322,8 +372,6 @@ jobs: # script the verification step pipes in. - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - fetch-depth: 0 - name: Download digests uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -375,6 +423,19 @@ jobs: io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }} io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }} + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + + # Revalidate the latest attempt after runner assignment without waiting + # here. A concurrent source retry must invalidate the earlier proof. + - name: Revalidate exact-source proof before publication + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' + env: + GITHUB_TOKEN: ${{ github.token }} + run: node scripts/watch-release-source.mjs --check-proof + - name: Create manifest list and push working-directory: /tmp/digests run: | diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index e36bdbccbd2..a8a29e433b5 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -34,7 +34,7 @@ jobs: typecheck: if: ${{ !inputs.runner_binaries_only }} name: Typecheck - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -120,7 +120,7 @@ jobs: general_tests: if: ${{ !inputs.runner_binaries_only }} name: General tests (${{ matrix.group_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -232,7 +232,7 @@ jobs: serialized_tests: if: ${{ !inputs.runner_binaries_only }} name: Serialized tests (${{ matrix.shard_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -282,7 +282,7 @@ jobs: runner_workflow_evals: if: ${{ !inputs.runner_binaries_only }} name: Runner workflow eval scorer contract - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -313,7 +313,7 @@ jobs: verify_paperclip_runner: if: ${{ !inputs.runner_binaries_only }} name: Verify Paperclip Runner (${{ matrix.lane }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -523,7 +523,7 @@ jobs: build: if: ${{ !inputs.runner_binaries_only }} name: Build - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c7b310f4ee1..d9bdc1824f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -320,12 +320,12 @@ jobs: verify_canary: name: Reuse exact-source verification - if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' runs-on: ubuntu-latest - timeout-minutes: 50 + timeout-minutes: 70 permissions: contents: read - actions: read + actions: write # restart only an exact-source push deferred before admission steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index 810d28ec2e7..fbb60238de3 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -20,8 +20,8 @@ concurrency: jobs: chaos_and_recovery: name: Restart, replay, trace, and recovery faults - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} permissions: contents: read diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index 18611b833b0..df1a595bf56 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -1,30 +1,42 @@ # Cloud build readiness -The `Cloud readiness` workflow starts for every master push and retains the -versioned `Cloud source verified v1` job. It calls the full `Release Verify` -workflow for that exact commit, including typecheck, builds, general and -serialized tests, and Runner verification. The source proof depends on every -source check and fails closed if verification fails, is cancelled, or is skipped. +The `Cloud readiness` workflow starts a small admission check for each master +push. Full verification starts only when the exact source has a running master +Docker or automatic npm release workflow. Superseded pending sources avoid the +full test matrix. The migrator producer similarly waits for image admission. + +A selected consumer restarts a previously deferred push run, preserving its +original source and push identity. This restarts only a successful admission +job followed entirely by skipped jobs. A failed check, cancelled job, incomplete +listing, or identity mismatch cannot trigger this automatic retry. Explicit +manual runs and whole-workflow retries request full verification directly. + +Admitted source runs retain the versioned `Cloud source verified v1` job. They +call the full `Release Verify` workflow for that exact commit, including +typecheck, builds, general and serialized tests, and Runner verification. The +proof depends on every source check and fails closed if verification fails, is +cancelled, or is skipped. Consumers accept only the latest attempt's proof. The recurring public `-cloud` publisher and its `Cloud deployable v1` gate are retired. The workflow no longer builds a legacy image or waits for one. Keep its filename and source-proof job name stable: npm canary publication and downstream image composers consume that exact contract. -Standard images still publish independently through `docker.yml`. The +Standard images build in parallel through `docker.yml`. Master image publication +waits for the exact-source proof; a failed source check cancels its image build +without waiting for the remaining test shards. The `cloud-migrator-artifacts.yml` workflow still publishes signed exact-source migrators independently. A downstream composer must verify those artifacts, build and test its own image, and record separate deployment readiness. -Verification runs outside the full npm release's concurrency group. Different -commits have independent groups. The npm canary release reuses the source proof +Verification runs outside the full npm release's concurrency group. Each source retains its independent verification run. The npm canary release reuses the source proof for its exact master push instead of starting another `Release Verify` run. Stable releases and candidate-branch betas still run full verification. The canary consumer requires the expected workflow ID and path, upstream source repository, master push event, full SHA, and a successful job in the latest run attempt. It checks the run again after reading the jobs to reject a concurrent -rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled, +rerun. Missing proof waits for up to 60 minutes; failed, skipped, cancelled, ambiguous, or mismatched proof cannot authorize publication. API failures fail closed. If a source check fails, fix it and rerun Cloud readiness before retrying the release. Use **Re-run all jobs** when a later attempt did not rerun the source @@ -120,15 +132,16 @@ excluded or sleeping targets, retries, and failures with the fleet result. Recor runner queue conditions and cache state; one warm or cold run is a sample, not a latency guarantee. -## Reserved AWS verification capacity +## Dedicated AWS verification capacity `AWS_POST_MERGE_CI_ENABLED=true` routes cloud source verification and exact-master migrator preparation to the `paperclip-post-merge` runner group. The separate Fleet label is -`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. Its 36 reserved slots use -the same four-vCPU, 16-GiB machines as approved PR jobs. PR capacity is reduced -to 64; the separately provisioned image capacity is unchanged by this retirement. -This keeps PR bursts from consuming every post-merge verification slot. +`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. The protected fleet uses +separate 8-vCPU, 32-GiB Spot workers that launch only when jobs arrive. Its +job ceiling and public PR capacity are managed independently. There are no +hot or stopped pools. Quota ceilings do not reserve hardware; measure cold +launch and runner assignment times as well as job duration. Every selector checks the canonical repository name and ID, master ref, and a push or manual event. Reusable verification also requires `inputs.ref` to equal @@ -142,7 +155,8 @@ its trusted-publisher identity. Before enabling the switch, deploy the separate Fleet and restrict its GitHub runner group to repository ID `1170821064` and these workflows at `refs/heads/master`: `cloud-readiness.yml`, -`release-verify.yml`, `runner-chaos-evals.yml`, and `release.yml`. Do not authorize +`release-verify.yml`, `runner-chaos-evals.yml`, `release.yml`, and `docker.yml`. +The Docker workflow uses this group for its exact-source monitor and proof wait. Do not authorize PR-controlled workflow versions. The direct migrator producer always uses GitHub-hosted runners and needs no AWS runner-group authorization. PR placement retains its independent pinned workflow and six-account author/actor allowlist. @@ -158,6 +172,43 @@ route a public image job after this retirement. This source change does not delete runner groups, Fleets, credentials, registry images, or cache tags. Review shared infrastructure ownership separately before removing those resources. +## Protected standard-image builders + +`CORE_RELEASE_AWS_ENABLED=true` routes canonical master image jobs to native +`paperclip-core-image-x64` and `paperclip-core-image-arm64` fleets. Each has two +32-vCPU / 128-GiB Spot slots, for four large image jobs in total. Workers +launch for jobs and terminate afterward; no idle pool is maintained. The `paperclip-core-release` runner group permits only +Core's `docker.yml@refs/heads/master`. PRs, branches, and tag releases keep their +existing GitHub-hosted placement. Disable the switch and rerun the whole image +workflow to restore hosted placement. + +Image checkout retains the complete commit graph for version stamping but filters +out historical blobs. The manifest job needs only a shallow source checkout. +Prepared AWS image disks skip the standard hosted runner's disk cleanup. Native +architecture builds, digest publication, the orphan-reaping smoke test, and +attestation remain required. The source monitor has Actions write permission to cancel its own canonical +master Docker run and failed exact-source verification siblings. It never +automatically retries a failed source check. Its guarded retry permission can +restart only an exact-source producer deferred before full admission. It checks run, attempt, SHA, +repository, ref, and workflow identity before cancellation. Its bounded +observation window does not cancel healthy pending checks. A successful observer +records proof for publication immediately. If healthy checks remain pending after +40 minutes, a separate required proof job waits for up to another 40 minutes on +the verification fleet. Both jobs fit within the Spot worker lifetime. Neither +wait occupies the four-slot publication pool. The hosted manifest job requires +successful source proof and admitted migrator work before it starts; a queued +migrator job alone is not evidence of admission. It retains its 20-minute budget +for manifest verification and signing. It revalidates the latest source attempt +once before publishing; changed or missing proof fails immediately without +starting another wait on the publication runner. +Image jobs retain +their existing package permissions. + +The monitor emits an Actions warning and step summary when an image or source job waits +more than 60 seconds for a runner. This measures runner assignment, not image +execution. Compare automatic merge-to-ready and merge-to-healthy-target timings; +a manual dispatch alone does not prove deployment latency. + ### Typecheck Rust dependency cache Source verification's typecheck job builds the native Runner binary through the @@ -236,3 +287,10 @@ fi A subsequent master install can populate the missing entry. Check the saved archive size and package reuse in install logs; a cache hit alone does not prove that the entry contains dependencies. + +Image manifest verification and attestation, plus exact-source migrator build +and publication, use the protected four-slot GitHub-hosted `cloud-release-32` +pool. `CORE_PUBLICATION_RUNNER=ubuntu-latest` restores standard hosted placement. +The signing jobs remain GitHub-hosted because downstream provenance explicitly +requires that certificate claim. Their master-only group also serves the private +Cloud publisher and harness deployment; it excludes PRs and branch benchmarks. diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 5167a32193f..3ca0bc25840 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -61,6 +61,39 @@ test("Docker Hub mirrors apply to both image builders without widening publicati "the registry-only manifest merge does not need a Docker Hub mirror"); }); +test("source proof gates publication without occupying a publication runner", () => { + const workflow = readWorkflow("docker.yml"); + const observer = workflow.split(" source_verification:\n")[1].split(" reuse_canary_image:")[0]; + const gate = workflow.split(" publication_verification:\n")[1].split(" merge-and-push:")[0]; + const publisher = workflow.split(" merge-and-push:\n")[1].split(" promote_canary_channel:")[0]; + assert.match(observer, /verified: \$\{\{ steps.watch.outputs.verified \}\}/); + assert.match(gate, /run: node scripts\/watch-release-source.mjs --require-proof/); + assert.match(gate, /actions: write # restart only exact-source producers deferred before admission/); + assert.match(gate, /needs: \[build-and-push, source_verification\]/); + assert.match(publisher, /needs: \[build-and-push, source_verification, publication_verification\]/); + assert.doesNotMatch(publisher, /--require-proof|actions: write/); + assert.ok(publisher.indexOf("run: node scripts/watch-release-source.mjs --check-proof") < publisher.indexOf("- name: Create manifest list and push")); + assert.match(publisher, /actions: read/); + assert.match(publisher, /timeout-minutes: 20/); + const expression = publisher.match(/^ if: \$\{\{ (.+) \}\}$/m)[1].replaceAll("needs.build-and-push", "needs.build"); + for (const [ref, observed, fallback, expected, repository = "paperclipai/paperclip", repository_id = "1170821064"] of [ + ["refs/heads/master", "true", "skipped", true], + ["refs/heads/master", "false", "skipped", false], + ["refs/heads/master", undefined, "skipped", false], + ["refs/heads/master", "false", "success", true], + ["refs/heads/master", "false", "failure", false], + ["refs/tags/v1", undefined, "skipped", true], + ["refs/heads/master", undefined, "skipped", true, "fork/paperclip", "123"], + ["refs/heads/master", undefined, "skipped", true, "paperclipai/paperclip", "123"], + ]) { + const canonical = repository === "paperclipai/paperclip" && repository_id === "1170821064" && ref === "refs/heads/master"; + assert.equal(runInNewContext(expression, { cancelled: () => false, github: { ref, repository, repository_id }, needs: { + build: { result: "success" }, source_verification: { result: canonical ? "success" : "skipped", outputs: { verified: observed } }, + publication_verification: { result: fallback }, + } }), expected); + } +}); + test("chaos verification isolates callers that verify the same source commit", () => { const chaosWorkflow = readWorkflow("runner-chaos-evals.yml"); const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1]; @@ -82,11 +115,11 @@ test("chaos verification isolates callers that verify the same source commit", ( assert.match(chaosWorkflow, /cancel-in-progress: true/); }); -test("canary reuses exact-source proof while stable keeps full verification", () => { +test("canary can admit deferred exact-source checks while stable keeps full verification", () => { const releaseWorkflow = readWorkflow("release.yml"); const canary = releaseWorkflow.match(/ verify_canary:\n[\s\S]*?(?=\n [a-z_]+:)/)[0]; - assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); - assert.match(canary, /actions: read/); + assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.repository_id == '1170821064' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/); + assert.match(canary, /actions: write # restart only an exact-source push deferred before admission/); assert.match(canary, /ref: \$\{\{ github\.sha \}\}/); assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/); assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/); diff --git a/scripts/cloud-release-admission.mjs b/scripts/cloud-release-admission.mjs new file mode 100644 index 00000000000..c7ed134ba4b --- /dev/null +++ b/scripts/cloud-release-admission.mjs @@ -0,0 +1,123 @@ +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; + +const repository = "paperclipai/paperclip"; +const repositoryId = 1170821064; +const workflows = new Set(["docker.yml", "release.yml", "cloud-readiness.yml", "cloud-migrator-artifacts.yml"]); +export const sourceAdmissionJob = "Cloud candidate admission v1"; +export const migratorAdmissionJob = "Cloud image candidate admission v1"; + +function assertSha(sha) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("Admission requires a full source SHA."); +} + +function trusted(run, sha, workflow) { + return run.workflow_id === workflow.id && run.path === workflow.path && run.head_sha === sha && + run.repository?.id === repositoryId && run.head_repository?.id === repositoryId && + run.repository?.full_name === repository && run.head_repository?.full_name === repository && + run.head_branch === "master" && run.event === "push" && + Number.isSafeInteger(run.id) && run.id > 0 && Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0; +} + +async function producer(file, sha, api, status) { + assertSha(sha); + if (!workflows.has(file)) throw new Error("Unknown release workflow."); + const workflow = await api(`/repos/${repository}/actions/workflows/${file}`); + if (workflow.path !== `.github/workflows/${file}` || !Number.isSafeInteger(workflow.id) || workflow.id < 1) throw new Error("Release workflow identity mismatch."); + const list = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?branch=master&event=push&head_sha=${sha}&per_page=100${status ? `&status=${status}` : ""}`); + if (!Array.isArray(list.workflow_runs) || !Number.isSafeInteger(list.total_count) || list.total_count < 0 || + list.total_count > 100 || list.workflow_runs.length !== list.total_count) throw new Error("Incomplete release workflow listing."); + const run = list.workflow_runs.filter(run => trusted(run, sha, workflow) && (!status || run.status === status)).sort((a, b) => b.id - a.id)[0]; + return { workflow, run }; +} + +// A retry is an explicit request for this exact immutable source. Normal push +// work starts only for a candidate admitted by an image build or npm canary. +export async function releaseAdmitted(sha, { api, event = "push", attempt = 1, imageOnly = false }) { + assertSha(sha); + if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release attempt."); + if (!["push", "workflow_dispatch"].includes(event)) return false; + if (event === "workflow_dispatch" || (Number.isSafeInteger(attempt) && attempt > 1)) return true; + for (const file of imageOnly ? ["docker.yml"] : ["docker.yml", "release.yml"]) { + const { workflow, run } = await producer(file, sha, api, "in_progress"); + if (!run) continue; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id === run.id && trusted(current, sha, workflow) && current.run_attempt === run.run_attempt && current.status === "in_progress") return true; + } + return false; +} + +export function deferredProducer(jobs, run, admissionJob) { + const admission = jobs.filter(job => job.name === admissionJob); + const remaining = jobs.filter(job => job.name !== admissionJob); + return run.status === "completed" && run.conclusion === "success" && admission.length === 1 && admission[0].status === "completed" && admission[0].conclusion === "success" && remaining.length > 0 && + remaining.every(job => job.status === "completed" && job.conclusion === "skipped") && + jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha); +} + +// Only skipped admission is restartable here. A failed check or build never +// becomes an automatic retry, and every write revalidates canonical identity. +export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) { + if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract."); + const { workflow, run } = await producer(file, sha, api); + if (!run) return "pending"; + const key = `${run.id}/${run.run_attempt}`; + if (requested.has(key)) return "pending"; + const jobs = []; + let total; + for (let page = 1; page <= 10; page++) { + const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`); + if (!Array.isArray(batch.jobs) || !Number.isSafeInteger(batch.total_count) || batch.total_count < 0) throw new Error("Invalid release producer job list."); + if ((total !== undefined && total !== batch.total_count) || + batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list."); + total = batch.total_count; + jobs.push(...batch.jobs); + if (page * 100 >= total) break; + if (page === 10) throw new Error("Incomplete release producer job list."); + } + if (run.status !== "completed") { + const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha); + // Admission itself succeeds for both outcomes. Only an actual build is + // evidence that its output admitted work; queued jobs may still be skipped. + return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped") + ? "admitted" : "pending"; + } + if (!deferredProducer(jobs, run, admissionJob)) return "finished"; + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed" || current.conclusion !== "success") return "pending"; + try { + await rerun(`/repos/${repository}/actions/runs/${run.id}/rerun`); + } catch (error) { + // Docker and npm may select the same source at the same time. A rejected + // duplicate write is harmless only when a fresh read proves a retry began. + const latest = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (latest.id !== run.id || !trusted(latest, sha, workflow)) throw error; + if (!(latest.run_attempt > run.run_attempt || (latest.run_attempt === run.run_attempt && + ["queued", "in_progress", "requested", "pending", "waiting"].includes(latest.status)))) { + // GitHub may reject a duplicate before its read API shows the retry. + // Leave it pending for the next poll; this is not proof of admission. + if ([409, 422].includes(error.status) && latest.run_attempt === run.run_attempt && latest.status === "completed" && latest.conclusion === "success") return "pending"; + throw error; + } + } + requested.add(key); + return "pending"; +} + +export function createReleaseRerunner(token, fetchImpl = fetch) { + return async path => { + if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/rerun$/.test(path)) throw new Error("Invalid release retry path."); + const response = await fetchImpl(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw Object.assign(new Error(`Could not admit deferred release work: HTTP ${response.status}.`), { status: response.status }); + }; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required."); + const { createActionsReader } = await import("./cloud-source-verification.mjs"); + const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }), + event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" }); + console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source."); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `admitted=${admitted}\n`); +} diff --git a/scripts/cloud-source-verification.mjs b/scripts/cloud-source-verification.mjs index bd40e33cd3c..6a6f316da6f 100644 --- a/scripts/cloud-source-verification.mjs +++ b/scripts/cloud-source-verification.mjs @@ -1,9 +1,18 @@ import { appendFile } from "node:fs/promises"; import { pathToFileURL } from "node:url"; +import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; const repository = "paperclipai/paperclip"; const workflowPath = ".github/workflows/cloud-readiness.yml"; export const sourceVerificationJob = "Cloud source verified v1"; +export class SourceVerificationFailed extends Error { + constructor(message, run) { + super(message); + this.runId = run.id; + this.attempt = run.run_attempt; + } +} +export class SourceVerificationDeferred extends Error {} function assertSha(sha) { if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required."); @@ -19,7 +28,7 @@ function trustedRun(run, sha, workflowId) { // Consume one versioned job, independent of image/migrator availability. A // failed image build must not invalidate source checks that already passed. -export async function readSourceVerification(sha, api) { +export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) { assertSha(sha); const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`); if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) { @@ -46,6 +55,17 @@ export async function readSourceVerification(sha, api) { } const matches = jobs.filter((job) => job.name === sourceVerificationJob); if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous."); + await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt)); + if (failFast) { + const failure = jobs.find((job) => job.status === "completed" && + ["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) && + job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt); + if (failure) { + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined; + throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run); + } + } const job = matches[0]; if (job?.status === "completed" && job.conclusion === "success" && job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) { @@ -56,13 +76,17 @@ export async function readSourceVerification(sha, api) { return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id }; } if (job?.status === "completed" || run.status === "completed") { - throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`); + const current = await api(`/repos/${repository}/actions/runs/${run.id}`); + if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined; + if (job?.conclusion === "skipped" && current.status !== "completed") return undefined; + if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission."); + throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run); } return undefined; } // Statuses that say "ask again", not "the answer is no". A release must not be -// blocked because GitHub returned a gateway error during a 45-minute poll. +// blocked because GitHub returned a gateway error during a 60-minute poll. const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]); // A rate-limited read also says "ask again", and GitHub reports both primary @@ -137,14 +161,20 @@ export function createActionsReader({ export async function waitForSourceVerification(sha, { api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log, + timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun, } = {}) { assertSha(sha); const deadline = now() + timeoutMs; + const requested = new Set(); log(`Waiting for ${sourceVerificationJob} for ${sha}.`); while (now() < deadline) { - const result = await readSourceVerification(sha, api); - if (result) return result; + try { + const result = await readSourceVerification(sha, api); + if (result) return result; + } catch (error) { + if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error; + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } const remaining = deadline - now(); if (remaining > 0) await sleep(Math.min(intervalMs, remaining)); } @@ -156,10 +186,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required."); // One deadline for both layers: the reader stops retrying when the poll it // serves is out of time, instead of extending the wait past its timeout. - const timeoutMs = 45 * 60_000; + const timeoutMs = 60 * 60_000; const deadline = Date.now() + timeoutMs; const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline }); - const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs }); + const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined; + const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun }); const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`; console.log(message); if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`); diff --git a/scripts/cloud-source-verification.test.mjs b/scripts/cloud-source-verification.test.mjs index 4a47abfd164..8221bba4bd3 100644 --- a/scripts/cloud-source-verification.test.mjs +++ b/scripts/cloud-source-verification.test.mjs @@ -1,6 +1,10 @@ import assert from "node:assert/strict"; -import test from "node:test"; -import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs"; +import test, { describe } from "node:test"; +import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; + +import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; + +import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs"; const sha = "a".repeat(40); const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; @@ -62,7 +66,8 @@ test("job identities and terminal failures fail closed", async () => { { head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 }, { conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" }, ]) { - const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] }); + const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }], + current: overrides.conclusion === "skipped" ? { ...baseRun, status: "completed", conclusion: "success" } : baseRun }); await assert.rejects(readSourceVerification(sha, api), /did not pass/); } }); @@ -231,3 +236,357 @@ test("retries stop at the caller's deadline rather than outliving the poll", asy assert.equal(seen.length, 1); assert.deepEqual(waits, []); }); + + +describe("image build source monitor", () => { + const sha = "a".repeat(40); + const runId = 50; + const attempt = 1; + const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" }; + const sourceRun = { id: 99, workflow_id: 123, path: workflow.path, run_attempt: 1, + repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" }, + head_sha: sha, head_branch: "master", event: "push", status: "in_progress" }; + const imageRun = { id: runId, run_attempt: attempt, head_sha: sha, head_branch: "master", event: "push", + repository: { id: 1170821064 }, head_repository: { id: 1170821064 }, + path: ".github/workflows/docker.yml", status: "in_progress" }; + const proof = { id: 111, name: "Cloud source verified v1", run_id: 99, run_attempt: 1, + head_sha: sha, status: "completed", conclusion: "success" }; + + function fixture({ jobs = [proof], run = sourceRun, current = run, image = imageRun, imageJobs = [], missing = false } = {}) { + const cancellations = []; + const messages = []; + const api = async path => { + if (path.endsWith("/workflows/cloud-readiness.yml")) return workflow; + if (path.includes("/workflows/123/runs?")) return { workflow_runs: missing ? [] : [run], total_count: missing ? 0 : 1 }; + if (path.includes("/runs/99/attempts/1/jobs?")) return { jobs }; + if (path.endsWith("/runs/99")) return current; + if (path.includes(`/runs/${runId}/attempts/${attempt}/jobs?`)) return { jobs: imageJobs, total_count: imageJobs.length }; + if (path.endsWith(`/runs/${runId}`)) return image; + throw new Error(`Unexpected request: ${path}`); + }; + return { sha, runId, attempt, api, cancel: async path => cancellations.push(path), + cancellations, messages, log: message => messages.push(message), summary: async message => messages.push(message) }; + } + + test("successful exact-source proof permits publication without canceling image work", async () => { + const f = fixture(); + assert.equal((await watchReleaseSource(f)).jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a required check failing cancels the same image attempt before siblings finish", async () => { + const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }] }); + await assert.rejects(watchReleaseSource(f), /General tests/); + assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]); + }); + + test("a superseded readiness attempt cancels image work with no deployable future", async () => { + const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } }); + await assert.rejects(watchReleaseSource(f), /did not pass/); + assert.equal(f.cancellations.length, 1); + }); + + test("a rerun racing a failed job is retried rather than canceling the image", async () => { + const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }], + current: { ...sourceRun, run_attempt: 2 } }); + let time = 0; + const api = f.api; + f.api = async path => time > 0 ? fixture().api(path) : api(path); + const result = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; } }); + assert.equal(result.jobId, proof.id); + assert.deepEqual(f.cancellations, []); + }); + + test("a transport failure does not issue a destructive cancellation", async () => { + const f = fixture(); + await assert.rejects(watchReleaseSource({ ...f, api: async () => { throw new Error("GitHub unavailable"); } }), /unavailable/); + assert.deepEqual(f.cancellations, []); + }); + + test("the observation window ends without cancelling healthy pending verification", async () => { + const f = fixture({ missing: true }); + let time = 0; + const proof = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; }, timeoutMs: 20 }); + assert.equal(proof, undefined); + assert.equal(time, 20); + assert.deepEqual(f.cancellations, []); + assert.ok(f.messages.some(message => message.includes("final publication still requires exact-source verification"))); + }); + + test("final publication requires proof even after the early observation window ends", async () => { + const f = fixture({ missing: true }); + let time = 0; + await assert.rejects(watchReleaseSource({ ...f, requireProof: true, now: () => time, + sleep: async ms => { time += ms; }, timeoutMs: 20 }), /before the publication deadline/); + assert.deepEqual(f.cancellations, []); + assert.equal((await watchReleaseSource({ ...fixture(), requireProof: true })).jobId, proof.id); + }); + + test("cancellation rejects another SHA, repository, ref, workflow, or attempt", async () => { + for (const change of [{ head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "pull_request" }, + { path: ".github/workflows/release.yml" }, { run_attempt: 2 }, { id: 51 }, + { repository: { id: 42 } }, { head_repository: { id: 42 } }]) { + const f = fixture({ image: { ...imageRun, ...change } }); + await assert.rejects(cancelImageRun(f), /different image workflow identity/); + assert.deepEqual(f.cancellations, []); + } + }); + + test("queue measurement does not count running time or skipped dependency jobs", () => { + const created_at = "2020-01-01T00:00:00Z"; + const started_at = "2020-01-01T00:00:20Z"; + const job = { id: 1, name: "amd64", labels: ["release"], created_at, started_at, runner_id: 10, status: "in_progress" }; + const now = Date.parse(created_at) + 120_000; + assert.deepEqual(runnerWaits([job, { ...job, status: "completed", conclusion: "skipped" }], now), [{ id: 1, job: "amd64", seconds: 20 }]); + assert.equal(runnerWaits([{ ...job, status: "queued", runner_id: 0 }], now)[0].seconds, 120); + }); + + test("a runner wait over 60 seconds produces an Actions warning and summary", async () => { + const f = fixture({ imageJobs: [{ id: 1, name: "amd64", labels: ["release"], status: "queued", runner_id: 0, created_at: "2020-01-01T00:00:00Z" }] }); + await watchReleaseSource({ ...f, now: () => Date.parse("2020-01-01T00:02:00Z") }); + assert.ok(f.messages.some(message => message.startsWith("::warning") && message.includes("120s"))); + assert.ok(f.messages.some(message => message.includes("Release runner wait: amd64"))); + }); +}); + +describe("release candidate admission", () => { + const repository = { id: 1170821064, full_name: "paperclipai/paperclip" }; + const ids = { "docker.yml": 101, "release.yml": 102, "cloud-readiness.yml": 123, "cloud-migrator-artifacts.yml": 124 }; + function run(file, change = {}) { + return { id: ids[file] + 1000, workflow_id: ids[file], path: `.github/workflows/${file}`, + repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push", + run_attempt: 1, status: "in_progress", conclusion: null, ...change }; + } + function job(run, name, conclusion = "skipped") { + return { id: 789, run_id: run.id, run_attempt: run.run_attempt, head_sha: run.head_sha, + name, status: "completed", conclusion }; + } + function fixture(file = "cloud-readiness.yml", change = {}) { + const producer = run(file, { status: "completed", conclusion: "success", ...change }); + const admission = file === "cloud-migrator-artifacts.yml" ? migratorAdmissionJob : sourceAdmissionJob; + const f = { runs: { [file]: [producer] }, jobs: { [producer.id]: [job(producer, admission, "success"), job(producer, sourceVerificationJob)] }, + current: {}, writes: [], calls: [], requested: new Set(), file, sha, admissionJob: admission }; + f.api = async path => { + f.calls.push(path); + const url = new URL(`https://api.github.com${path}`); + const fileMatch = path.match(/\/workflows\/([^/?]+\.yml)$/); + if (fileMatch) return { id: ids[fileMatch[1]], path: `.github/workflows/${fileMatch[1]}` }; + const listMatch = path.match(/\/workflows\/(\d+)\/runs\?/); + if (listMatch) { + const name = Object.keys(ids).find(file => ids[file] === Number(listMatch[1])); + const runs = f.runs[name] ?? []; + return { workflow_runs: runs, total_count: f.total ?? runs.length }; + } + const jobMatch = path.match(/\/runs\/(\d+)\/attempts\/(\d+)\/jobs\?/); + if (jobMatch) { + const page = Number(url.searchParams.get("page") ?? "1"); + const jobs = f.jobs[Number(jobMatch[1])] ?? []; + return { total_count: f.jobTotal ?? jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100) }; + } + const currentMatch = path.match(/\/runs\/(\d+)$/); + if (currentMatch) return f.current[Number(currentMatch[1])] ?? Object.values(f.runs).flat().find(run => run.id === Number(currentMatch[1])); + throw new Error(`Unexpected admission API path: ${path}`); + }; + f.rerun = async path => f.writes.push(path); + return f; + } + + test("only a running image or npm release admits a normal push", async () => { + for (const file of ["docker.yml", "release.yml"]) { + const f = fixture(); + assert.equal(await releaseAdmitted(sha, f), false); + f.runs[file] = [run(file)]; + assert.equal(await releaseAdmitted(sha, f), true); + assert.equal(await releaseAdmitted(sha, { ...f, imageOnly: true }), file === "docker.yml"); + for (const status of ["queued", "pending", "completed"]) { + f.runs[file] = [run(file, { status })]; + assert.equal(await releaseAdmitted(sha, f), false, status); + } + } + }); + + test("manual and retry requests admit exact source without consumer discovery", async () => { + const api = () => assert.fail("explicit request needs no discovery"); + assert.equal(await releaseAdmitted(sha, { api, event: "workflow_dispatch" }), true); + assert.equal(await releaseAdmitted(sha, { api, attempt: 2 }), true); + assert.equal(await releaseAdmitted(sha, { api, event: "pull_request", attempt: 2 }), false); + for (const attempt of [0, "2"]) { + await assert.rejects(releaseAdmitted(sha, { api, attempt }), /Invalid release attempt/); + } + }); + + test("forks, stale identities, tags, and non-push consumers do not admit work", async () => { + for (const change of [{ repository: { ...repository, id: 2 } }, { head_repository: { ...repository, id: 2 } }, + { repository: { ...repository, full_name: "fork/paperclip" } }, { path: ".github/workflows/pr.yml" }, + { workflow_id: 999 }, { head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "workflow_dispatch" }, + { id: 0 }, { run_attempt: 0 }]) { + const f = fixture(); + f.runs["docker.yml"] = [run("docker.yml", change)]; + assert.equal(await releaseAdmitted(sha, f), false, JSON.stringify(change)); + } + const f = fixture(); + const active = run("docker.yml"); + f.runs["docker.yml"] = [active]; + f.current[active.id] = { ...active, run_attempt: 2 }; + assert.equal(await releaseAdmitted(sha, f), false); + }); + + test("incomplete discovery and transport failure fail closed", async () => { + for (const total of [1, 101, -1]) { + const f = fixture(); + f.total = total; + await assert.rejects(releaseAdmitted(sha, f), /Incomplete/); + assert.deepEqual(f.writes, []); + } + await assert.rejects(releaseAdmitted(sha, { api: async () => { throw new Error("API unavailable"); } }), /unavailable/); + await assert.rejects(releaseAdmitted("master", { api: () => assert.fail() }), /full source SHA/); + }); + + test("a selected consumer restarts only a strictly deferred producer, once", async () => { + for (const file of ["cloud-readiness.yml", "cloud-migrator-artifacts.yml"]) { + const f = fixture(file); + assert.equal(await requestDeferredProducer(f), "pending"); + assert.deepEqual(f.writes, [`/repos/paperclipai/paperclip/actions/runs/${ids[file] + 1000}/rerun`]); + await requestDeferredProducer(f); + assert.equal(f.writes.length, 1); + } + }); + + test("failed, incomplete, ambiguous, or mismatched work is never automatically retried", async () => { + for (const change of [jobs => { jobs[1].conclusion = "failure"; }, jobs => { jobs[0].conclusion = "failure"; }, + jobs => { jobs[1].run_attempt = 2; }, jobs => { jobs[1].head_sha = "b".repeat(40); }, + jobs => { jobs[1].run_id = 2; }, jobs => { jobs[1].status = "queued"; }, + jobs => { jobs.push({ ...jobs[0] }); }, jobs => { jobs.splice(1); }]) { + const f = fixture(); + change(f.jobs[1123]); + assert.equal(await requestDeferredProducer(f), "finished"); + assert.deepEqual(f.writes, []); + } + const f = fixture(); + for (const change of [{ run_attempt: 2 }, { status: "in_progress" }, { head_sha: "b".repeat(40) }, { id: 2 }]) { + f.current[1123] = { ...f.runs[f.file][0], ...change }; + assert.equal(await requestDeferredProducer(f), "pending"); + assert.deepEqual(f.writes, []); + } + }); + + test("an explicitly cancelled or failed producer cannot be restarted by admission", async () => { + for (const conclusion of ["cancelled", "failure", "timed_out"]) { + const f = fixture("cloud-readiness.yml", { conclusion }); + assert.equal(await requestDeferredProducer(f), "finished"); + assert.deepEqual(f.writes, []); + } + }); + + test("a truncated job page cannot hide a failed check and trigger a retry", async () => { + const f = fixture(); + f.jobTotal = 3; + await assert.rejects(requestDeferredProducer(f), /Incomplete release producer job list/); + assert.deepEqual(f.writes, []); + f.jobTotal = "2"; + await assert.rejects(requestDeferredProducer(f), /Invalid release producer job list/); + assert.deepEqual(f.writes, []); + }); + + test("producer retries reject arbitrary workflow and admission contracts", async () => { + const f = fixture(); + for (const change of [{ file: "docker.yml" }, { admissionJob: migratorAdmissionJob }]) { + await assert.rejects(requestDeferredProducer({ ...f, ...change }), /Unknown deferred/); + } + assert.deepEqual(f.calls, []); + }); + + test("a deferred completion racing the run listing does not become a source failure", async () => { + const f = fixture(); + const completed = f.runs[f.file][0]; + f.runs[f.file] = [{ ...completed, status: "in_progress", conclusion: null }]; + f.current[completed.id] = completed; + await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred); + delete f.current[completed.id]; + assert.equal(await readSourceVerification(sha, f.api), undefined); + }); + + test("a competing consumer's accepted retry does not fail the losing consumer", async () => { + const f = fixture(); + f.rerun = async () => { + f.current[1123] = run(f.file, { run_attempt: 2 }); + throw new Error("duplicate restart rejected"); + }; + assert.equal(await requestDeferredProducer(f), "pending"); + assert.equal(f.requested.size, 1); + const rejected = fixture(); + rejected.rerun = async () => { throw new Error("token denied"); }; + await assert.rejects(requestDeferredProducer(rejected), /token denied/); + assert.equal(rejected.requested.size, 0); + }); + + test("a queued migrator is pending until its build actually starts", async () => { + const f = fixture("cloud-migrator-artifacts.yml", { status: "in_progress", conclusion: null }); + const producer = f.runs[f.file][0]; + const build = { ...job(producer, "Build migrator", null), status: "queued", labels: ["cloud-release-32"] }; + f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build]; + assert.equal(await requestDeferredProducer(f), "pending"); + build.status = "in_progress"; + assert.equal(await requestDeferredProducer(f), "admitted"); + build.run_attempt = 2; + assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work"); + assert.deepEqual(f.writes, []); + }); + + test("a rejected duplicate with stale read visibility remains pending without claiming admission", async () => { + for (const status of [409, 422, 403]) { + const f = fixture(); + f.rerun = async () => { throw Object.assign(new Error("retry rejected"), { status }); }; + if (status === 403) await assert.rejects(requestDeferredProducer(f), /retry rejected/); + else assert.equal(await requestDeferredProducer(f), "pending"); + assert.equal(f.requested.size, 0); + } + }); + + test("source proof waits for admission, restarts it, and accepts only the new attempt", async () => { + const f = fixture(); + await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred); + let time = 0; + const proof = await waitForSourceVerification(sha, { ...f, now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, + sleep: async ms => { + time += ms; + const selected = run(f.file, { run_attempt: 2 }); + f.runs[f.file] = [selected]; + f.jobs[selected.id] = [job(selected, sourceAdmissionJob, "success"), job(selected, sourceVerificationJob, "success")]; + } }); + assert.equal(proof.attempt, 2); + assert.equal(time, 10); + assert.equal(f.writes.length, 1); + }); + + test("the image monitor admits a late migrator even when source proof is ready", async () => { + const f = fixture("cloud-migrator-artifacts.yml"); + const source = run("cloud-readiness.yml"); + f.runs["cloud-readiness.yml"] = [source]; + f.jobs[source.id] = [job(source, sourceVerificationJob, "success")]; + const image = run("docker.yml"); + f.runs["docker.yml"] = [image]; + f.jobs[image.id] = []; + let time = 0; + const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"), + now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => { + time += ms; + const selected = run(f.file, { run_attempt: 2 }); + f.runs[f.file] = [selected]; + f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }]; + } }); + assert.equal(proof.sha, sha); + assert.equal(time, 10); + assert.equal(f.writes.length, 1); + }); + + test("the retry transport is a bounded POST to a canonical run", async () => { + const calls = []; + const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; }); + await assert.rejects(rerun("/repos/fork/paperclip/actions/runs/1/rerun"), /Invalid release retry/); + await rerun("/repos/paperclipai/paperclip/actions/runs/1/rerun"); + assert.equal(calls.length, 1); + assert.equal(calls[0].options.method, "POST"); + assert.equal(calls[0].options.redirect, "error"); + }); +}); diff --git a/scripts/watch-release-source.mjs b/scripts/watch-release-source.mjs new file mode 100644 index 00000000000..bf4fe032774 --- /dev/null +++ b/scripts/watch-release-source.mjs @@ -0,0 +1,108 @@ +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs"; +import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs"; + +const repository = "paperclipai/paperclip"; + +export function runnerWaits(jobs, now = Date.now()) { + return jobs.filter(job => ["queued", "in_progress", "completed"].includes(job.status) && + job.conclusion !== "skipped" && job.labels?.length > 0).map(job => ({ + job: job.name, id: job.id, + seconds: Math.max(0, Math.floor(((job.runner_id > 0 && job.started_at ? Date.parse(job.started_at) : now) - Date.parse(job.created_at)) / 1000)), + })).filter(job => Number.isFinite(job.seconds)); +} + +// The write is limited to this canonical master Docker run. Never cancel +// another workflow, a tag release, or a run selected from untrusted input. +export async function cancelImageRun({ api, cancel, runId, attempt, sha }) { + const run = await api(`/repos/${repository}/actions/runs/${runId}`); + if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== sha || run.head_branch !== "master" || + run.repository?.id !== 1170821064 || run.head_repository?.id !== 1170821064 || + run.path !== ".github/workflows/docker.yml" || !["push", "workflow_dispatch"].includes(run.event)) { + throw new Error("Refusing to cancel a different image workflow identity."); + } + if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${runId}/cancel`); +} + +async function cancelFailedVerification({ api, cancel, sha }, failure) { + const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`); + // A concurrent retry owns the new attempt; a late observer leaves it alone. + if (run.run_attempt !== failure.attempt) return; + if (run.id !== failure.runId || run.head_sha !== sha || run.head_branch !== "master" || run.event !== "push" || + run.repository?.full_name !== repository || run.head_repository?.full_name !== repository || + run.path !== ".github/workflows/cloud-readiness.yml") throw new Error("Failed verification identity changed."); + if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`); +} + +export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now, + sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log, + timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) { + if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 || + !Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release identity."); + const deadline = now() + timeoutMs; + const warned = new Set(); + const requested = new Set(); + let migratorAdmitted = !rerun; + const reportWaits = async jobs => { + for (const wait of runnerWaits(jobs, now())) { + if (wait.seconds > 60 && !warned.has(wait.id)) { + warned.add(wait.id); + log(`::warning title=Release runner waited over 60 seconds::Job ${wait.id} waited ${wait.seconds}s for a runner.`); + await summary(`Release runner wait: ${wait.job} — ${wait.seconds}s (job ${wait.id}).\n`); + } + } + }; + while (now() < deadline) { + if (!migratorAdmitted) { + migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending"; + } + const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`); + if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing."); + await reportWaits(batch.jobs); + try { + const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits }); + if (proof && migratorAdmitted) { + await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`); + return proof; + } + } catch (error) { + if (error instanceof SourceVerificationDeferred && rerun) { + await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested }); + } else { + if (error instanceof SourceVerificationFailed) { + await cancelFailedVerification({ api, cancel, sha }, error); + await cancelImageRun({ api, cancel, runId, attempt, sha }); + } + throw error; + } + } + await sleep(Math.min(intervalMs, Math.max(0, deadline - now()))); + } + // This early failure observer is not the publication deadline. Healthy + // checks can outlive a Spot worker; a separate exact-source gate still + // waits for proof before publishing any image. + if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline."); + await summary("Source monitoring window ended; final publication still requires exact-source verification.\n"); + return undefined; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const token = process.env.GITHUB_TOKEN; + if (!token || process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Canonical master token required."); + const requireProof = process.argv.includes("--require-proof"); + const checkProof = process.argv.includes("--check-proof"); + const api = createActionsReader({ token }); + const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT), + api, + rerun: createReleaseRerunner(token), + cancel: async path => { + const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error", + headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) }); + if (!response.ok) throw new Error(`Could not cancel failed-source image work: HTTP ${response.status}.`); + }, + summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(), + }); + if (checkProof && !proof) throw new Error("Exact-source proof changed before publication."); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`); +}