perf(ci): admit release candidates and isolate native image capacity (#18869)

Admit full source verification only for exact-source Docker or npm release work, while superseded pending sources complete metadata-only admission. Restart only strictly deferred runs; failed or cancelled checks require an explicit request.

Route canonical master native images to dedicated 32-core Spot profiles and keep source checks on protected verification capacity. Use the four-slot GitHub-hosted publication pool for registry work and signing, with proof waits outside that pool and a fresh source-proof check before tagging. Trim historical checkout blobs and prepared-runner disk cleanup while preserving tag and fork publication behavior.

Validation: release/source and source/artifact regressions, 665 workflow contracts, actionlint, complete latest-head PR CI, and Greptile 5/5 with all comments resolved. Existing CLI and large-file repository timing tests also passed independently; their CI shards received one scoped retry.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip authored and GitHub committed 2026-11-03 18:01:07 -06:00
1 parent 8771606fd8
commit f774992832
13 files changed
+886 -63

No files matched your search

@@ -10,7 +10,8 @@ const base = {
ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1,
};
const expectedJobs = {
"cloud-readiness.yml": [],
"docker.yml": ["source_verification", "publication_verification"],
"cloud-readiness.yml": ["admission", "source_verified"],
"cloud-migrator-artifacts.yml": [],
"release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"],
"runner-chaos-evals.yml": ["chaos_and_recovery"],
@@ -59,9 +60,9 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) {
cases.push({ name: "stable release", inputs: { channel: "stable" } });
}
cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet });
cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } });
cases.push({ name: "later rerun", github: { run_attempt: 3 } });
cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } });
cases.push({ name: "admitted retry", github: { run_attempt: 2 }, expected: release ? "ubuntu-latest" : fleet });
cases.push({ name: "later retry", github: { run_attempt: 3 }, expected: release ? "ubuntu-latest" : fleet });
cases.push({ name: "retry attempt as a string", github: { run_attempt: "2" }, expected: release ? "ubuntu-latest" : fleet });
cases.push({ name: "missing attempt", github: { run_attempt: undefined } });
for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) {
test(`${file}/${job}: ${name}`, () => {
@@ -88,7 +89,7 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) {
}
test("Cloud readiness bookkeeping never waits for the AWS verification fleet", () => {
test("Cloud source proof keeps source-only gating on protected verification capacity", () => {
const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8");
const bodies = new Map();
for (const [name, needs] of [
@@ -97,7 +98,7 @@ test("Cloud readiness bookkeeping never waits for the AWS verification fleet", (
const body = workflow.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=^ [a-z_]+:|(?![\\s\\S]))`, "m"))?.[1];
assert.ok(body, `missing ${name} job`);
bodies.set(name, body);
assert.match(body, /^ runs-on: ubuntu-latest$/m);
assert.match(body, /runs-on:.*paperclip-post-merge-x64/);
assert.doesNotMatch(body, /^ +continue-on-error:|^ +if:.*always\(\)/m);
assert.match(body, /^ if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'$/m);
assert.match(body, /^ +SOURCE_SHA: \$\{\{ github.sha \}\}$/m);
+27 -3
View File
@@ -12,9 +12,33 @@ concurrency:
cancel-in-progress: false
jobs:
build:
admission:
name: Cloud image candidate admission v1
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }}
timeout-minutes: 5
permissions:
contents: read
actions: read
outputs:
admitted: ${{ steps.admission.outputs.admitted }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
sparse-checkout: scripts
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
- id: admission
env:
GITHUB_TOKEN: ${{ github.token }}
run: node scripts/cloud-release-admission.mjs image-only
build:
needs: admission
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true'
runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }}
timeout-minutes: 10
permissions:
contents: read
@@ -52,7 +76,7 @@ jobs:
publish:
needs: build
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }}
timeout-minutes: 5
permissions:
contents: read
+29 -4
View File
@@ -8,14 +8,38 @@ on:
permissions: {}
# Source verification must start outside the full npm release's queue.
# Source verification stays independent of npm publication and image builds.
concurrency:
group: cloud-readiness-${{ github.sha }}
cancel-in-progress: false
jobs:
admission:
name: Cloud candidate admission v1
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 5
permissions:
contents: read
actions: read
outputs:
admitted: ${{ steps.admission.outputs.admitted }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
sparse-checkout: scripts
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
- id: admission
env:
GITHUB_TOKEN: ${{ github.token }}
run: node scripts/cloud-release-admission.mjs
verify:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
needs: admission
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true'
permissions:
contents: read
uses: ./.github/workflows/release-verify.yml
@@ -29,8 +53,9 @@ jobs:
name: Cloud source verified v1
needs: [verify]
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
# Bookkeeping must not wait for the AWS builders it observes.
runs-on: ubuntu-latest
# Keep the final proof off the shared hosted queue too. The protected
# fleet has a separate job ceiling and launches Spot workers on demand.
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 5
permissions:
contents: read
+71 -10
View File
@@ -33,6 +33,29 @@ concurrency:
cancel-in-progress: false
jobs:
source_verification:
name: Watch exact-source verification
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 44
permissions:
contents: read
actions: write
outputs:
verified: ${{ steps.watch.outputs.verified }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
- name: Stop images when source checks fail and report runner waits
id: watch
env:
GITHUB_TOKEN: ${{ github.token }}
run: node scripts/watch-release-source.mjs
reuse_canary_image:
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v')
runs-on: ubuntu-latest
@@ -100,11 +123,10 @@ jobs:
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ${{ matrix.runner }}
# Native builds land well inside this; it is a backstop, not a budget.
# (The interim fix while this PR landed raised the single QEMU job's cap
# to 120 minutes; native per-arch legs make that headroom unnecessary.)
timeout-minutes: 60
runs-on: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && format('runs-on/fleet=paperclip-core-image-{0}/env=public-ci', matrix.arch == 'amd64' && 'x64' || 'arm64') || matrix.runner }}
# Stop before the AWS worker's 45-minute maximum lifetime. Hosted tag
# builds retain their existing limit.
timeout-minutes: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && 40 || 60 }}
permissions:
contents: read
packages: write
@@ -115,6 +137,9 @@ jobs:
# Full history and tags so `git describe` below can compute the
# release version to stamp into the image.
fetch-depth: 0
# Keep the version's full commit graph without downloading blobs
# from every historical revision of this large repository.
filter: blob:none
# `.git` is dockerignored, so a running image cannot derive its own
# version and otherwise reports the source package.json placeholder in
@@ -183,6 +208,7 @@ jobs:
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
- name: Free runner disk
if: ${{ !(vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master') }}
run: |
set -euo pipefail
echo "Disk before cleanup:"
@@ -304,16 +330,40 @@ jobs:
if-no-files-found: error
retention-days: 1
# Wait for proof on ephemeral verification capacity. Publication runners do
# only registry/signing work. Each Spot observer stays below its 45m lifetime.
publication_verification:
name: Require exact-source verification
if: ${{ !cancelled() && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.build-and-push.result == 'success' && needs.source_verification.result == 'success' && needs.source_verification.outputs.verified != 'true' }}
needs: [build-and-push, source_verification]
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 44
permissions:
contents: read
actions: write # restart only exact-source producers deferred before admission
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
- name: Require source proof and migrator admission before publication
env:
GITHUB_TOKEN: ${{ github.token }}
run: node scripts/watch-release-source.mjs --require-proof
# Names the per-architecture digests as one manifest list under the real
# tags. Nothing is publicly tagged until this runs, so a half-published
# multi-arch image is not a state anything can pull.
merge-and-push:
if: ${{ !cancelled() && needs.build-and-push.result == 'success' }}
needs: build-and-push
runs-on: ubuntu-latest
if: ${{ !cancelled() && needs.build-and-push.result == 'success' && (needs.source_verification.result == 'success' || needs.source_verification.result == 'skipped') && (needs.publication_verification.result == 'success' || (needs.publication_verification.result == 'skipped' && (github.repository != 'paperclipai/paperclip' || github.repository_id != '1170821064' || github.ref != 'refs/heads/master' || needs.source_verification.outputs.verified == 'true'))) }}
needs: [build-and-push, source_verification, publication_verification]
runs-on: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32') || 'ubuntu-latest' }}
timeout-minutes: 20
permissions:
contents: read
actions: read
packages: write
id-token: write
attestations: write
@@ -322,8 +372,6 @@ jobs:
# script the verification step pipes in.
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- name: Download digests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
@@ -375,6 +423,19 @@ jobs:
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
with:
node-version: 24
# Revalidate the latest attempt after runner assignment without waiting
# here. A concurrent source retry must invalidate the earlier proof.
- name: Revalidate exact-source proof before publication
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
env:
GITHUB_TOKEN: ${{ github.token }}
run: node scripts/watch-release-source.mjs --check-proof
- name: Create manifest list and push
working-directory: /tmp/digests
run: |
+6 -6
View File
@@ -34,7 +34,7 @@ jobs:
typecheck:
if: ${{ !inputs.runner_binaries_only }}
name: Typecheck
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 20
permissions:
contents: read
@@ -120,7 +120,7 @@ jobs:
general_tests:
if: ${{ !inputs.runner_binaries_only }}
name: General tests (${{ matrix.group_label }})
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 20
permissions:
contents: read
@@ -232,7 +232,7 @@ jobs:
serialized_tests:
if: ${{ !inputs.runner_binaries_only }}
name: Serialized tests (${{ matrix.shard_label }})
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 20
permissions:
contents: read
@@ -282,7 +282,7 @@ jobs:
runner_workflow_evals:
if: ${{ !inputs.runner_binaries_only }}
name: Runner workflow eval scorer contract
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 10
permissions:
contents: read
@@ -313,7 +313,7 @@ jobs:
verify_paperclip_runner:
if: ${{ !inputs.runner_binaries_only }}
name: Verify Paperclip Runner (${{ matrix.lane }})
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 20
permissions:
contents: read
@@ -523,7 +523,7 @@ jobs:
build:
if: ${{ !inputs.runner_binaries_only }}
name: Build
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: 20
permissions:
contents: read
+3 -3
View File
@@ -320,12 +320,12 @@ jobs:
verify_canary:
name: Reuse exact-source verification
if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master'
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 50
timeout-minutes: 70
permissions:
contents: read
actions: read
actions: write # restart only an exact-source push deferred before admission
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
+2 -2
View File
@@ -20,8 +20,8 @@ concurrency:
jobs:
chaos_and_recovery:
name: Restart, replay, trace, and recovery faults
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }}
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }}
permissions:
contents: read
+73 -15
View File
@@ -1,30 +1,42 @@
# Cloud build readiness
The `Cloud readiness` workflow starts for every master push and retains the
versioned `Cloud source verified v1` job. It calls the full `Release Verify`
workflow for that exact commit, including typecheck, builds, general and
serialized tests, and Runner verification. The source proof depends on every
source check and fails closed if verification fails, is cancelled, or is skipped.
The `Cloud readiness` workflow starts a small admission check for each master
push. Full verification starts only when the exact source has a running master
Docker or automatic npm release workflow. Superseded pending sources avoid the
full test matrix. The migrator producer similarly waits for image admission.
A selected consumer restarts a previously deferred push run, preserving its
original source and push identity. This restarts only a successful admission
job followed entirely by skipped jobs. A failed check, cancelled job, incomplete
listing, or identity mismatch cannot trigger this automatic retry. Explicit
manual runs and whole-workflow retries request full verification directly.
Admitted source runs retain the versioned `Cloud source verified v1` job. They
call the full `Release Verify` workflow for that exact commit, including
typecheck, builds, general and serialized tests, and Runner verification. The
proof depends on every source check and fails closed if verification fails, is
cancelled, or is skipped. Consumers accept only the latest attempt's proof.
The recurring public `-cloud` publisher and its `Cloud deployable v1` gate are
retired. The workflow no longer builds a legacy image or waits for one.
Keep its filename and source-proof job name stable: npm canary publication and
downstream image composers consume that exact contract.
Standard images still publish independently through `docker.yml`. The
Standard images build in parallel through `docker.yml`. Master image publication
waits for the exact-source proof; a failed source check cancels its image build
without waiting for the remaining test shards. The
`cloud-migrator-artifacts.yml` workflow still publishes signed exact-source
migrators independently. A downstream composer must verify those artifacts,
build and test its own image, and record separate deployment readiness.
Verification runs outside the full npm release's concurrency group. Different
commits have independent groups. The npm canary release reuses the source proof
Verification runs outside the full npm release's concurrency group. Each source retains its independent verification run. The npm canary release reuses the source proof
for its exact master push instead of starting another `Release Verify` run.
Stable releases and candidate-branch betas still run full verification.
The canary consumer requires the expected workflow ID and path, upstream source
repository, master push event, full SHA, and a successful job in the latest run
attempt. It checks the run again after reading the jobs to reject a concurrent
rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled,
rerun. Missing proof waits for up to 60 minutes; failed, skipped, cancelled,
ambiguous, or mismatched proof cannot authorize publication. API failures fail
closed. If a source check fails, fix it and rerun Cloud readiness before retrying
the release. Use **Re-run all jobs** when a later attempt did not rerun the source
@@ -120,15 +132,16 @@ excluded or sleeping targets, retries, and failures with the fleet result. Recor
runner queue conditions and cache state; one warm or cold run is a sample, not a
latency guarantee.
## Reserved AWS verification capacity
## Dedicated AWS verification capacity
`AWS_POST_MERGE_CI_ENABLED=true` routes cloud source verification and
exact-master migrator preparation to the
`paperclip-post-merge` runner group. The separate Fleet label is
`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. Its 36 reserved slots use
the same four-vCPU, 16-GiB machines as approved PR jobs. PR capacity is reduced
to 64; the separately provisioned image capacity is unchanged by this retirement.
This keeps PR bursts from consuming every post-merge verification slot.
`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. The protected fleet uses
separate 8-vCPU, 32-GiB Spot workers that launch only when jobs arrive. Its
job ceiling and public PR capacity are managed independently. There are no
hot or stopped pools. Quota ceilings do not reserve hardware; measure cold
launch and runner assignment times as well as job duration.
Every selector checks the canonical repository name and ID, master ref, and a
push or manual event. Reusable verification also requires `inputs.ref` to equal
@@ -142,7 +155,8 @@ its trusted-publisher identity.
Before enabling the switch, deploy the separate Fleet and restrict its GitHub
runner group to repository ID `1170821064` and these workflows at
`refs/heads/master`: `cloud-readiness.yml`,
`release-verify.yml`, `runner-chaos-evals.yml`, and `release.yml`. Do not authorize
`release-verify.yml`, `runner-chaos-evals.yml`, `release.yml`, and `docker.yml`.
The Docker workflow uses this group for its exact-source monitor and proof wait. Do not authorize
PR-controlled workflow versions. The direct migrator producer always uses
GitHub-hosted runners and needs no AWS runner-group authorization. PR placement retains its independent pinned
workflow and six-account author/actor allowlist.
@@ -158,6 +172,43 @@ route a public image job after this retirement. This source change does not
delete runner groups, Fleets, credentials, registry images, or cache tags. Review
shared infrastructure ownership separately before removing those resources.
## Protected standard-image builders
`CORE_RELEASE_AWS_ENABLED=true` routes canonical master image jobs to native
`paperclip-core-image-x64` and `paperclip-core-image-arm64` fleets. Each has two
32-vCPU / 128-GiB Spot slots, for four large image jobs in total. Workers
launch for jobs and terminate afterward; no idle pool is maintained. The `paperclip-core-release` runner group permits only
Core's `docker.yml@refs/heads/master`. PRs, branches, and tag releases keep their
existing GitHub-hosted placement. Disable the switch and rerun the whole image
workflow to restore hosted placement.
Image checkout retains the complete commit graph for version stamping but filters
out historical blobs. The manifest job needs only a shallow source checkout.
Prepared AWS image disks skip the standard hosted runner's disk cleanup. Native
architecture builds, digest publication, the orphan-reaping smoke test, and
attestation remain required. The source monitor has Actions write permission to cancel its own canonical
master Docker run and failed exact-source verification siblings. It never
automatically retries a failed source check. Its guarded retry permission can
restart only an exact-source producer deferred before full admission. It checks run, attempt, SHA,
repository, ref, and workflow identity before cancellation. Its bounded
observation window does not cancel healthy pending checks. A successful observer
records proof for publication immediately. If healthy checks remain pending after
40 minutes, a separate required proof job waits for up to another 40 minutes on
the verification fleet. Both jobs fit within the Spot worker lifetime. Neither
wait occupies the four-slot publication pool. The hosted manifest job requires
successful source proof and admitted migrator work before it starts; a queued
migrator job alone is not evidence of admission. It retains its 20-minute budget
for manifest verification and signing. It revalidates the latest source attempt
once before publishing; changed or missing proof fails immediately without
starting another wait on the publication runner.
Image jobs retain
their existing package permissions.
The monitor emits an Actions warning and step summary when an image or source job waits
more than 60 seconds for a runner. This measures runner assignment, not image
execution. Compare automatic merge-to-ready and merge-to-healthy-target timings;
a manual dispatch alone does not prove deployment latency.
### Typecheck Rust dependency cache
Source verification's typecheck job builds the native Runner binary through the
@@ -236,3 +287,10 @@ fi
A subsequent master install can populate the missing entry. Check the saved
archive size and package reuse in install logs; a cache hit alone does not prove
that the entry contains dependencies.
Image manifest verification and attestation, plus exact-source migrator build
and publication, use the protected four-slot GitHub-hosted `cloud-release-32`
pool. `CORE_PUBLICATION_RUNNER=ubuntu-latest` restores standard hosted placement.
The signing jobs remain GitHub-hosted because downstream provenance explicitly
requires that certificate claim. Their master-only group also serves the private
Cloud publisher and harness deployment; it excludes PRs and branch benchmarks.
@@ -61,6 +61,39 @@ test("Docker Hub mirrors apply to both image builders without widening publicati
"the registry-only manifest merge does not need a Docker Hub mirror");
});
test("source proof gates publication without occupying a publication runner", () => {
const workflow = readWorkflow("docker.yml");
const observer = workflow.split(" source_verification:\n")[1].split(" reuse_canary_image:")[0];
const gate = workflow.split(" publication_verification:\n")[1].split(" merge-and-push:")[0];
const publisher = workflow.split(" merge-and-push:\n")[1].split(" promote_canary_channel:")[0];
assert.match(observer, /verified: \$\{\{ steps.watch.outputs.verified \}\}/);
assert.match(gate, /run: node scripts\/watch-release-source.mjs --require-proof/);
assert.match(gate, /actions: write # restart only exact-source producers deferred before admission/);
assert.match(gate, /needs: \[build-and-push, source_verification\]/);
assert.match(publisher, /needs: \[build-and-push, source_verification, publication_verification\]/);
assert.doesNotMatch(publisher, /--require-proof|actions: write/);
assert.ok(publisher.indexOf("run: node scripts/watch-release-source.mjs --check-proof") < publisher.indexOf("- name: Create manifest list and push"));
assert.match(publisher, /actions: read/);
assert.match(publisher, /timeout-minutes: 20/);
const expression = publisher.match(/^ if: \$\{\{ (.+) \}\}$/m)[1].replaceAll("needs.build-and-push", "needs.build");
for (const [ref, observed, fallback, expected, repository = "paperclipai/paperclip", repository_id = "1170821064"] of [
["refs/heads/master", "true", "skipped", true],
["refs/heads/master", "false", "skipped", false],
["refs/heads/master", undefined, "skipped", false],
["refs/heads/master", "false", "success", true],
["refs/heads/master", "false", "failure", false],
["refs/tags/v1", undefined, "skipped", true],
["refs/heads/master", undefined, "skipped", true, "fork/paperclip", "123"],
["refs/heads/master", undefined, "skipped", true, "paperclipai/paperclip", "123"],
]) {
const canonical = repository === "paperclipai/paperclip" && repository_id === "1170821064" && ref === "refs/heads/master";
assert.equal(runInNewContext(expression, { cancelled: () => false, github: { ref, repository, repository_id }, needs: {
build: { result: "success" }, source_verification: { result: canonical ? "success" : "skipped", outputs: { verified: observed } },
publication_verification: { result: fallback },
} }), expected);
}
});
test("chaos verification isolates callers that verify the same source commit", () => {
const chaosWorkflow = readWorkflow("runner-chaos-evals.yml");
const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1];
@@ -82,11 +115,11 @@ test("chaos verification isolates callers that verify the same source commit", (
assert.match(chaosWorkflow, /cancel-in-progress: true/);
});
test("canary reuses exact-source proof while stable keeps full verification", () => {
test("canary can admit deferred exact-source checks while stable keeps full verification", () => {
const releaseWorkflow = readWorkflow("release.yml");
const canary = releaseWorkflow.match(/ verify_canary:\n[\s\S]*?(?=\n [a-z_]+:)/)[0];
assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
assert.match(canary, /actions: read/);
assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.repository_id == '1170821064' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
assert.match(canary, /actions: write # restart only an exact-source push deferred before admission/);
assert.match(canary, /ref: \$\{\{ github\.sha \}\}/);
assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/);
+123
View File
@@ -0,0 +1,123 @@
import { appendFile } from "node:fs/promises";
import { pathToFileURL } from "node:url";
const repository = "paperclipai/paperclip";
const repositoryId = 1170821064;
const workflows = new Set(["docker.yml", "release.yml", "cloud-readiness.yml", "cloud-migrator-artifacts.yml"]);
export const sourceAdmissionJob = "Cloud candidate admission v1";
export const migratorAdmissionJob = "Cloud image candidate admission v1";
function assertSha(sha) {
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("Admission requires a full source SHA.");
}
function trusted(run, sha, workflow) {
return run.workflow_id === workflow.id && run.path === workflow.path && run.head_sha === sha &&
run.repository?.id === repositoryId && run.head_repository?.id === repositoryId &&
run.repository?.full_name === repository && run.head_repository?.full_name === repository &&
run.head_branch === "master" && run.event === "push" &&
Number.isSafeInteger(run.id) && run.id > 0 && Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0;
}
async function producer(file, sha, api, status) {
assertSha(sha);
if (!workflows.has(file)) throw new Error("Unknown release workflow.");
const workflow = await api(`/repos/${repository}/actions/workflows/${file}`);
if (workflow.path !== `.github/workflows/${file}` || !Number.isSafeInteger(workflow.id) || workflow.id < 1) throw new Error("Release workflow identity mismatch.");
const list = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?branch=master&event=push&head_sha=${sha}&per_page=100${status ? `&status=${status}` : ""}`);
if (!Array.isArray(list.workflow_runs) || !Number.isSafeInteger(list.total_count) || list.total_count < 0 ||
list.total_count > 100 || list.workflow_runs.length !== list.total_count) throw new Error("Incomplete release workflow listing.");
const run = list.workflow_runs.filter(run => trusted(run, sha, workflow) && (!status || run.status === status)).sort((a, b) => b.id - a.id)[0];
return { workflow, run };
}
// A retry is an explicit request for this exact immutable source. Normal push
// work starts only for a candidate admitted by an image build or npm canary.
export async function releaseAdmitted(sha, { api, event = "push", attempt = 1, imageOnly = false }) {
assertSha(sha);
if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release attempt.");
if (!["push", "workflow_dispatch"].includes(event)) return false;
if (event === "workflow_dispatch" || (Number.isSafeInteger(attempt) && attempt > 1)) return true;
for (const file of imageOnly ? ["docker.yml"] : ["docker.yml", "release.yml"]) {
const { workflow, run } = await producer(file, sha, api, "in_progress");
if (!run) continue;
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
if (current.id === run.id && trusted(current, sha, workflow) && current.run_attempt === run.run_attempt && current.status === "in_progress") return true;
}
return false;
}
export function deferredProducer(jobs, run, admissionJob) {
const admission = jobs.filter(job => job.name === admissionJob);
const remaining = jobs.filter(job => job.name !== admissionJob);
return run.status === "completed" && run.conclusion === "success" && admission.length === 1 && admission[0].status === "completed" && admission[0].conclusion === "success" && remaining.length > 0 &&
remaining.every(job => job.status === "completed" && job.conclusion === "skipped") &&
jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha);
}
// Only skipped admission is restartable here. A failed check or build never
// becomes an automatic retry, and every write revalidates canonical identity.
export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) {
if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract.");
const { workflow, run } = await producer(file, sha, api);
if (!run) return "pending";
const key = `${run.id}/${run.run_attempt}`;
if (requested.has(key)) return "pending";
const jobs = [];
let total;
for (let page = 1; page <= 10; page++) {
const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`);
if (!Array.isArray(batch.jobs) || !Number.isSafeInteger(batch.total_count) || batch.total_count < 0) throw new Error("Invalid release producer job list.");
if ((total !== undefined && total !== batch.total_count) ||
batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list.");
total = batch.total_count;
jobs.push(...batch.jobs);
if (page * 100 >= total) break;
if (page === 10) throw new Error("Incomplete release producer job list.");
}
if (run.status !== "completed") {
const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha);
// Admission itself succeeds for both outcomes. Only an actual build is
// evidence that its output admitted work; queued jobs may still be skipped.
return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped")
? "admitted" : "pending";
}
if (!deferredProducer(jobs, run, admissionJob)) return "finished";
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed" || current.conclusion !== "success") return "pending";
try {
await rerun(`/repos/${repository}/actions/runs/${run.id}/rerun`);
} catch (error) {
// Docker and npm may select the same source at the same time. A rejected
// duplicate write is harmless only when a fresh read proves a retry began.
const latest = await api(`/repos/${repository}/actions/runs/${run.id}`);
if (latest.id !== run.id || !trusted(latest, sha, workflow)) throw error;
if (!(latest.run_attempt > run.run_attempt || (latest.run_attempt === run.run_attempt &&
["queued", "in_progress", "requested", "pending", "waiting"].includes(latest.status)))) {
// GitHub may reject a duplicate before its read API shows the retry.
// Leave it pending for the next poll; this is not proof of admission.
if ([409, 422].includes(error.status) && latest.run_attempt === run.run_attempt && latest.status === "completed" && latest.conclusion === "success") return "pending";
throw error;
}
}
requested.add(key);
return "pending";
}
export function createReleaseRerunner(token, fetchImpl = fetch) {
return async path => {
if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/rerun$/.test(path)) throw new Error("Invalid release retry path.");
const response = await fetchImpl(`https://api.github.com${path}`, { method: "POST", redirect: "error",
headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });
if (!response.ok) throw Object.assign(new Error(`Could not admit deferred release work: HTTP ${response.status}.`), { status: response.status });
};
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required.");
const { createActionsReader } = await import("./cloud-source-verification.mjs");
const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }),
event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" });
console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source.");
if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `admitted=${admitted}\n`);
}
+39 -8
View File
@@ -1,9 +1,18 @@
import { appendFile } from "node:fs/promises";
import { pathToFileURL } from "node:url";
import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
const repository = "paperclipai/paperclip";
const workflowPath = ".github/workflows/cloud-readiness.yml";
export const sourceVerificationJob = "Cloud source verified v1";
export class SourceVerificationFailed extends Error {
constructor(message, run) {
super(message);
this.runId = run.id;
this.attempt = run.run_attempt;
}
}
export class SourceVerificationDeferred extends Error {}
function assertSha(sha) {
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required.");
@@ -19,7 +28,7 @@ function trustedRun(run, sha, workflowId) {
// Consume one versioned job, independent of image/migrator availability. A
// failed image build must not invalidate source checks that already passed.
export async function readSourceVerification(sha, api) {
export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) {
assertSha(sha);
const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`);
if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) {
@@ -46,6 +55,17 @@ export async function readSourceVerification(sha, api) {
}
const matches = jobs.filter((job) => job.name === sourceVerificationJob);
if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous.");
await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt));
if (failFast) {
const failure = jobs.find((job) => job.status === "completed" &&
["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) &&
job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt);
if (failure) {
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run);
}
}
const job = matches[0];
if (job?.status === "completed" && job.conclusion === "success" &&
job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) {
@@ -56,13 +76,17 @@ export async function readSourceVerification(sha, api) {
return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id };
}
if (job?.status === "completed" || run.status === "completed") {
throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`);
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
if (job?.conclusion === "skipped" && current.status !== "completed") return undefined;
if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission.");
throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run);
}
return undefined;
}
// Statuses that say "ask again", not "the answer is no". A release must not be
// blocked because GitHub returned a gateway error during a 45-minute poll.
// blocked because GitHub returned a gateway error during a 60-minute poll.
const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
// A rate-limited read also says "ask again", and GitHub reports both primary
@@ -137,14 +161,20 @@ export function createActionsReader({
export async function waitForSourceVerification(sha, {
api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun,
} = {}) {
assertSha(sha);
const deadline = now() + timeoutMs;
const requested = new Set();
log(`Waiting for ${sourceVerificationJob} for ${sha}.`);
while (now() < deadline) {
const result = await readSourceVerification(sha, api);
if (result) return result;
try {
const result = await readSourceVerification(sha, api);
if (result) return result;
} catch (error) {
if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error;
await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
}
const remaining = deadline - now();
if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
}
@@ -156,10 +186,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
// One deadline for both layers: the reader stops retrying when the poll it
// serves is out of time, instead of extending the wait past its timeout.
const timeoutMs = 45 * 60_000;
const timeoutMs = 60 * 60_000;
const deadline = Date.now() + timeoutMs;
const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline });
const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs });
const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined;
const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun });
const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
console.log(message);
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);
+362 -3
View File
@@ -1,6 +1,10 @@
import assert from "node:assert/strict";
import test from "node:test";
import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
import test, { describe } from "node:test";
import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs";
import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs";
const sha = "a".repeat(40);
const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
@@ -62,7 +66,8 @@ test("job identities and terminal failures fail closed", async () => {
{ head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 },
{ conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" },
]) {
const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] });
const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }],
current: overrides.conclusion === "skipped" ? { ...baseRun, status: "completed", conclusion: "success" } : baseRun });
await assert.rejects(readSourceVerification(sha, api), /did not pass/);
}
});
@@ -231,3 +236,357 @@ test("retries stop at the caller's deadline rather than outliving the poll", asy
assert.equal(seen.length, 1);
assert.deepEqual(waits, []);
});
describe("image build source monitor", () => {
const sha = "a".repeat(40);
const runId = 50;
const attempt = 1;
const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
const sourceRun = { id: 99, workflow_id: 123, path: workflow.path, run_attempt: 1,
repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" },
head_sha: sha, head_branch: "master", event: "push", status: "in_progress" };
const imageRun = { id: runId, run_attempt: attempt, head_sha: sha, head_branch: "master", event: "push",
repository: { id: 1170821064 }, head_repository: { id: 1170821064 },
path: ".github/workflows/docker.yml", status: "in_progress" };
const proof = { id: 111, name: "Cloud source verified v1", run_id: 99, run_attempt: 1,
head_sha: sha, status: "completed", conclusion: "success" };
function fixture({ jobs = [proof], run = sourceRun, current = run, image = imageRun, imageJobs = [], missing = false } = {}) {
const cancellations = [];
const messages = [];
const api = async path => {
if (path.endsWith("/workflows/cloud-readiness.yml")) return workflow;
if (path.includes("/workflows/123/runs?")) return { workflow_runs: missing ? [] : [run], total_count: missing ? 0 : 1 };
if (path.includes("/runs/99/attempts/1/jobs?")) return { jobs };
if (path.endsWith("/runs/99")) return current;
if (path.includes(`/runs/${runId}/attempts/${attempt}/jobs?`)) return { jobs: imageJobs, total_count: imageJobs.length };
if (path.endsWith(`/runs/${runId}`)) return image;
throw new Error(`Unexpected request: ${path}`);
};
return { sha, runId, attempt, api, cancel: async path => cancellations.push(path),
cancellations, messages, log: message => messages.push(message), summary: async message => messages.push(message) };
}
test("successful exact-source proof permits publication without canceling image work", async () => {
const f = fixture();
assert.equal((await watchReleaseSource(f)).jobId, proof.id);
assert.deepEqual(f.cancellations, []);
});
test("a required check failing cancels the same image attempt before siblings finish", async () => {
const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }] });
await assert.rejects(watchReleaseSource(f), /General tests/);
assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]);
});
test("a superseded readiness attempt cancels image work with no deployable future", async () => {
const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } });
await assert.rejects(watchReleaseSource(f), /did not pass/);
assert.equal(f.cancellations.length, 1);
});
test("a rerun racing a failed job is retried rather than canceling the image", async () => {
const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }],
current: { ...sourceRun, run_attempt: 2 } });
let time = 0;
const api = f.api;
f.api = async path => time > 0 ? fixture().api(path) : api(path);
const result = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; } });
assert.equal(result.jobId, proof.id);
assert.deepEqual(f.cancellations, []);
});
test("a transport failure does not issue a destructive cancellation", async () => {
const f = fixture();
await assert.rejects(watchReleaseSource({ ...f, api: async () => { throw new Error("GitHub unavailable"); } }), /unavailable/);
assert.deepEqual(f.cancellations, []);
});
test("the observation window ends without cancelling healthy pending verification", async () => {
const f = fixture({ missing: true });
let time = 0;
const proof = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; }, timeoutMs: 20 });
assert.equal(proof, undefined);
assert.equal(time, 20);
assert.deepEqual(f.cancellations, []);
assert.ok(f.messages.some(message => message.includes("final publication still requires exact-source verification")));
});
test("final publication requires proof even after the early observation window ends", async () => {
const f = fixture({ missing: true });
let time = 0;
await assert.rejects(watchReleaseSource({ ...f, requireProof: true, now: () => time,
sleep: async ms => { time += ms; }, timeoutMs: 20 }), /before the publication deadline/);
assert.deepEqual(f.cancellations, []);
assert.equal((await watchReleaseSource({ ...fixture(), requireProof: true })).jobId, proof.id);
});
test("cancellation rejects another SHA, repository, ref, workflow, or attempt", async () => {
for (const change of [{ head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "pull_request" },
{ path: ".github/workflows/release.yml" }, { run_attempt: 2 }, { id: 51 },
{ repository: { id: 42 } }, { head_repository: { id: 42 } }]) {
const f = fixture({ image: { ...imageRun, ...change } });
await assert.rejects(cancelImageRun(f), /different image workflow identity/);
assert.deepEqual(f.cancellations, []);
}
});
test("queue measurement does not count running time or skipped dependency jobs", () => {
const created_at = "2020-01-01T00:00:00Z";
const started_at = "2020-01-01T00:00:20Z";
const job = { id: 1, name: "amd64", labels: ["release"], created_at, started_at, runner_id: 10, status: "in_progress" };
const now = Date.parse(created_at) + 120_000;
assert.deepEqual(runnerWaits([job, { ...job, status: "completed", conclusion: "skipped" }], now), [{ id: 1, job: "amd64", seconds: 20 }]);
assert.equal(runnerWaits([{ ...job, status: "queued", runner_id: 0 }], now)[0].seconds, 120);
});
test("a runner wait over 60 seconds produces an Actions warning and summary", async () => {
const f = fixture({ imageJobs: [{ id: 1, name: "amd64", labels: ["release"], status: "queued", runner_id: 0, created_at: "2020-01-01T00:00:00Z" }] });
await watchReleaseSource({ ...f, now: () => Date.parse("2020-01-01T00:02:00Z") });
assert.ok(f.messages.some(message => message.startsWith("::warning") && message.includes("120s")));
assert.ok(f.messages.some(message => message.includes("Release runner wait: amd64")));
});
});
describe("release candidate admission", () => {
const repository = { id: 1170821064, full_name: "paperclipai/paperclip" };
const ids = { "docker.yml": 101, "release.yml": 102, "cloud-readiness.yml": 123, "cloud-migrator-artifacts.yml": 124 };
function run(file, change = {}) {
return { id: ids[file] + 1000, workflow_id: ids[file], path: `.github/workflows/${file}`,
repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push",
run_attempt: 1, status: "in_progress", conclusion: null, ...change };
}
function job(run, name, conclusion = "skipped") {
return { id: 789, run_id: run.id, run_attempt: run.run_attempt, head_sha: run.head_sha,
name, status: "completed", conclusion };
}
function fixture(file = "cloud-readiness.yml", change = {}) {
const producer = run(file, { status: "completed", conclusion: "success", ...change });
const admission = file === "cloud-migrator-artifacts.yml" ? migratorAdmissionJob : sourceAdmissionJob;
const f = { runs: { [file]: [producer] }, jobs: { [producer.id]: [job(producer, admission, "success"), job(producer, sourceVerificationJob)] },
current: {}, writes: [], calls: [], requested: new Set(), file, sha, admissionJob: admission };
f.api = async path => {
f.calls.push(path);
const url = new URL(`https://api.github.com${path}`);
const fileMatch = path.match(/\/workflows\/([^/?]+\.yml)$/);
if (fileMatch) return { id: ids[fileMatch[1]], path: `.github/workflows/${fileMatch[1]}` };
const listMatch = path.match(/\/workflows\/(\d+)\/runs\?/);
if (listMatch) {
const name = Object.keys(ids).find(file => ids[file] === Number(listMatch[1]));
const runs = f.runs[name] ?? [];
return { workflow_runs: runs, total_count: f.total ?? runs.length };
}
const jobMatch = path.match(/\/runs\/(\d+)\/attempts\/(\d+)\/jobs\?/);
if (jobMatch) {
const page = Number(url.searchParams.get("page") ?? "1");
const jobs = f.jobs[Number(jobMatch[1])] ?? [];
return { total_count: f.jobTotal ?? jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100) };
}
const currentMatch = path.match(/\/runs\/(\d+)$/);
if (currentMatch) return f.current[Number(currentMatch[1])] ?? Object.values(f.runs).flat().find(run => run.id === Number(currentMatch[1]));
throw new Error(`Unexpected admission API path: ${path}`);
};
f.rerun = async path => f.writes.push(path);
return f;
}
test("only a running image or npm release admits a normal push", async () => {
for (const file of ["docker.yml", "release.yml"]) {
const f = fixture();
assert.equal(await releaseAdmitted(sha, f), false);
f.runs[file] = [run(file)];
assert.equal(await releaseAdmitted(sha, f), true);
assert.equal(await releaseAdmitted(sha, { ...f, imageOnly: true }), file === "docker.yml");
for (const status of ["queued", "pending", "completed"]) {
f.runs[file] = [run(file, { status })];
assert.equal(await releaseAdmitted(sha, f), false, status);
}
}
});
test("manual and retry requests admit exact source without consumer discovery", async () => {
const api = () => assert.fail("explicit request needs no discovery");
assert.equal(await releaseAdmitted(sha, { api, event: "workflow_dispatch" }), true);
assert.equal(await releaseAdmitted(sha, { api, attempt: 2 }), true);
assert.equal(await releaseAdmitted(sha, { api, event: "pull_request", attempt: 2 }), false);
for (const attempt of [0, "2"]) {
await assert.rejects(releaseAdmitted(sha, { api, attempt }), /Invalid release attempt/);
}
});
test("forks, stale identities, tags, and non-push consumers do not admit work", async () => {
for (const change of [{ repository: { ...repository, id: 2 } }, { head_repository: { ...repository, id: 2 } },
{ repository: { ...repository, full_name: "fork/paperclip" } }, { path: ".github/workflows/pr.yml" },
{ workflow_id: 999 }, { head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "workflow_dispatch" },
{ id: 0 }, { run_attempt: 0 }]) {
const f = fixture();
f.runs["docker.yml"] = [run("docker.yml", change)];
assert.equal(await releaseAdmitted(sha, f), false, JSON.stringify(change));
}
const f = fixture();
const active = run("docker.yml");
f.runs["docker.yml"] = [active];
f.current[active.id] = { ...active, run_attempt: 2 };
assert.equal(await releaseAdmitted(sha, f), false);
});
test("incomplete discovery and transport failure fail closed", async () => {
for (const total of [1, 101, -1]) {
const f = fixture();
f.total = total;
await assert.rejects(releaseAdmitted(sha, f), /Incomplete/);
assert.deepEqual(f.writes, []);
}
await assert.rejects(releaseAdmitted(sha, { api: async () => { throw new Error("API unavailable"); } }), /unavailable/);
await assert.rejects(releaseAdmitted("master", { api: () => assert.fail() }), /full source SHA/);
});
test("a selected consumer restarts only a strictly deferred producer, once", async () => {
for (const file of ["cloud-readiness.yml", "cloud-migrator-artifacts.yml"]) {
const f = fixture(file);
assert.equal(await requestDeferredProducer(f), "pending");
assert.deepEqual(f.writes, [`/repos/paperclipai/paperclip/actions/runs/${ids[file] + 1000}/rerun`]);
await requestDeferredProducer(f);
assert.equal(f.writes.length, 1);
}
});
test("failed, incomplete, ambiguous, or mismatched work is never automatically retried", async () => {
for (const change of [jobs => { jobs[1].conclusion = "failure"; }, jobs => { jobs[0].conclusion = "failure"; },
jobs => { jobs[1].run_attempt = 2; }, jobs => { jobs[1].head_sha = "b".repeat(40); },
jobs => { jobs[1].run_id = 2; }, jobs => { jobs[1].status = "queued"; },
jobs => { jobs.push({ ...jobs[0] }); }, jobs => { jobs.splice(1); }]) {
const f = fixture();
change(f.jobs[1123]);
assert.equal(await requestDeferredProducer(f), "finished");
assert.deepEqual(f.writes, []);
}
const f = fixture();
for (const change of [{ run_attempt: 2 }, { status: "in_progress" }, { head_sha: "b".repeat(40) }, { id: 2 }]) {
f.current[1123] = { ...f.runs[f.file][0], ...change };
assert.equal(await requestDeferredProducer(f), "pending");
assert.deepEqual(f.writes, []);
}
});
test("an explicitly cancelled or failed producer cannot be restarted by admission", async () => {
for (const conclusion of ["cancelled", "failure", "timed_out"]) {
const f = fixture("cloud-readiness.yml", { conclusion });
assert.equal(await requestDeferredProducer(f), "finished");
assert.deepEqual(f.writes, []);
}
});
test("a truncated job page cannot hide a failed check and trigger a retry", async () => {
const f = fixture();
f.jobTotal = 3;
await assert.rejects(requestDeferredProducer(f), /Incomplete release producer job list/);
assert.deepEqual(f.writes, []);
f.jobTotal = "2";
await assert.rejects(requestDeferredProducer(f), /Invalid release producer job list/);
assert.deepEqual(f.writes, []);
});
test("producer retries reject arbitrary workflow and admission contracts", async () => {
const f = fixture();
for (const change of [{ file: "docker.yml" }, { admissionJob: migratorAdmissionJob }]) {
await assert.rejects(requestDeferredProducer({ ...f, ...change }), /Unknown deferred/);
}
assert.deepEqual(f.calls, []);
});
test("a deferred completion racing the run listing does not become a source failure", async () => {
const f = fixture();
const completed = f.runs[f.file][0];
f.runs[f.file] = [{ ...completed, status: "in_progress", conclusion: null }];
f.current[completed.id] = completed;
await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred);
delete f.current[completed.id];
assert.equal(await readSourceVerification(sha, f.api), undefined);
});
test("a competing consumer's accepted retry does not fail the losing consumer", async () => {
const f = fixture();
f.rerun = async () => {
f.current[1123] = run(f.file, { run_attempt: 2 });
throw new Error("duplicate restart rejected");
};
assert.equal(await requestDeferredProducer(f), "pending");
assert.equal(f.requested.size, 1);
const rejected = fixture();
rejected.rerun = async () => { throw new Error("token denied"); };
await assert.rejects(requestDeferredProducer(rejected), /token denied/);
assert.equal(rejected.requested.size, 0);
});
test("a queued migrator is pending until its build actually starts", async () => {
const f = fixture("cloud-migrator-artifacts.yml", { status: "in_progress", conclusion: null });
const producer = f.runs[f.file][0];
const build = { ...job(producer, "Build migrator", null), status: "queued", labels: ["cloud-release-32"] };
f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build];
assert.equal(await requestDeferredProducer(f), "pending");
build.status = "in_progress";
assert.equal(await requestDeferredProducer(f), "admitted");
build.run_attempt = 2;
assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work");
assert.deepEqual(f.writes, []);
});
test("a rejected duplicate with stale read visibility remains pending without claiming admission", async () => {
for (const status of [409, 422, 403]) {
const f = fixture();
f.rerun = async () => { throw Object.assign(new Error("retry rejected"), { status }); };
if (status === 403) await assert.rejects(requestDeferredProducer(f), /retry rejected/);
else assert.equal(await requestDeferredProducer(f), "pending");
assert.equal(f.requested.size, 0);
}
});
test("source proof waits for admission, restarts it, and accepts only the new attempt", async () => {
const f = fixture();
await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred);
let time = 0;
const proof = await waitForSourceVerification(sha, { ...f, now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {},
sleep: async ms => {
time += ms;
const selected = run(f.file, { run_attempt: 2 });
f.runs[f.file] = [selected];
f.jobs[selected.id] = [job(selected, sourceAdmissionJob, "success"), job(selected, sourceVerificationJob, "success")];
} });
assert.equal(proof.attempt, 2);
assert.equal(time, 10);
assert.equal(f.writes.length, 1);
});
test("the image monitor admits a late migrator even when source proof is ready", async () => {
const f = fixture("cloud-migrator-artifacts.yml");
const source = run("cloud-readiness.yml");
f.runs["cloud-readiness.yml"] = [source];
f.jobs[source.id] = [job(source, sourceVerificationJob, "success")];
const image = run("docker.yml");
f.runs["docker.yml"] = [image];
f.jobs[image.id] = [];
let time = 0;
const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"),
now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => {
time += ms;
const selected = run(f.file, { run_attempt: 2 });
f.runs[f.file] = [selected];
f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }];
} });
assert.equal(proof.sha, sha);
assert.equal(time, 10);
assert.equal(f.writes.length, 1);
});
test("the retry transport is a bounded POST to a canonical run", async () => {
const calls = [];
const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; });
await assert.rejects(rerun("/repos/fork/paperclip/actions/runs/1/rerun"), /Invalid release retry/);
await rerun("/repos/paperclipai/paperclip/actions/runs/1/rerun");
assert.equal(calls.length, 1);
assert.equal(calls[0].options.method, "POST");
assert.equal(calls[0].options.redirect, "error");
});
});
+108
View File
@@ -0,0 +1,108 @@
import { appendFile } from "node:fs/promises";
import { pathToFileURL } from "node:url";
import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs";
import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
const repository = "paperclipai/paperclip";
export function runnerWaits(jobs, now = Date.now()) {
return jobs.filter(job => ["queued", "in_progress", "completed"].includes(job.status) &&
job.conclusion !== "skipped" && job.labels?.length > 0).map(job => ({
job: job.name, id: job.id,
seconds: Math.max(0, Math.floor(((job.runner_id > 0 && job.started_at ? Date.parse(job.started_at) : now) - Date.parse(job.created_at)) / 1000)),
})).filter(job => Number.isFinite(job.seconds));
}
// The write is limited to this canonical master Docker run. Never cancel
// another workflow, a tag release, or a run selected from untrusted input.
export async function cancelImageRun({ api, cancel, runId, attempt, sha }) {
const run = await api(`/repos/${repository}/actions/runs/${runId}`);
if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== sha || run.head_branch !== "master" ||
run.repository?.id !== 1170821064 || run.head_repository?.id !== 1170821064 ||
run.path !== ".github/workflows/docker.yml" || !["push", "workflow_dispatch"].includes(run.event)) {
throw new Error("Refusing to cancel a different image workflow identity.");
}
if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${runId}/cancel`);
}
async function cancelFailedVerification({ api, cancel, sha }, failure) {
const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`);
// A concurrent retry owns the new attempt; a late observer leaves it alone.
if (run.run_attempt !== failure.attempt) return;
if (run.id !== failure.runId || run.head_sha !== sha || run.head_branch !== "master" || run.event !== "push" ||
run.repository?.full_name !== repository || run.head_repository?.full_name !== repository ||
run.path !== ".github/workflows/cloud-readiness.yml") throw new Error("Failed verification identity changed.");
if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`);
}
export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now,
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log,
timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) {
if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 ||
!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release identity.");
const deadline = now() + timeoutMs;
const warned = new Set();
const requested = new Set();
let migratorAdmitted = !rerun;
const reportWaits = async jobs => {
for (const wait of runnerWaits(jobs, now())) {
if (wait.seconds > 60 && !warned.has(wait.id)) {
warned.add(wait.id);
log(`::warning title=Release runner waited over 60 seconds::Job ${wait.id} waited ${wait.seconds}s for a runner.`);
await summary(`Release runner wait: ${wait.job} — ${wait.seconds}s (job ${wait.id}).\n`);
}
}
};
while (now() < deadline) {
if (!migratorAdmitted) {
migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending";
}
const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`);
if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing.");
await reportWaits(batch.jobs);
try {
const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits });
if (proof && migratorAdmitted) {
await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`);
return proof;
}
} catch (error) {
if (error instanceof SourceVerificationDeferred && rerun) {
await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
} else {
if (error instanceof SourceVerificationFailed) {
await cancelFailedVerification({ api, cancel, sha }, error);
await cancelImageRun({ api, cancel, runId, attempt, sha });
}
throw error;
}
}
await sleep(Math.min(intervalMs, Math.max(0, deadline - now())));
}
// This early failure observer is not the publication deadline. Healthy
// checks can outlive a Spot worker; a separate exact-source gate still
// waits for proof before publishing any image.
if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline.");
await summary("Source monitoring window ended; final publication still requires exact-source verification.\n");
return undefined;
}
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const token = process.env.GITHUB_TOKEN;
if (!token || process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Canonical master token required.");
const requireProof = process.argv.includes("--require-proof");
const checkProof = process.argv.includes("--check-proof");
const api = createActionsReader({ token });
const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT),
api,
rerun: createReleaseRerunner(token),
cancel: async path => {
const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error",
headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });
if (!response.ok) throw new Error(`Could not cancel failed-source image work: HTTP ${response.status}.`);
},
summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(),
});
if (checkProof && !proof) throw new Error("Exact-source proof changed before publication.");
if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`);
}