mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-04 11:01:16 +01:00
perf(ci): admit release candidates and isolate native image capacity (#18869)
Admit full source verification only for exact-source Docker or npm release work, while superseded pending sources complete metadata-only admission. Restart only strictly deferred runs; failed or cancelled checks require an explicit request. Route canonical master native images to dedicated 32-core Spot profiles and keep source checks on protected verification capacity. Use the four-slot GitHub-hosted publication pool for registry work and signing, with proof waits outside that pool and a fresh source-proof check before tagging. Trim historical checkout blobs and prepared-runner disk cleanup while preserving tag and fork publication behavior. Validation: release/source and source/artifact regressions, 665 workflow contracts, actionlint, complete latest-head PR CI, and Greptile 5/5 with all comments resolved. Existing CLI and large-file repository timing tests also passed independently; their CI shards received one scoped retry. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
8771606fd8
commit
f774992832
13 files changed
+886
-63
No files matched your search
@@ -10,7 +10,8 @@ const base = {
|
||||
ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1,
|
||||
};
|
||||
const expectedJobs = {
|
||||
"cloud-readiness.yml": [],
|
||||
"docker.yml": ["source_verification", "publication_verification"],
|
||||
"cloud-readiness.yml": ["admission", "source_verified"],
|
||||
"cloud-migrator-artifacts.yml": [],
|
||||
"release-verify.yml": ["typecheck", "general_tests", "serialized_tests", "runner_workflow_evals", "verify_paperclip_runner", "build"],
|
||||
"runner-chaos-evals.yml": ["chaos_and_recovery"],
|
||||
@@ -59,9 +60,9 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) {
|
||||
cases.push({ name: "stable release", inputs: { channel: "stable" } });
|
||||
}
|
||||
cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet });
|
||||
cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } });
|
||||
cases.push({ name: "later rerun", github: { run_attempt: 3 } });
|
||||
cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } });
|
||||
cases.push({ name: "admitted retry", github: { run_attempt: 2 }, expected: release ? "ubuntu-latest" : fleet });
|
||||
cases.push({ name: "later retry", github: { run_attempt: 3 }, expected: release ? "ubuntu-latest" : fleet });
|
||||
cases.push({ name: "retry attempt as a string", github: { run_attempt: "2" }, expected: release ? "ubuntu-latest" : fleet });
|
||||
cases.push({ name: "missing attempt", github: { run_attempt: undefined } });
|
||||
for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) {
|
||||
test(`${file}/${job}: ${name}`, () => {
|
||||
@@ -88,7 +89,7 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) {
|
||||
}
|
||||
|
||||
|
||||
test("Cloud readiness bookkeeping never waits for the AWS verification fleet", () => {
|
||||
test("Cloud source proof keeps source-only gating on protected verification capacity", () => {
|
||||
const workflow = readFileSync(new URL("../../workflows/cloud-readiness.yml", import.meta.url), "utf8");
|
||||
const bodies = new Map();
|
||||
for (const [name, needs] of [
|
||||
@@ -97,7 +98,7 @@ test("Cloud readiness bookkeeping never waits for the AWS verification fleet", (
|
||||
const body = workflow.match(new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=^ [a-z_]+:|(?![\\s\\S]))`, "m"))?.[1];
|
||||
assert.ok(body, `missing ${name} job`);
|
||||
bodies.set(name, body);
|
||||
assert.match(body, /^ runs-on: ubuntu-latest$/m);
|
||||
assert.match(body, /runs-on:.*paperclip-post-merge-x64/);
|
||||
assert.doesNotMatch(body, /^ +continue-on-error:|^ +if:.*always\(\)/m);
|
||||
assert.match(body, /^ if: github.repository == 'paperclipai\/paperclip' && github.ref == 'refs\/heads\/master'$/m);
|
||||
assert.match(body, /^ +SOURCE_SHA: \$\{\{ github.sha \}\}$/m);
|
||||
|
||||
@@ -12,9 +12,33 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
build:
|
||||
admission:
|
||||
name: Cloud image candidate admission v1
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
admitted: ${{ steps.admission.outputs.admitted }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: scripts
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
- id: admission
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: node scripts/cloud-release-admission.mjs image-only
|
||||
|
||||
build:
|
||||
needs: admission
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true'
|
||||
runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }}
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -52,7 +76,7 @@ jobs:
|
||||
publish:
|
||||
needs: build
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -8,14 +8,38 @@ on:
|
||||
|
||||
permissions: {}
|
||||
|
||||
# Source verification must start outside the full npm release's queue.
|
||||
# Source verification stays independent of npm publication and image builds.
|
||||
concurrency:
|
||||
group: cloud-readiness-${{ github.sha }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
admission:
|
||||
name: Cloud candidate admission v1
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
admitted: ${{ steps.admission.outputs.admitted }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: scripts
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
- id: admission
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: node scripts/cloud-release-admission.mjs
|
||||
|
||||
verify:
|
||||
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
|
||||
needs: admission
|
||||
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master' && needs.admission.outputs.admitted == 'true'
|
||||
permissions:
|
||||
contents: read
|
||||
uses: ./.github/workflows/release-verify.yml
|
||||
@@ -29,8 +53,9 @@ jobs:
|
||||
name: Cloud source verified v1
|
||||
needs: [verify]
|
||||
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
|
||||
# Bookkeeping must not wait for the AWS builders it observes.
|
||||
runs-on: ubuntu-latest
|
||||
# Keep the final proof off the shared hosted queue too. The protected
|
||||
# fleet has a separate job ceiling and launches Spot workers on demand.
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -33,6 +33,29 @@ concurrency:
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
source_verification:
|
||||
name: Watch exact-source verification
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 44
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write
|
||||
outputs:
|
||||
verified: ${{ steps.watch.outputs.verified }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Stop images when source checks fail and report runner waits
|
||||
id: watch
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: node scripts/watch-release-source.mjs
|
||||
|
||||
reuse_canary_image:
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v')
|
||||
runs-on: ubuntu-latest
|
||||
@@ -100,11 +123,10 @@ jobs:
|
||||
- platform: linux/arm64
|
||||
runner: ubuntu-24.04-arm
|
||||
arch: arm64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
# Native builds land well inside this; it is a backstop, not a budget.
|
||||
# (The interim fix while this PR landed raised the single QEMU job's cap
|
||||
# to 120 minutes; native per-arch legs make that headroom unnecessary.)
|
||||
timeout-minutes: 60
|
||||
runs-on: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && format('runs-on/fleet=paperclip-core-image-{0}/env=public-ci', matrix.arch == 'amd64' && 'x64' || 'arm64') || matrix.runner }}
|
||||
# Stop before the AWS worker's 45-minute maximum lifetime. Hosted tag
|
||||
# builds retain their existing limit.
|
||||
timeout-minutes: ${{ vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && 40 || 60 }}
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
@@ -115,6 +137,9 @@ jobs:
|
||||
# Full history and tags so `git describe` below can compute the
|
||||
# release version to stamp into the image.
|
||||
fetch-depth: 0
|
||||
# Keep the version's full commit graph without downloading blobs
|
||||
# from every historical revision of this large repository.
|
||||
filter: blob:none
|
||||
|
||||
# `.git` is dockerignored, so a running image cannot derive its own
|
||||
# version and otherwise reports the source package.json placeholder in
|
||||
@@ -183,6 +208,7 @@ jobs:
|
||||
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
|
||||
|
||||
- name: Free runner disk
|
||||
if: ${{ !(vars.CORE_RELEASE_AWS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
echo "Disk before cleanup:"
|
||||
@@ -304,16 +330,40 @@ jobs:
|
||||
if-no-files-found: error
|
||||
retention-days: 1
|
||||
|
||||
# Wait for proof on ephemeral verification capacity. Publication runners do
|
||||
# only registry/signing work. Each Spot observer stays below its 45m lifetime.
|
||||
publication_verification:
|
||||
name: Require exact-source verification
|
||||
if: ${{ !cancelled() && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && needs.build-and-push.result == 'success' && needs.source_verification.result == 'success' && needs.source_verification.outputs.verified != 'true' }}
|
||||
needs: [build-and-push, source_verification]
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 44
|
||||
permissions:
|
||||
contents: read
|
||||
actions: write # restart only exact-source producers deferred before admission
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
- name: Require source proof and migrator admission before publication
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: node scripts/watch-release-source.mjs --require-proof
|
||||
|
||||
# Names the per-architecture digests as one manifest list under the real
|
||||
# tags. Nothing is publicly tagged until this runs, so a half-published
|
||||
# multi-arch image is not a state anything can pull.
|
||||
merge-and-push:
|
||||
if: ${{ !cancelled() && needs.build-and-push.result == 'success' }}
|
||||
needs: build-and-push
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ !cancelled() && needs.build-and-push.result == 'success' && (needs.source_verification.result == 'success' || needs.source_verification.result == 'skipped') && (needs.publication_verification.result == 'success' || (needs.publication_verification.result == 'skipped' && (github.repository != 'paperclipai/paperclip' || github.repository_id != '1170821064' || github.ref != 'refs/heads/master' || needs.source_verification.outputs.verified == 'true'))) }}
|
||||
needs: [build-and-push, source_verification, publication_verification]
|
||||
runs-on: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (vars.CORE_PUBLICATION_RUNNER || 'cloud-release-32') || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
packages: write
|
||||
id-token: write
|
||||
attestations: write
|
||||
@@ -322,8 +372,6 @@ jobs:
|
||||
# script the verification step pipes in.
|
||||
- name: Checkout
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download digests
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
@@ -375,6 +423,19 @@ jobs:
|
||||
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
|
||||
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
# Revalidate the latest attempt after runner assignment without waiting
|
||||
# here. A concurrent source retry must invalidate the earlier proof.
|
||||
- name: Revalidate exact-source proof before publication
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ github.token }}
|
||||
run: node scripts/watch-release-source.mjs --check-proof
|
||||
|
||||
- name: Create manifest list and push
|
||||
working-directory: /tmp/digests
|
||||
run: |
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
typecheck:
|
||||
if: ${{ !inputs.runner_binaries_only }}
|
||||
name: Typecheck
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -120,7 +120,7 @@ jobs:
|
||||
general_tests:
|
||||
if: ${{ !inputs.runner_binaries_only }}
|
||||
name: General tests (${{ matrix.group_label }})
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -232,7 +232,7 @@ jobs:
|
||||
serialized_tests:
|
||||
if: ${{ !inputs.runner_binaries_only }}
|
||||
name: Serialized tests (${{ matrix.shard_label }})
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -282,7 +282,7 @@ jobs:
|
||||
runner_workflow_evals:
|
||||
if: ${{ !inputs.runner_binaries_only }}
|
||||
name: Runner workflow eval scorer contract
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -313,7 +313,7 @@ jobs:
|
||||
verify_paperclip_runner:
|
||||
if: ${{ !inputs.runner_binaries_only }}
|
||||
name: Verify Paperclip Runner (${{ matrix.lane }})
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -523,7 +523,7 @@ jobs:
|
||||
build:
|
||||
if: ${{ !inputs.runner_binaries_only }}
|
||||
name: Build
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -320,12 +320,12 @@ jobs:
|
||||
|
||||
verify_canary:
|
||||
name: Reuse exact-source verification
|
||||
if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 50
|
||||
timeout-minutes: 70
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
actions: write # restart only an exact-source push deferred before admission
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
|
||||
@@ -20,8 +20,8 @@ concurrency:
|
||||
jobs:
|
||||
chaos_and_recovery:
|
||||
name: Restart, replay, trace, and recovery faults
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }}
|
||||
runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt >= 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }}
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
|
||||
@@ -1,30 +1,42 @@
|
||||
# Cloud build readiness
|
||||
|
||||
The `Cloud readiness` workflow starts for every master push and retains the
|
||||
versioned `Cloud source verified v1` job. It calls the full `Release Verify`
|
||||
workflow for that exact commit, including typecheck, builds, general and
|
||||
serialized tests, and Runner verification. The source proof depends on every
|
||||
source check and fails closed if verification fails, is cancelled, or is skipped.
|
||||
The `Cloud readiness` workflow starts a small admission check for each master
|
||||
push. Full verification starts only when the exact source has a running master
|
||||
Docker or automatic npm release workflow. Superseded pending sources avoid the
|
||||
full test matrix. The migrator producer similarly waits for image admission.
|
||||
|
||||
A selected consumer restarts a previously deferred push run, preserving its
|
||||
original source and push identity. This restarts only a successful admission
|
||||
job followed entirely by skipped jobs. A failed check, cancelled job, incomplete
|
||||
listing, or identity mismatch cannot trigger this automatic retry. Explicit
|
||||
manual runs and whole-workflow retries request full verification directly.
|
||||
|
||||
Admitted source runs retain the versioned `Cloud source verified v1` job. They
|
||||
call the full `Release Verify` workflow for that exact commit, including
|
||||
typecheck, builds, general and serialized tests, and Runner verification. The
|
||||
proof depends on every source check and fails closed if verification fails, is
|
||||
cancelled, or is skipped. Consumers accept only the latest attempt's proof.
|
||||
|
||||
The recurring public `-cloud` publisher and its `Cloud deployable v1` gate are
|
||||
retired. The workflow no longer builds a legacy image or waits for one.
|
||||
Keep its filename and source-proof job name stable: npm canary publication and
|
||||
downstream image composers consume that exact contract.
|
||||
|
||||
Standard images still publish independently through `docker.yml`. The
|
||||
Standard images build in parallel through `docker.yml`. Master image publication
|
||||
waits for the exact-source proof; a failed source check cancels its image build
|
||||
without waiting for the remaining test shards. The
|
||||
`cloud-migrator-artifacts.yml` workflow still publishes signed exact-source
|
||||
migrators independently. A downstream composer must verify those artifacts,
|
||||
build and test its own image, and record separate deployment readiness.
|
||||
|
||||
Verification runs outside the full npm release's concurrency group. Different
|
||||
commits have independent groups. The npm canary release reuses the source proof
|
||||
Verification runs outside the full npm release's concurrency group. Each source retains its independent verification run. The npm canary release reuses the source proof
|
||||
for its exact master push instead of starting another `Release Verify` run.
|
||||
Stable releases and candidate-branch betas still run full verification.
|
||||
|
||||
The canary consumer requires the expected workflow ID and path, upstream source
|
||||
repository, master push event, full SHA, and a successful job in the latest run
|
||||
attempt. It checks the run again after reading the jobs to reject a concurrent
|
||||
rerun. Missing proof waits for up to 45 minutes; failed, skipped, cancelled,
|
||||
rerun. Missing proof waits for up to 60 minutes; failed, skipped, cancelled,
|
||||
ambiguous, or mismatched proof cannot authorize publication. API failures fail
|
||||
closed. If a source check fails, fix it and rerun Cloud readiness before retrying
|
||||
the release. Use **Re-run all jobs** when a later attempt did not rerun the source
|
||||
@@ -120,15 +132,16 @@ excluded or sleeping targets, retries, and failures with the fleet result. Recor
|
||||
runner queue conditions and cache state; one warm or cold run is a sample, not a
|
||||
latency guarantee.
|
||||
|
||||
## Reserved AWS verification capacity
|
||||
## Dedicated AWS verification capacity
|
||||
|
||||
`AWS_POST_MERGE_CI_ENABLED=true` routes cloud source verification and
|
||||
exact-master migrator preparation to the
|
||||
`paperclip-post-merge` runner group. The separate Fleet label is
|
||||
`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. Its 36 reserved slots use
|
||||
the same four-vCPU, 16-GiB machines as approved PR jobs. PR capacity is reduced
|
||||
to 64; the separately provisioned image capacity is unchanged by this retirement.
|
||||
This keeps PR bursts from consuming every post-merge verification slot.
|
||||
`runs-on/fleet=paperclip-post-merge-x64/env=public-ci`. The protected fleet uses
|
||||
separate 8-vCPU, 32-GiB Spot workers that launch only when jobs arrive. Its
|
||||
job ceiling and public PR capacity are managed independently. There are no
|
||||
hot or stopped pools. Quota ceilings do not reserve hardware; measure cold
|
||||
launch and runner assignment times as well as job duration.
|
||||
|
||||
Every selector checks the canonical repository name and ID, master ref, and a
|
||||
push or manual event. Reusable verification also requires `inputs.ref` to equal
|
||||
@@ -142,7 +155,8 @@ its trusted-publisher identity.
|
||||
Before enabling the switch, deploy the separate Fleet and restrict its GitHub
|
||||
runner group to repository ID `1170821064` and these workflows at
|
||||
`refs/heads/master`: `cloud-readiness.yml`,
|
||||
`release-verify.yml`, `runner-chaos-evals.yml`, and `release.yml`. Do not authorize
|
||||
`release-verify.yml`, `runner-chaos-evals.yml`, `release.yml`, and `docker.yml`.
|
||||
The Docker workflow uses this group for its exact-source monitor and proof wait. Do not authorize
|
||||
PR-controlled workflow versions. The direct migrator producer always uses
|
||||
GitHub-hosted runners and needs no AWS runner-group authorization. PR placement retains its independent pinned
|
||||
workflow and six-account author/actor allowlist.
|
||||
@@ -158,6 +172,43 @@ route a public image job after this retirement. This source change does not
|
||||
delete runner groups, Fleets, credentials, registry images, or cache tags. Review
|
||||
shared infrastructure ownership separately before removing those resources.
|
||||
|
||||
## Protected standard-image builders
|
||||
|
||||
`CORE_RELEASE_AWS_ENABLED=true` routes canonical master image jobs to native
|
||||
`paperclip-core-image-x64` and `paperclip-core-image-arm64` fleets. Each has two
|
||||
32-vCPU / 128-GiB Spot slots, for four large image jobs in total. Workers
|
||||
launch for jobs and terminate afterward; no idle pool is maintained. The `paperclip-core-release` runner group permits only
|
||||
Core's `docker.yml@refs/heads/master`. PRs, branches, and tag releases keep their
|
||||
existing GitHub-hosted placement. Disable the switch and rerun the whole image
|
||||
workflow to restore hosted placement.
|
||||
|
||||
Image checkout retains the complete commit graph for version stamping but filters
|
||||
out historical blobs. The manifest job needs only a shallow source checkout.
|
||||
Prepared AWS image disks skip the standard hosted runner's disk cleanup. Native
|
||||
architecture builds, digest publication, the orphan-reaping smoke test, and
|
||||
attestation remain required. The source monitor has Actions write permission to cancel its own canonical
|
||||
master Docker run and failed exact-source verification siblings. It never
|
||||
automatically retries a failed source check. Its guarded retry permission can
|
||||
restart only an exact-source producer deferred before full admission. It checks run, attempt, SHA,
|
||||
repository, ref, and workflow identity before cancellation. Its bounded
|
||||
observation window does not cancel healthy pending checks. A successful observer
|
||||
records proof for publication immediately. If healthy checks remain pending after
|
||||
40 minutes, a separate required proof job waits for up to another 40 minutes on
|
||||
the verification fleet. Both jobs fit within the Spot worker lifetime. Neither
|
||||
wait occupies the four-slot publication pool. The hosted manifest job requires
|
||||
successful source proof and admitted migrator work before it starts; a queued
|
||||
migrator job alone is not evidence of admission. It retains its 20-minute budget
|
||||
for manifest verification and signing. It revalidates the latest source attempt
|
||||
once before publishing; changed or missing proof fails immediately without
|
||||
starting another wait on the publication runner.
|
||||
Image jobs retain
|
||||
their existing package permissions.
|
||||
|
||||
The monitor emits an Actions warning and step summary when an image or source job waits
|
||||
more than 60 seconds for a runner. This measures runner assignment, not image
|
||||
execution. Compare automatic merge-to-ready and merge-to-healthy-target timings;
|
||||
a manual dispatch alone does not prove deployment latency.
|
||||
|
||||
### Typecheck Rust dependency cache
|
||||
|
||||
Source verification's typecheck job builds the native Runner binary through the
|
||||
@@ -236,3 +287,10 @@ fi
|
||||
A subsequent master install can populate the missing entry. Check the saved
|
||||
archive size and package reuse in install logs; a cache hit alone does not prove
|
||||
that the entry contains dependencies.
|
||||
|
||||
Image manifest verification and attestation, plus exact-source migrator build
|
||||
and publication, use the protected four-slot GitHub-hosted `cloud-release-32`
|
||||
pool. `CORE_PUBLICATION_RUNNER=ubuntu-latest` restores standard hosted placement.
|
||||
The signing jobs remain GitHub-hosted because downstream provenance explicitly
|
||||
requires that certificate claim. Their master-only group also serves the private
|
||||
Cloud publisher and harness deployment; it excludes PRs and branch benchmarks.
|
||||
@@ -61,6 +61,39 @@ test("Docker Hub mirrors apply to both image builders without widening publicati
|
||||
"the registry-only manifest merge does not need a Docker Hub mirror");
|
||||
});
|
||||
|
||||
test("source proof gates publication without occupying a publication runner", () => {
|
||||
const workflow = readWorkflow("docker.yml");
|
||||
const observer = workflow.split(" source_verification:\n")[1].split(" reuse_canary_image:")[0];
|
||||
const gate = workflow.split(" publication_verification:\n")[1].split(" merge-and-push:")[0];
|
||||
const publisher = workflow.split(" merge-and-push:\n")[1].split(" promote_canary_channel:")[0];
|
||||
assert.match(observer, /verified: \$\{\{ steps.watch.outputs.verified \}\}/);
|
||||
assert.match(gate, /run: node scripts\/watch-release-source.mjs --require-proof/);
|
||||
assert.match(gate, /actions: write # restart only exact-source producers deferred before admission/);
|
||||
assert.match(gate, /needs: \[build-and-push, source_verification\]/);
|
||||
assert.match(publisher, /needs: \[build-and-push, source_verification, publication_verification\]/);
|
||||
assert.doesNotMatch(publisher, /--require-proof|actions: write/);
|
||||
assert.ok(publisher.indexOf("run: node scripts/watch-release-source.mjs --check-proof") < publisher.indexOf("- name: Create manifest list and push"));
|
||||
assert.match(publisher, /actions: read/);
|
||||
assert.match(publisher, /timeout-minutes: 20/);
|
||||
const expression = publisher.match(/^ if: \$\{\{ (.+) \}\}$/m)[1].replaceAll("needs.build-and-push", "needs.build");
|
||||
for (const [ref, observed, fallback, expected, repository = "paperclipai/paperclip", repository_id = "1170821064"] of [
|
||||
["refs/heads/master", "true", "skipped", true],
|
||||
["refs/heads/master", "false", "skipped", false],
|
||||
["refs/heads/master", undefined, "skipped", false],
|
||||
["refs/heads/master", "false", "success", true],
|
||||
["refs/heads/master", "false", "failure", false],
|
||||
["refs/tags/v1", undefined, "skipped", true],
|
||||
["refs/heads/master", undefined, "skipped", true, "fork/paperclip", "123"],
|
||||
["refs/heads/master", undefined, "skipped", true, "paperclipai/paperclip", "123"],
|
||||
]) {
|
||||
const canonical = repository === "paperclipai/paperclip" && repository_id === "1170821064" && ref === "refs/heads/master";
|
||||
assert.equal(runInNewContext(expression, { cancelled: () => false, github: { ref, repository, repository_id }, needs: {
|
||||
build: { result: "success" }, source_verification: { result: canonical ? "success" : "skipped", outputs: { verified: observed } },
|
||||
publication_verification: { result: fallback },
|
||||
} }), expected);
|
||||
}
|
||||
});
|
||||
|
||||
test("chaos verification isolates callers that verify the same source commit", () => {
|
||||
const chaosWorkflow = readWorkflow("runner-chaos-evals.yml");
|
||||
const group = chaosWorkflow.match(/^ group: (.+)$/m)?.[1];
|
||||
@@ -82,11 +115,11 @@ test("chaos verification isolates callers that verify the same source commit", (
|
||||
assert.match(chaosWorkflow, /cancel-in-progress: true/);
|
||||
});
|
||||
|
||||
test("canary reuses exact-source proof while stable keeps full verification", () => {
|
||||
test("canary can admit deferred exact-source checks while stable keeps full verification", () => {
|
||||
const releaseWorkflow = readWorkflow("release.yml");
|
||||
const canary = releaseWorkflow.match(/ verify_canary:\n[\s\S]*?(?=\n [a-z_]+:)/)[0];
|
||||
assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
|
||||
assert.match(canary, /actions: read/);
|
||||
assert.match(canary, /github\.repository == 'paperclipai\/paperclip' && github\.repository_id == '1170821064' && github\.event_name == 'push' && github\.ref == 'refs\/heads\/master'/);
|
||||
assert.match(canary, /actions: write # restart only an exact-source push deferred before admission/);
|
||||
assert.match(canary, /ref: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(canary, /SOURCE_SHA: \$\{\{ github\.sha \}\}/);
|
||||
assert.match(canary, /run: node scripts\/cloud-source-verification\.mjs "\$SOURCE_SHA"/);
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
import { appendFile } from "node:fs/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const repository = "paperclipai/paperclip";
|
||||
const repositoryId = 1170821064;
|
||||
const workflows = new Set(["docker.yml", "release.yml", "cloud-readiness.yml", "cloud-migrator-artifacts.yml"]);
|
||||
export const sourceAdmissionJob = "Cloud candidate admission v1";
|
||||
export const migratorAdmissionJob = "Cloud image candidate admission v1";
|
||||
|
||||
function assertSha(sha) {
|
||||
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("Admission requires a full source SHA.");
|
||||
}
|
||||
|
||||
function trusted(run, sha, workflow) {
|
||||
return run.workflow_id === workflow.id && run.path === workflow.path && run.head_sha === sha &&
|
||||
run.repository?.id === repositoryId && run.head_repository?.id === repositoryId &&
|
||||
run.repository?.full_name === repository && run.head_repository?.full_name === repository &&
|
||||
run.head_branch === "master" && run.event === "push" &&
|
||||
Number.isSafeInteger(run.id) && run.id > 0 && Number.isSafeInteger(run.run_attempt) && run.run_attempt > 0;
|
||||
}
|
||||
|
||||
async function producer(file, sha, api, status) {
|
||||
assertSha(sha);
|
||||
if (!workflows.has(file)) throw new Error("Unknown release workflow.");
|
||||
const workflow = await api(`/repos/${repository}/actions/workflows/${file}`);
|
||||
if (workflow.path !== `.github/workflows/${file}` || !Number.isSafeInteger(workflow.id) || workflow.id < 1) throw new Error("Release workflow identity mismatch.");
|
||||
const list = await api(`/repos/${repository}/actions/workflows/${workflow.id}/runs?branch=master&event=push&head_sha=${sha}&per_page=100${status ? `&status=${status}` : ""}`);
|
||||
if (!Array.isArray(list.workflow_runs) || !Number.isSafeInteger(list.total_count) || list.total_count < 0 ||
|
||||
list.total_count > 100 || list.workflow_runs.length !== list.total_count) throw new Error("Incomplete release workflow listing.");
|
||||
const run = list.workflow_runs.filter(run => trusted(run, sha, workflow) && (!status || run.status === status)).sort((a, b) => b.id - a.id)[0];
|
||||
return { workflow, run };
|
||||
}
|
||||
|
||||
// A retry is an explicit request for this exact immutable source. Normal push
|
||||
// work starts only for a candidate admitted by an image build or npm canary.
|
||||
export async function releaseAdmitted(sha, { api, event = "push", attempt = 1, imageOnly = false }) {
|
||||
assertSha(sha);
|
||||
if (!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release attempt.");
|
||||
if (!["push", "workflow_dispatch"].includes(event)) return false;
|
||||
if (event === "workflow_dispatch" || (Number.isSafeInteger(attempt) && attempt > 1)) return true;
|
||||
for (const file of imageOnly ? ["docker.yml"] : ["docker.yml", "release.yml"]) {
|
||||
const { workflow, run } = await producer(file, sha, api, "in_progress");
|
||||
if (!run) continue;
|
||||
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
|
||||
if (current.id === run.id && trusted(current, sha, workflow) && current.run_attempt === run.run_attempt && current.status === "in_progress") return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function deferredProducer(jobs, run, admissionJob) {
|
||||
const admission = jobs.filter(job => job.name === admissionJob);
|
||||
const remaining = jobs.filter(job => job.name !== admissionJob);
|
||||
return run.status === "completed" && run.conclusion === "success" && admission.length === 1 && admission[0].status === "completed" && admission[0].conclusion === "success" && remaining.length > 0 &&
|
||||
remaining.every(job => job.status === "completed" && job.conclusion === "skipped") &&
|
||||
jobs.every(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === run.head_sha);
|
||||
}
|
||||
|
||||
// Only skipped admission is restartable here. A failed check or build never
|
||||
// becomes an automatic retry, and every write revalidates canonical identity.
|
||||
export async function requestDeferredProducer({ file, sha, admissionJob, api, rerun, requested = new Set() }) {
|
||||
if (({ "cloud-readiness.yml": sourceAdmissionJob, "cloud-migrator-artifacts.yml": migratorAdmissionJob })[file] !== admissionJob) throw new Error("Unknown deferred producer contract.");
|
||||
const { workflow, run } = await producer(file, sha, api);
|
||||
if (!run) return "pending";
|
||||
const key = `${run.id}/${run.run_attempt}`;
|
||||
if (requested.has(key)) return "pending";
|
||||
const jobs = [];
|
||||
let total;
|
||||
for (let page = 1; page <= 10; page++) {
|
||||
const batch = await api(`/repos/${repository}/actions/runs/${run.id}/attempts/${run.run_attempt}/jobs?per_page=100&page=${page}`);
|
||||
if (!Array.isArray(batch.jobs) || !Number.isSafeInteger(batch.total_count) || batch.total_count < 0) throw new Error("Invalid release producer job list.");
|
||||
if ((total !== undefined && total !== batch.total_count) ||
|
||||
batch.jobs.length !== Math.min(100, Math.max(0, batch.total_count - (page - 1) * 100))) throw new Error("Incomplete release producer job list.");
|
||||
total = batch.total_count;
|
||||
jobs.push(...batch.jobs);
|
||||
if (page * 100 >= total) break;
|
||||
if (page === 10) throw new Error("Incomplete release producer job list.");
|
||||
}
|
||||
if (run.status !== "completed") {
|
||||
const current = jobs.filter(job => job.run_id === run.id && job.run_attempt === run.run_attempt && job.head_sha === sha);
|
||||
// Admission itself succeeds for both outcomes. Only an actual build is
|
||||
// evidence that its output admitted work; queued jobs may still be skipped.
|
||||
return current.some(job => job.name !== admissionJob && job.labels?.length > 0 && ["in_progress", "completed"].includes(job.status) && job.conclusion !== "skipped")
|
||||
? "admitted" : "pending";
|
||||
}
|
||||
if (!deferredProducer(jobs, run, admissionJob)) return "finished";
|
||||
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
|
||||
if (current.id !== run.id || !trusted(current, sha, workflow) || current.run_attempt !== run.run_attempt || current.status !== "completed" || current.conclusion !== "success") return "pending";
|
||||
try {
|
||||
await rerun(`/repos/${repository}/actions/runs/${run.id}/rerun`);
|
||||
} catch (error) {
|
||||
// Docker and npm may select the same source at the same time. A rejected
|
||||
// duplicate write is harmless only when a fresh read proves a retry began.
|
||||
const latest = await api(`/repos/${repository}/actions/runs/${run.id}`);
|
||||
if (latest.id !== run.id || !trusted(latest, sha, workflow)) throw error;
|
||||
if (!(latest.run_attempt > run.run_attempt || (latest.run_attempt === run.run_attempt &&
|
||||
["queued", "in_progress", "requested", "pending", "waiting"].includes(latest.status)))) {
|
||||
// GitHub may reject a duplicate before its read API shows the retry.
|
||||
// Leave it pending for the next poll; this is not proof of admission.
|
||||
if ([409, 422].includes(error.status) && latest.run_attempt === run.run_attempt && latest.status === "completed" && latest.conclusion === "success") return "pending";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
requested.add(key);
|
||||
return "pending";
|
||||
}
|
||||
|
||||
export function createReleaseRerunner(token, fetchImpl = fetch) {
|
||||
return async path => {
|
||||
if (!/^\/repos\/paperclipai\/paperclip\/actions\/runs\/[1-9]\d*\/rerun$/.test(path)) throw new Error("Invalid release retry path.");
|
||||
const response = await fetchImpl(`https://api.github.com${path}`, { method: "POST", redirect: "error",
|
||||
headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });
|
||||
if (!response.ok) throw Object.assign(new Error(`Could not admit deferred release work: HTTP ${response.status}.`), { status: response.status });
|
||||
};
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
if (process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master" || !process.env.GITHUB_TOKEN) throw new Error("Canonical master admission context required.");
|
||||
const { createActionsReader } = await import("./cloud-source-verification.mjs");
|
||||
const admitted = await releaseAdmitted(process.env.GITHUB_SHA, { api: createActionsReader({ token: process.env.GITHUB_TOKEN }),
|
||||
event: process.env.GITHUB_EVENT_NAME, attempt: Number(process.env.GITHUB_RUN_ATTEMPT), imageOnly: process.argv[2] === "image-only" });
|
||||
console.log(admitted ? "Release candidate admitted." : "Full build deferred until a release consumer admits this source.");
|
||||
if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `admitted=${admitted}\n`);
|
||||
}
|
||||
@@ -1,9 +1,18 @@
|
||||
import { appendFile } from "node:fs/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { deferredProducer, requestDeferredProducer, sourceAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
|
||||
|
||||
const repository = "paperclipai/paperclip";
|
||||
const workflowPath = ".github/workflows/cloud-readiness.yml";
|
||||
export const sourceVerificationJob = "Cloud source verified v1";
|
||||
export class SourceVerificationFailed extends Error {
|
||||
constructor(message, run) {
|
||||
super(message);
|
||||
this.runId = run.id;
|
||||
this.attempt = run.run_attempt;
|
||||
}
|
||||
}
|
||||
export class SourceVerificationDeferred extends Error {}
|
||||
|
||||
function assertSha(sha) {
|
||||
if (!/^[a-f0-9]{40}$/.test(sha ?? "")) throw new Error("A full lowercase source SHA is required.");
|
||||
@@ -19,7 +28,7 @@ function trustedRun(run, sha, workflowId) {
|
||||
|
||||
// Consume one versioned job, independent of image/migrator availability. A
|
||||
// failed image build must not invalidate source checks that already passed.
|
||||
export async function readSourceVerification(sha, api) {
|
||||
export async function readSourceVerification(sha, api, { failFast = false, onJobs } = {}) {
|
||||
assertSha(sha);
|
||||
const workflow = await api(`/repos/${repository}/actions/workflows/cloud-readiness.yml`);
|
||||
if (workflow.path !== workflowPath || !Number.isSafeInteger(workflow.id) || workflow.id < 1) {
|
||||
@@ -46,6 +55,17 @@ export async function readSourceVerification(sha, api) {
|
||||
}
|
||||
const matches = jobs.filter((job) => job.name === sourceVerificationJob);
|
||||
if (matches.length > 1) throw new Error("Cloud source verification job is ambiguous.");
|
||||
await onJobs?.(jobs.filter(job => job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt));
|
||||
if (failFast) {
|
||||
const failure = jobs.find((job) => job.status === "completed" &&
|
||||
["failure", "cancelled", "timed_out", "action_required", "startup_failure", "stale"].includes(job.conclusion) &&
|
||||
job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt);
|
||||
if (failure) {
|
||||
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
|
||||
if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
|
||||
throw new SourceVerificationFailed(`Cloud source verification failed: ${failure.name} (${failure.conclusion}).`, run);
|
||||
}
|
||||
}
|
||||
const job = matches[0];
|
||||
if (job?.status === "completed" && job.conclusion === "success" &&
|
||||
job.head_sha === sha && job.run_id === run.id && job.run_attempt === run.run_attempt) {
|
||||
@@ -56,13 +76,17 @@ export async function readSourceVerification(sha, api) {
|
||||
return { sha, runId: run.id, attempt: run.run_attempt, jobId: job.id };
|
||||
}
|
||||
if (job?.status === "completed" || run.status === "completed") {
|
||||
throw new Error(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`);
|
||||
const current = await api(`/repos/${repository}/actions/runs/${run.id}`);
|
||||
if (!trustedRun(current, sha, workflow.id) || current.run_attempt !== run.run_attempt) return undefined;
|
||||
if (job?.conclusion === "skipped" && current.status !== "completed") return undefined;
|
||||
if (deferredProducer(jobs, current, sourceAdmissionJob)) throw new SourceVerificationDeferred("Source checks are deferred until release admission.");
|
||||
throw new SourceVerificationFailed(`Cloud source verification did not pass for ${sha} (run ${run.id}, attempt ${run.run_attempt}). Rerun Cloud readiness before retrying the release.`, run);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
// Statuses that say "ask again", not "the answer is no". A release must not be
|
||||
// blocked because GitHub returned a gateway error during a 45-minute poll.
|
||||
// blocked because GitHub returned a gateway error during a 60-minute poll.
|
||||
const TRANSIENT_READ_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504]);
|
||||
|
||||
// A rate-limited read also says "ask again", and GitHub reports both primary
|
||||
@@ -137,14 +161,20 @@ export function createActionsReader({
|
||||
|
||||
export async function waitForSourceVerification(sha, {
|
||||
api, now = Date.now, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
timeoutMs = 45 * 60_000, intervalMs = 30_000, log = console.log,
|
||||
timeoutMs = 60 * 60_000, intervalMs = 30_000, log = console.log, rerun,
|
||||
} = {}) {
|
||||
assertSha(sha);
|
||||
const deadline = now() + timeoutMs;
|
||||
const requested = new Set();
|
||||
log(`Waiting for ${sourceVerificationJob} for ${sha}.`);
|
||||
while (now() < deadline) {
|
||||
const result = await readSourceVerification(sha, api);
|
||||
if (result) return result;
|
||||
try {
|
||||
const result = await readSourceVerification(sha, api);
|
||||
if (result) return result;
|
||||
} catch (error) {
|
||||
if (!(error instanceof SourceVerificationDeferred) || !rerun) throw error;
|
||||
await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
|
||||
}
|
||||
const remaining = deadline - now();
|
||||
if (remaining > 0) await sleep(Math.min(intervalMs, remaining));
|
||||
}
|
||||
@@ -156,10 +186,11 @@ if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href)
|
||||
if (!process.env.GITHUB_TOKEN) throw new Error("GITHUB_TOKEN with Actions read access is required.");
|
||||
// One deadline for both layers: the reader stops retrying when the poll it
|
||||
// serves is out of time, instead of extending the wait past its timeout.
|
||||
const timeoutMs = 45 * 60_000;
|
||||
const timeoutMs = 60 * 60_000;
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
const api = createActionsReader({ token: process.env.GITHUB_TOKEN, deadlineAt: () => deadline });
|
||||
const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs });
|
||||
const rerun = process.env.GITHUB_REPOSITORY === repository && process.env.GITHUB_REF === "refs/heads/master" ? createReleaseRerunner(process.env.GITHUB_TOKEN) : undefined;
|
||||
const proof = await waitForSourceVerification(process.argv[2], { api, timeoutMs, rerun });
|
||||
const message = `Source verification passed for ${proof.sha}: https://github.com/${repository}/actions/runs/${proof.runId}/attempts/${proof.attempt} (job ${proof.jobId}).`;
|
||||
console.log(message);
|
||||
if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${message}\n`);
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification } from "./cloud-source-verification.mjs";
|
||||
import test, { describe } from "node:test";
|
||||
import { createActionsReader, readSourceVerification, sourceVerificationJob, waitForSourceVerification, SourceVerificationDeferred } from "./cloud-source-verification.mjs";
|
||||
|
||||
import { releaseAdmitted, deferredProducer, requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
|
||||
|
||||
import { cancelImageRun, runnerWaits, watchReleaseSource } from "./watch-release-source.mjs";
|
||||
|
||||
const sha = "a".repeat(40);
|
||||
const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
|
||||
@@ -62,7 +66,8 @@ test("job identities and terminal failures fail closed", async () => {
|
||||
{ head_sha: "b".repeat(40) }, { run_id: 999 }, { run_attempt: 1 },
|
||||
{ conclusion: "failure" }, { conclusion: "cancelled" }, { conclusion: "skipped" },
|
||||
]) {
|
||||
const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }] });
|
||||
const { api } = fixture({ jobs: [{ ...baseJob, ...overrides }],
|
||||
current: overrides.conclusion === "skipped" ? { ...baseRun, status: "completed", conclusion: "success" } : baseRun });
|
||||
await assert.rejects(readSourceVerification(sha, api), /did not pass/);
|
||||
}
|
||||
});
|
||||
@@ -231,3 +236,357 @@ test("retries stop at the caller's deadline rather than outliving the poll", asy
|
||||
assert.equal(seen.length, 1);
|
||||
assert.deepEqual(waits, []);
|
||||
});
|
||||
|
||||
|
||||
describe("image build source monitor", () => {
|
||||
const sha = "a".repeat(40);
|
||||
const runId = 50;
|
||||
const attempt = 1;
|
||||
const workflow = { id: 123, path: ".github/workflows/cloud-readiness.yml" };
|
||||
const sourceRun = { id: 99, workflow_id: 123, path: workflow.path, run_attempt: 1,
|
||||
repository: { full_name: "paperclipai/paperclip" }, head_repository: { full_name: "paperclipai/paperclip" },
|
||||
head_sha: sha, head_branch: "master", event: "push", status: "in_progress" };
|
||||
const imageRun = { id: runId, run_attempt: attempt, head_sha: sha, head_branch: "master", event: "push",
|
||||
repository: { id: 1170821064 }, head_repository: { id: 1170821064 },
|
||||
path: ".github/workflows/docker.yml", status: "in_progress" };
|
||||
const proof = { id: 111, name: "Cloud source verified v1", run_id: 99, run_attempt: 1,
|
||||
head_sha: sha, status: "completed", conclusion: "success" };
|
||||
|
||||
function fixture({ jobs = [proof], run = sourceRun, current = run, image = imageRun, imageJobs = [], missing = false } = {}) {
|
||||
const cancellations = [];
|
||||
const messages = [];
|
||||
const api = async path => {
|
||||
if (path.endsWith("/workflows/cloud-readiness.yml")) return workflow;
|
||||
if (path.includes("/workflows/123/runs?")) return { workflow_runs: missing ? [] : [run], total_count: missing ? 0 : 1 };
|
||||
if (path.includes("/runs/99/attempts/1/jobs?")) return { jobs };
|
||||
if (path.endsWith("/runs/99")) return current;
|
||||
if (path.includes(`/runs/${runId}/attempts/${attempt}/jobs?`)) return { jobs: imageJobs, total_count: imageJobs.length };
|
||||
if (path.endsWith(`/runs/${runId}`)) return image;
|
||||
throw new Error(`Unexpected request: ${path}`);
|
||||
};
|
||||
return { sha, runId, attempt, api, cancel: async path => cancellations.push(path),
|
||||
cancellations, messages, log: message => messages.push(message), summary: async message => messages.push(message) };
|
||||
}
|
||||
|
||||
test("successful exact-source proof permits publication without canceling image work", async () => {
|
||||
const f = fixture();
|
||||
assert.equal((await watchReleaseSource(f)).jobId, proof.id);
|
||||
assert.deepEqual(f.cancellations, []);
|
||||
});
|
||||
|
||||
test("a required check failing cancels the same image attempt before siblings finish", async () => {
|
||||
const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }] });
|
||||
await assert.rejects(watchReleaseSource(f), /General tests/);
|
||||
assert.deepEqual(f.cancellations, ["/repos/paperclipai/paperclip/actions/runs/99/cancel", `/repos/paperclipai/paperclip/actions/runs/${runId}/cancel`]);
|
||||
});
|
||||
|
||||
test("a superseded readiness attempt cancels image work with no deployable future", async () => {
|
||||
const f = fixture({ jobs: [], run: { ...sourceRun, status: "completed", conclusion: "cancelled" } });
|
||||
await assert.rejects(watchReleaseSource(f), /did not pass/);
|
||||
assert.equal(f.cancellations.length, 1);
|
||||
});
|
||||
|
||||
test("a rerun racing a failed job is retried rather than canceling the image", async () => {
|
||||
const f = fixture({ jobs: [{ ...proof, name: "General tests", conclusion: "failure" }],
|
||||
current: { ...sourceRun, run_attempt: 2 } });
|
||||
let time = 0;
|
||||
const api = f.api;
|
||||
f.api = async path => time > 0 ? fixture().api(path) : api(path);
|
||||
const result = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; } });
|
||||
assert.equal(result.jobId, proof.id);
|
||||
assert.deepEqual(f.cancellations, []);
|
||||
});
|
||||
|
||||
test("a transport failure does not issue a destructive cancellation", async () => {
|
||||
const f = fixture();
|
||||
await assert.rejects(watchReleaseSource({ ...f, api: async () => { throw new Error("GitHub unavailable"); } }), /unavailable/);
|
||||
assert.deepEqual(f.cancellations, []);
|
||||
});
|
||||
|
||||
test("the observation window ends without cancelling healthy pending verification", async () => {
|
||||
const f = fixture({ missing: true });
|
||||
let time = 0;
|
||||
const proof = await watchReleaseSource({ ...f, now: () => time, sleep: async ms => { time += ms; }, timeoutMs: 20 });
|
||||
assert.equal(proof, undefined);
|
||||
assert.equal(time, 20);
|
||||
assert.deepEqual(f.cancellations, []);
|
||||
assert.ok(f.messages.some(message => message.includes("final publication still requires exact-source verification")));
|
||||
});
|
||||
|
||||
test("final publication requires proof even after the early observation window ends", async () => {
|
||||
const f = fixture({ missing: true });
|
||||
let time = 0;
|
||||
await assert.rejects(watchReleaseSource({ ...f, requireProof: true, now: () => time,
|
||||
sleep: async ms => { time += ms; }, timeoutMs: 20 }), /before the publication deadline/);
|
||||
assert.deepEqual(f.cancellations, []);
|
||||
assert.equal((await watchReleaseSource({ ...fixture(), requireProof: true })).jobId, proof.id);
|
||||
});
|
||||
|
||||
test("cancellation rejects another SHA, repository, ref, workflow, or attempt", async () => {
|
||||
for (const change of [{ head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "pull_request" },
|
||||
{ path: ".github/workflows/release.yml" }, { run_attempt: 2 }, { id: 51 },
|
||||
{ repository: { id: 42 } }, { head_repository: { id: 42 } }]) {
|
||||
const f = fixture({ image: { ...imageRun, ...change } });
|
||||
await assert.rejects(cancelImageRun(f), /different image workflow identity/);
|
||||
assert.deepEqual(f.cancellations, []);
|
||||
}
|
||||
});
|
||||
|
||||
test("queue measurement does not count running time or skipped dependency jobs", () => {
|
||||
const created_at = "2020-01-01T00:00:00Z";
|
||||
const started_at = "2020-01-01T00:00:20Z";
|
||||
const job = { id: 1, name: "amd64", labels: ["release"], created_at, started_at, runner_id: 10, status: "in_progress" };
|
||||
const now = Date.parse(created_at) + 120_000;
|
||||
assert.deepEqual(runnerWaits([job, { ...job, status: "completed", conclusion: "skipped" }], now), [{ id: 1, job: "amd64", seconds: 20 }]);
|
||||
assert.equal(runnerWaits([{ ...job, status: "queued", runner_id: 0 }], now)[0].seconds, 120);
|
||||
});
|
||||
|
||||
test("a runner wait over 60 seconds produces an Actions warning and summary", async () => {
|
||||
const f = fixture({ imageJobs: [{ id: 1, name: "amd64", labels: ["release"], status: "queued", runner_id: 0, created_at: "2020-01-01T00:00:00Z" }] });
|
||||
await watchReleaseSource({ ...f, now: () => Date.parse("2020-01-01T00:02:00Z") });
|
||||
assert.ok(f.messages.some(message => message.startsWith("::warning") && message.includes("120s")));
|
||||
assert.ok(f.messages.some(message => message.includes("Release runner wait: amd64")));
|
||||
});
|
||||
});
|
||||
|
||||
describe("release candidate admission", () => {
|
||||
const repository = { id: 1170821064, full_name: "paperclipai/paperclip" };
|
||||
const ids = { "docker.yml": 101, "release.yml": 102, "cloud-readiness.yml": 123, "cloud-migrator-artifacts.yml": 124 };
|
||||
function run(file, change = {}) {
|
||||
return { id: ids[file] + 1000, workflow_id: ids[file], path: `.github/workflows/${file}`,
|
||||
repository, head_repository: repository, head_sha: sha, head_branch: "master", event: "push",
|
||||
run_attempt: 1, status: "in_progress", conclusion: null, ...change };
|
||||
}
|
||||
function job(run, name, conclusion = "skipped") {
|
||||
return { id: 789, run_id: run.id, run_attempt: run.run_attempt, head_sha: run.head_sha,
|
||||
name, status: "completed", conclusion };
|
||||
}
|
||||
function fixture(file = "cloud-readiness.yml", change = {}) {
|
||||
const producer = run(file, { status: "completed", conclusion: "success", ...change });
|
||||
const admission = file === "cloud-migrator-artifacts.yml" ? migratorAdmissionJob : sourceAdmissionJob;
|
||||
const f = { runs: { [file]: [producer] }, jobs: { [producer.id]: [job(producer, admission, "success"), job(producer, sourceVerificationJob)] },
|
||||
current: {}, writes: [], calls: [], requested: new Set(), file, sha, admissionJob: admission };
|
||||
f.api = async path => {
|
||||
f.calls.push(path);
|
||||
const url = new URL(`https://api.github.com${path}`);
|
||||
const fileMatch = path.match(/\/workflows\/([^/?]+\.yml)$/);
|
||||
if (fileMatch) return { id: ids[fileMatch[1]], path: `.github/workflows/${fileMatch[1]}` };
|
||||
const listMatch = path.match(/\/workflows\/(\d+)\/runs\?/);
|
||||
if (listMatch) {
|
||||
const name = Object.keys(ids).find(file => ids[file] === Number(listMatch[1]));
|
||||
const runs = f.runs[name] ?? [];
|
||||
return { workflow_runs: runs, total_count: f.total ?? runs.length };
|
||||
}
|
||||
const jobMatch = path.match(/\/runs\/(\d+)\/attempts\/(\d+)\/jobs\?/);
|
||||
if (jobMatch) {
|
||||
const page = Number(url.searchParams.get("page") ?? "1");
|
||||
const jobs = f.jobs[Number(jobMatch[1])] ?? [];
|
||||
return { total_count: f.jobTotal ?? jobs.length, jobs: jobs.slice((page - 1) * 100, page * 100) };
|
||||
}
|
||||
const currentMatch = path.match(/\/runs\/(\d+)$/);
|
||||
if (currentMatch) return f.current[Number(currentMatch[1])] ?? Object.values(f.runs).flat().find(run => run.id === Number(currentMatch[1]));
|
||||
throw new Error(`Unexpected admission API path: ${path}`);
|
||||
};
|
||||
f.rerun = async path => f.writes.push(path);
|
||||
return f;
|
||||
}
|
||||
|
||||
test("only a running image or npm release admits a normal push", async () => {
|
||||
for (const file of ["docker.yml", "release.yml"]) {
|
||||
const f = fixture();
|
||||
assert.equal(await releaseAdmitted(sha, f), false);
|
||||
f.runs[file] = [run(file)];
|
||||
assert.equal(await releaseAdmitted(sha, f), true);
|
||||
assert.equal(await releaseAdmitted(sha, { ...f, imageOnly: true }), file === "docker.yml");
|
||||
for (const status of ["queued", "pending", "completed"]) {
|
||||
f.runs[file] = [run(file, { status })];
|
||||
assert.equal(await releaseAdmitted(sha, f), false, status);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("manual and retry requests admit exact source without consumer discovery", async () => {
|
||||
const api = () => assert.fail("explicit request needs no discovery");
|
||||
assert.equal(await releaseAdmitted(sha, { api, event: "workflow_dispatch" }), true);
|
||||
assert.equal(await releaseAdmitted(sha, { api, attempt: 2 }), true);
|
||||
assert.equal(await releaseAdmitted(sha, { api, event: "pull_request", attempt: 2 }), false);
|
||||
for (const attempt of [0, "2"]) {
|
||||
await assert.rejects(releaseAdmitted(sha, { api, attempt }), /Invalid release attempt/);
|
||||
}
|
||||
});
|
||||
|
||||
test("forks, stale identities, tags, and non-push consumers do not admit work", async () => {
|
||||
for (const change of [{ repository: { ...repository, id: 2 } }, { head_repository: { ...repository, id: 2 } },
|
||||
{ repository: { ...repository, full_name: "fork/paperclip" } }, { path: ".github/workflows/pr.yml" },
|
||||
{ workflow_id: 999 }, { head_sha: "b".repeat(40) }, { head_branch: "v1" }, { event: "workflow_dispatch" },
|
||||
{ id: 0 }, { run_attempt: 0 }]) {
|
||||
const f = fixture();
|
||||
f.runs["docker.yml"] = [run("docker.yml", change)];
|
||||
assert.equal(await releaseAdmitted(sha, f), false, JSON.stringify(change));
|
||||
}
|
||||
const f = fixture();
|
||||
const active = run("docker.yml");
|
||||
f.runs["docker.yml"] = [active];
|
||||
f.current[active.id] = { ...active, run_attempt: 2 };
|
||||
assert.equal(await releaseAdmitted(sha, f), false);
|
||||
});
|
||||
|
||||
test("incomplete discovery and transport failure fail closed", async () => {
|
||||
for (const total of [1, 101, -1]) {
|
||||
const f = fixture();
|
||||
f.total = total;
|
||||
await assert.rejects(releaseAdmitted(sha, f), /Incomplete/);
|
||||
assert.deepEqual(f.writes, []);
|
||||
}
|
||||
await assert.rejects(releaseAdmitted(sha, { api: async () => { throw new Error("API unavailable"); } }), /unavailable/);
|
||||
await assert.rejects(releaseAdmitted("master", { api: () => assert.fail() }), /full source SHA/);
|
||||
});
|
||||
|
||||
test("a selected consumer restarts only a strictly deferred producer, once", async () => {
|
||||
for (const file of ["cloud-readiness.yml", "cloud-migrator-artifacts.yml"]) {
|
||||
const f = fixture(file);
|
||||
assert.equal(await requestDeferredProducer(f), "pending");
|
||||
assert.deepEqual(f.writes, [`/repos/paperclipai/paperclip/actions/runs/${ids[file] + 1000}/rerun`]);
|
||||
await requestDeferredProducer(f);
|
||||
assert.equal(f.writes.length, 1);
|
||||
}
|
||||
});
|
||||
|
||||
test("failed, incomplete, ambiguous, or mismatched work is never automatically retried", async () => {
|
||||
for (const change of [jobs => { jobs[1].conclusion = "failure"; }, jobs => { jobs[0].conclusion = "failure"; },
|
||||
jobs => { jobs[1].run_attempt = 2; }, jobs => { jobs[1].head_sha = "b".repeat(40); },
|
||||
jobs => { jobs[1].run_id = 2; }, jobs => { jobs[1].status = "queued"; },
|
||||
jobs => { jobs.push({ ...jobs[0] }); }, jobs => { jobs.splice(1); }]) {
|
||||
const f = fixture();
|
||||
change(f.jobs[1123]);
|
||||
assert.equal(await requestDeferredProducer(f), "finished");
|
||||
assert.deepEqual(f.writes, []);
|
||||
}
|
||||
const f = fixture();
|
||||
for (const change of [{ run_attempt: 2 }, { status: "in_progress" }, { head_sha: "b".repeat(40) }, { id: 2 }]) {
|
||||
f.current[1123] = { ...f.runs[f.file][0], ...change };
|
||||
assert.equal(await requestDeferredProducer(f), "pending");
|
||||
assert.deepEqual(f.writes, []);
|
||||
}
|
||||
});
|
||||
|
||||
test("an explicitly cancelled or failed producer cannot be restarted by admission", async () => {
|
||||
for (const conclusion of ["cancelled", "failure", "timed_out"]) {
|
||||
const f = fixture("cloud-readiness.yml", { conclusion });
|
||||
assert.equal(await requestDeferredProducer(f), "finished");
|
||||
assert.deepEqual(f.writes, []);
|
||||
}
|
||||
});
|
||||
|
||||
test("a truncated job page cannot hide a failed check and trigger a retry", async () => {
|
||||
const f = fixture();
|
||||
f.jobTotal = 3;
|
||||
await assert.rejects(requestDeferredProducer(f), /Incomplete release producer job list/);
|
||||
assert.deepEqual(f.writes, []);
|
||||
f.jobTotal = "2";
|
||||
await assert.rejects(requestDeferredProducer(f), /Invalid release producer job list/);
|
||||
assert.deepEqual(f.writes, []);
|
||||
});
|
||||
|
||||
test("producer retries reject arbitrary workflow and admission contracts", async () => {
|
||||
const f = fixture();
|
||||
for (const change of [{ file: "docker.yml" }, { admissionJob: migratorAdmissionJob }]) {
|
||||
await assert.rejects(requestDeferredProducer({ ...f, ...change }), /Unknown deferred/);
|
||||
}
|
||||
assert.deepEqual(f.calls, []);
|
||||
});
|
||||
|
||||
test("a deferred completion racing the run listing does not become a source failure", async () => {
|
||||
const f = fixture();
|
||||
const completed = f.runs[f.file][0];
|
||||
f.runs[f.file] = [{ ...completed, status: "in_progress", conclusion: null }];
|
||||
f.current[completed.id] = completed;
|
||||
await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred);
|
||||
delete f.current[completed.id];
|
||||
assert.equal(await readSourceVerification(sha, f.api), undefined);
|
||||
});
|
||||
|
||||
test("a competing consumer's accepted retry does not fail the losing consumer", async () => {
|
||||
const f = fixture();
|
||||
f.rerun = async () => {
|
||||
f.current[1123] = run(f.file, { run_attempt: 2 });
|
||||
throw new Error("duplicate restart rejected");
|
||||
};
|
||||
assert.equal(await requestDeferredProducer(f), "pending");
|
||||
assert.equal(f.requested.size, 1);
|
||||
const rejected = fixture();
|
||||
rejected.rerun = async () => { throw new Error("token denied"); };
|
||||
await assert.rejects(requestDeferredProducer(rejected), /token denied/);
|
||||
assert.equal(rejected.requested.size, 0);
|
||||
});
|
||||
|
||||
test("a queued migrator is pending until its build actually starts", async () => {
|
||||
const f = fixture("cloud-migrator-artifacts.yml", { status: "in_progress", conclusion: null });
|
||||
const producer = f.runs[f.file][0];
|
||||
const build = { ...job(producer, "Build migrator", null), status: "queued", labels: ["cloud-release-32"] };
|
||||
f.jobs[producer.id] = [job(producer, migratorAdmissionJob, "success"), build];
|
||||
assert.equal(await requestDeferredProducer(f), "pending");
|
||||
build.status = "in_progress";
|
||||
assert.equal(await requestDeferredProducer(f), "admitted");
|
||||
build.run_attempt = 2;
|
||||
assert.equal(await requestDeferredProducer(f), "pending", "a stale build cannot admit work");
|
||||
assert.deepEqual(f.writes, []);
|
||||
});
|
||||
|
||||
test("a rejected duplicate with stale read visibility remains pending without claiming admission", async () => {
|
||||
for (const status of [409, 422, 403]) {
|
||||
const f = fixture();
|
||||
f.rerun = async () => { throw Object.assign(new Error("retry rejected"), { status }); };
|
||||
if (status === 403) await assert.rejects(requestDeferredProducer(f), /retry rejected/);
|
||||
else assert.equal(await requestDeferredProducer(f), "pending");
|
||||
assert.equal(f.requested.size, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test("source proof waits for admission, restarts it, and accepts only the new attempt", async () => {
|
||||
const f = fixture();
|
||||
await assert.rejects(readSourceVerification(sha, f.api), SourceVerificationDeferred);
|
||||
let time = 0;
|
||||
const proof = await waitForSourceVerification(sha, { ...f, now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {},
|
||||
sleep: async ms => {
|
||||
time += ms;
|
||||
const selected = run(f.file, { run_attempt: 2 });
|
||||
f.runs[f.file] = [selected];
|
||||
f.jobs[selected.id] = [job(selected, sourceAdmissionJob, "success"), job(selected, sourceVerificationJob, "success")];
|
||||
} });
|
||||
assert.equal(proof.attempt, 2);
|
||||
assert.equal(time, 10);
|
||||
assert.equal(f.writes.length, 1);
|
||||
});
|
||||
|
||||
test("the image monitor admits a late migrator even when source proof is ready", async () => {
|
||||
const f = fixture("cloud-migrator-artifacts.yml");
|
||||
const source = run("cloud-readiness.yml");
|
||||
f.runs["cloud-readiness.yml"] = [source];
|
||||
f.jobs[source.id] = [job(source, sourceVerificationJob, "success")];
|
||||
const image = run("docker.yml");
|
||||
f.runs["docker.yml"] = [image];
|
||||
f.jobs[image.id] = [];
|
||||
let time = 0;
|
||||
const proof = await watchReleaseSource({ ...f, runId: image.id, attempt: 1, cancel: () => assert.fail("must not cancel"),
|
||||
now: () => time, intervalMs: 10, timeoutMs: 30, log: () => {}, summary: async () => {}, sleep: async ms => {
|
||||
time += ms;
|
||||
const selected = run(f.file, { run_attempt: 2 });
|
||||
f.runs[f.file] = [selected];
|
||||
f.jobs[selected.id] = [{ ...job(selected, "Build migrator", null), status: "in_progress", labels: ["cloud-release-32"] }];
|
||||
} });
|
||||
assert.equal(proof.sha, sha);
|
||||
assert.equal(time, 10);
|
||||
assert.equal(f.writes.length, 1);
|
||||
});
|
||||
|
||||
test("the retry transport is a bounded POST to a canonical run", async () => {
|
||||
const calls = [];
|
||||
const rerun = createReleaseRerunner("fixture", async (url, options) => { calls.push({ url, options }); return { ok: true }; });
|
||||
await assert.rejects(rerun("/repos/fork/paperclip/actions/runs/1/rerun"), /Invalid release retry/);
|
||||
await rerun("/repos/paperclipai/paperclip/actions/runs/1/rerun");
|
||||
assert.equal(calls.length, 1);
|
||||
assert.equal(calls[0].options.method, "POST");
|
||||
assert.equal(calls[0].options.redirect, "error");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
import { appendFile } from "node:fs/promises";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { createActionsReader, readSourceVerification, SourceVerificationFailed, SourceVerificationDeferred } from "./cloud-source-verification.mjs";
|
||||
import { requestDeferredProducer, sourceAdmissionJob, migratorAdmissionJob, createReleaseRerunner } from "./cloud-release-admission.mjs";
|
||||
|
||||
const repository = "paperclipai/paperclip";
|
||||
|
||||
export function runnerWaits(jobs, now = Date.now()) {
|
||||
return jobs.filter(job => ["queued", "in_progress", "completed"].includes(job.status) &&
|
||||
job.conclusion !== "skipped" && job.labels?.length > 0).map(job => ({
|
||||
job: job.name, id: job.id,
|
||||
seconds: Math.max(0, Math.floor(((job.runner_id > 0 && job.started_at ? Date.parse(job.started_at) : now) - Date.parse(job.created_at)) / 1000)),
|
||||
})).filter(job => Number.isFinite(job.seconds));
|
||||
}
|
||||
|
||||
// The write is limited to this canonical master Docker run. Never cancel
|
||||
// another workflow, a tag release, or a run selected from untrusted input.
|
||||
export async function cancelImageRun({ api, cancel, runId, attempt, sha }) {
|
||||
const run = await api(`/repos/${repository}/actions/runs/${runId}`);
|
||||
if (run.id !== runId || run.run_attempt !== attempt || run.head_sha !== sha || run.head_branch !== "master" ||
|
||||
run.repository?.id !== 1170821064 || run.head_repository?.id !== 1170821064 ||
|
||||
run.path !== ".github/workflows/docker.yml" || !["push", "workflow_dispatch"].includes(run.event)) {
|
||||
throw new Error("Refusing to cancel a different image workflow identity.");
|
||||
}
|
||||
if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${runId}/cancel`);
|
||||
}
|
||||
|
||||
async function cancelFailedVerification({ api, cancel, sha }, failure) {
|
||||
const run = await api(`/repos/${repository}/actions/runs/${failure.runId}`);
|
||||
// A concurrent retry owns the new attempt; a late observer leaves it alone.
|
||||
if (run.run_attempt !== failure.attempt) return;
|
||||
if (run.id !== failure.runId || run.head_sha !== sha || run.head_branch !== "master" || run.event !== "push" ||
|
||||
run.repository?.full_name !== repository || run.head_repository?.full_name !== repository ||
|
||||
run.path !== ".github/workflows/cloud-readiness.yml") throw new Error("Failed verification identity changed.");
|
||||
if (run.status !== "completed") await cancel(`/repos/${repository}/actions/runs/${failure.runId}/cancel`);
|
||||
}
|
||||
|
||||
export async function watchReleaseSource({ sha, runId, attempt, api, cancel, rerun, now = Date.now,
|
||||
sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), log = console.log,
|
||||
timeoutMs = 40 * 60_000, intervalMs = 30_000, requireProof = false, summary = async () => {} }) {
|
||||
if (!/^[a-f0-9]{40}$/.test(sha ?? "") || !Number.isSafeInteger(runId) || runId < 1 ||
|
||||
!Number.isSafeInteger(attempt) || attempt < 1) throw new Error("Invalid release identity.");
|
||||
const deadline = now() + timeoutMs;
|
||||
const warned = new Set();
|
||||
const requested = new Set();
|
||||
let migratorAdmitted = !rerun;
|
||||
const reportWaits = async jobs => {
|
||||
for (const wait of runnerWaits(jobs, now())) {
|
||||
if (wait.seconds > 60 && !warned.has(wait.id)) {
|
||||
warned.add(wait.id);
|
||||
log(`::warning title=Release runner waited over 60 seconds::Job ${wait.id} waited ${wait.seconds}s for a runner.`);
|
||||
await summary(`Release runner wait: ${wait.job} — ${wait.seconds}s (job ${wait.id}).\n`);
|
||||
}
|
||||
}
|
||||
};
|
||||
while (now() < deadline) {
|
||||
if (!migratorAdmitted) {
|
||||
migratorAdmitted = await requestDeferredProducer({ file: "cloud-migrator-artifacts.yml", sha, admissionJob: migratorAdmissionJob, api, rerun, requested }) !== "pending";
|
||||
}
|
||||
const batch = await api(`/repos/${repository}/actions/runs/${runId}/attempts/${attempt}/jobs?per_page=100`);
|
||||
if (!Array.isArray(batch.jobs) || batch.total_count > 100) throw new Error("Incomplete image job listing.");
|
||||
await reportWaits(batch.jobs);
|
||||
try {
|
||||
const proof = await readSourceVerification(sha, api, { failFast: true, onJobs: reportWaits });
|
||||
if (proof && migratorAdmitted) {
|
||||
await summary(`Exact source verified: ${sha}; readiness run ${proof.runId}, attempt ${proof.attempt}.\n`);
|
||||
return proof;
|
||||
}
|
||||
} catch (error) {
|
||||
if (error instanceof SourceVerificationDeferred && rerun) {
|
||||
await requestDeferredProducer({ file: "cloud-readiness.yml", sha, admissionJob: sourceAdmissionJob, api, rerun, requested });
|
||||
} else {
|
||||
if (error instanceof SourceVerificationFailed) {
|
||||
await cancelFailedVerification({ api, cancel, sha }, error);
|
||||
await cancelImageRun({ api, cancel, runId, attempt, sha });
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
await sleep(Math.min(intervalMs, Math.max(0, deadline - now())));
|
||||
}
|
||||
// This early failure observer is not the publication deadline. Healthy
|
||||
// checks can outlive a Spot worker; a separate exact-source gate still
|
||||
// waits for proof before publishing any image.
|
||||
if (requireProof) throw new Error("Exact-source verification did not become ready before the publication deadline.");
|
||||
await summary("Source monitoring window ended; final publication still requires exact-source verification.\n");
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const token = process.env.GITHUB_TOKEN;
|
||||
if (!token || process.env.GITHUB_REPOSITORY !== repository || process.env.GITHUB_REF !== "refs/heads/master") throw new Error("Canonical master token required.");
|
||||
const requireProof = process.argv.includes("--require-proof");
|
||||
const checkProof = process.argv.includes("--check-proof");
|
||||
const api = createActionsReader({ token });
|
||||
const proof = checkProof ? await readSourceVerification(process.env.GITHUB_SHA, api, { failFast: true }) : await watchReleaseSource({ requireProof, sha: process.env.GITHUB_SHA, runId: Number(process.env.GITHUB_RUN_ID), attempt: Number(process.env.GITHUB_RUN_ATTEMPT),
|
||||
api,
|
||||
rerun: createReleaseRerunner(token),
|
||||
cancel: async path => {
|
||||
const response = await fetch(`https://api.github.com${path}`, { method: "POST", redirect: "error",
|
||||
headers: { Authorization: `Bearer ${token}`, Accept: "application/vnd.github+json" }, signal: AbortSignal.timeout(30_000) });
|
||||
if (!response.ok) throw new Error(`Could not cancel failed-source image work: HTTP ${response.status}.`);
|
||||
},
|
||||
summary: text => process.env.GITHUB_STEP_SUMMARY ? appendFile(process.env.GITHUB_STEP_SUMMARY, text) : Promise.resolve(),
|
||||
});
|
||||
if (checkProof && !proof) throw new Error("Exact-source proof changed before publication.");
|
||||
if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `verified=${Boolean(proof)}\n`);
|
||||
}
|
||||
Reference in new issue
Block a user