diff --git a/.github/workflows/merge-queue-checks.yml b/.github/workflows/merge-queue-checks.yml new file mode 100644 index 00000000000..1e8bfb68498 --- /dev/null +++ b/.github/workflows/merge-queue-checks.yml @@ -0,0 +1,41 @@ +name: Merge queue checks + +# The master ruleset requires `ci / verify` and `ci / e2e`. pr.yml reports them +# for a pull request, and a pull request enters the merge queue only after both +# pass there. The queue then waits for the same two names on its merge-group +# commit, which no pull_request workflow can report. These jobs report them, so +# the queue does not stall. +# +# They run no tests on purpose: a queued merge gets the protection a direct +# merge has today, and the exact-source gate still tests every master commit +# before its image is published. To test the combined tree before it lands, +# replace these jobs with real ones that keep the same names. +# +# Nothing here reads the merge-group commit: no checkout, no secrets, no token +# permissions, GitHub-hosted runners. +# +# Inert until the merge queue is enabled for master: no merge_group event fires +# before then. +on: + merge_group: + +permissions: {} + +concurrency: + group: merge-queue-checks-${{ github.ref }} + cancel-in-progress: true + +jobs: + verify: + name: ci / verify + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - run: echo "ci / verify passed on every pull request in this merge group before it was queued." + + e2e: + name: ci / e2e + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - run: echo "ci / e2e passed on every pull request in this merge group before it was queued." diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 513b9bdc52a..94672521794 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -251,6 +251,16 @@ jobs: - name: Test release verify workflow wiring run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/cloud-readiness-runner-loss.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs + - name: Test merge-queue workflow wiring + # This workflow loads from master and tests the caller's checkout. A + # rerun of an older pull request has no copy of these tests. + run: | + tests=() + for test in ./scripts/merge-group-image.test.mjs ./scripts/__tests__/merge-queue-checks-workflow.test.mjs; do + if [[ -f "$test" ]]; then tests+=("$test"); fi + done + if (( ${#tests[@]} > 0 )); then node "$CI_NODE_TEST" "${tests[@]}"; fi + - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/doc/RELEASING.md b/doc/RELEASING.md index 16fcb0f8b46..ac2b1a02405 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -388,6 +388,22 @@ Without a matching successful merge-group run, or on any mismatch or error, the push builds from source. The PID 1 probe, exact-source gate and signing apply on both paths. +The `master` ruleset requires `ci / verify` and `ci / e2e`, which `pr.yml` +reports only for pull requests. +[`merge-queue-checks.yml`](../.github/workflows/merge-queue-checks.yml) reports +both names on each merge-group commit so the queue does not stall. It runs no +tests: a pull request enters the queue only after both checks pass on it, and +the exact-source gate still tests every `master` commit before publication. +The queue can therefore land a commit while its image is still building. The +`master` push waits up to 15 minutes for that build before it builds from +source. Publication waits for the exact-source gate in any case. + +To turn the queue on, add **Require merge queue** to the `master` ruleset with +merge method Squash, build concurrency 5, group size 1 to 5 and a 60 minute +status check timeout. After the first queued merge, confirm that the `master` +commit SHA equals the SHA of its merge-group runs; promotion depends on it. +To turn it off, remove the rule. `master` then builds from source again. + Canary dispatches first look up the full-SHA master image. Reuse requires a valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching registry digest, and signed provenance from this repository's `docker.yml` diff --git a/scripts/__tests__/merge-queue-checks-workflow.test.mjs b/scripts/__tests__/merge-queue-checks-workflow.test.mjs new file mode 100644 index 00000000000..f2ec7f27e9c --- /dev/null +++ b/scripts/__tests__/merge-queue-checks-workflow.test.mjs @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +const workflows = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".github", "workflows"); +const workflow = readFileSync(path.join(workflows, "merge-queue-checks.yml"), "utf8"); +const jobNames = [...workflow.matchAll(/^ name: (.+)$/gm)].map(match => match[1]); + +test("the merge queue reports the check names that pull requests report", () => { + const caller = readFileSync(path.join(workflows, "pr.yml"), "utf8"); + const trusted = readFileSync(path.join(workflows, "pr-trusted.yml"), "utf8"); + assert.match(caller, /^ ci:\n(?: {4}.*\n)*? {4}uses: .+\/pr-trusted\.yml@/m, "pr.yml reports its checks under the ci job"); + for (const job of ["verify", "e2e"]) { + assert.match(trusted, new RegExp(`^ ${job}:\\n name: ${job}$`, "m"), `pull requests report ci / ${job}`); + } + assert.deepEqual(jobNames, ["ci / verify", "ci / e2e"]); +}); + +test("the merge queue checks cannot read the queued commit or any credential", () => { + const triggers = workflow.split("\non:\n")[1].split("\n\n")[0]; + assert.equal(triggers.trim(), "merge_group:", "only the merge queue triggers these checks"); + assert.match(workflow, /^permissions: \{\}$/m); + assert.doesNotMatch(workflow, /^ +permissions:/m, "no job widens the empty token"); + assert.doesNotMatch(workflow, /secrets\.|uses:|environment:|id-token|runs-on\//); + assert.deepEqual([...workflow.matchAll(/^ runs-on: (.+)$/gm)].map(match => match[1]), ["ubuntu-latest", "ubuntu-latest"]); +}); diff --git a/scripts/merge-group-image.mjs b/scripts/merge-group-image.mjs index b22e311a468..19f8be48249 100644 --- a/scripts/merge-group-image.mjs +++ b/scripts/merge-group-image.mjs @@ -21,6 +21,10 @@ export const artifactName = arch => `merge-group-image-${arch}`; const architectures = ["amd64", "arm64"]; const indexTypes = ["application/vnd.oci.image.index.v1+json", "application/vnd.docker.distribution.manifest.list.v2+json"]; const manifestTypes = ["application/vnd.oci.image.manifest.v1+json", "application/vnd.docker.distribution.manifest.v2+json"]; +// How long a master push waits for a merge-queue build that is still running. +// A build takes about 12 minutes and the exact-source gate about 14. +const findWaitMs = 15 * 60_000; +const findPollMs = 20_000; const fullSha = /^[a-f0-9]{40}$/; const digestPattern = /^sha256:[a-f0-9]{64}$/; const sha256 = bytes => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; @@ -43,6 +47,11 @@ export function selectVerifiedRun({ sha, runs }) { typeof run.head_branch === "string" && run.head_branch.startsWith("gh-readonly-queue/master/")); if (eligible.length === 0) return { reason: "no merge-queue image build exists for this commit" }; const passed = eligible.filter(run => run.status === "completed" && run.conclusion === "success").sort((a, b) => b.id - a.id); + // The queue can land a commit while its image is still building. `pending` + // tells the caller that waiting can still produce a usable run. + if (passed.length === 0 && eligible.some(run => run.status !== "completed")) { + return { pending: true, reason: "the merge-queue image build for this commit did not finish in time" }; + } if (passed.length === 0) return { reason: "the merge-queue image build for this commit did not succeed" }; return { runId: passed[0].id }; } @@ -163,8 +172,18 @@ async function main(command) { assert.ok(response.ok, `GitHub API ${path}: HTTP ${response.status}`); return response.json(); }; - const { workflow_runs: runs } = await api(`/actions/workflows/docker-merge-group.yml/runs?event=merge_group&head_sha=${sha}&per_page=20`); - const run = selectVerifiedRun({ sha, runs }); + // Publication waits for the exact-source gate in any case, so waiting here + // for an unfinished build costs nothing while the build is the shorter of + // the two. Past the deadline the push builds from source. + const deadline = Date.now() + findWaitMs; + let run; + for (;;) { + const { workflow_runs: runs } = await api(`/actions/workflows/docker-merge-group.yml/runs?event=merge_group&head_sha=${sha}&per_page=20`); + run = selectVerifiedRun({ sha, runs }); + if (!run.pending || Date.now() >= deadline) break; + console.log("The merge-queue image build for this commit is still running. Waiting."); + await new Promise(resolve => setTimeout(resolve, findPollMs)); + } if (!run.runId) { console.log(`No promotion: ${run.reason}. Building from source.`); return; } const { artifacts } = await api(`/actions/runs/${run.runId}/artifacts?per_page=100`); const artifact = selectArtifact({ artifacts, arch }); diff --git a/scripts/merge-group-image.test.mjs b/scripts/merge-group-image.test.mjs index 5654c65e415..c582abefae6 100644 --- a/scripts/merge-group-image.test.mjs +++ b/scripts/merge-group-image.test.mjs @@ -30,8 +30,17 @@ test("promotes only a successful merge-queue build of this repository for this e ["a queue for another branch", run({ head_branch: "gh-readonly-queue/release/pr-1-x" })], ["an ordinary branch", run({ head_branch: "master" })], ]) assert.match(selectVerifiedRun({ sha, runs: [candidate] }).reason, /no merge-queue image build/, label); - for (const candidate of [run({ conclusion: "failure" }), run({ conclusion: "cancelled" }), run({ status: "in_progress", conclusion: null })]) { - assert.match(selectVerifiedRun({ sha, runs: [candidate] }).reason, /did not succeed/); + for (const candidate of [run({ conclusion: "failure" }), run({ conclusion: "cancelled" })]) { + assert.deepEqual(selectVerifiedRun({ sha, runs: [candidate] }), { reason: "the merge-queue image build for this commit did not succeed" }); + } + // An unfinished build is worth waiting for, even beside a failed attempt. A + // finished one is used at once, and an unfinished run of another commit is not. + for (const status of ["queued", "in_progress"]) { + const unfinished = run({ id: 11, status, conclusion: null }); + assert.equal(selectVerifiedRun({ sha, runs: [unfinished] }).pending, true); + assert.equal(selectVerifiedRun({ sha, runs: [run({ conclusion: "failure" }), unfinished] }).pending, true); + assert.deepEqual(selectVerifiedRun({ sha, runs: [run(), unfinished] }), { runId: 10 }); + assert.equal(selectVerifiedRun({ sha, runs: [run({ id: 11, status, conclusion: null, head_sha: "b".repeat(40) })] }).pending, undefined); } assert.throws(() => selectVerifiedRun({ sha: "master", runs: [] })); });