diff --git a/.github/scripts/tests/docker-canary-promotion.test.mjs b/.github/scripts/tests/docker-canary-promotion.test.mjs index a9b0727465f..fe80e25168b 100644 --- a/.github/scripts/tests/docker-canary-promotion.test.mjs +++ b/.github/scripts/tests/docker-canary-promotion.test.mjs @@ -11,7 +11,7 @@ const script = job.split(" run: |\n")[1].split("\n").map(line => line.rep const sha = "a".repeat(40); test("canary promotion waits for the standard manifest and keeps its serialized channel", () => { - assert.match(job, /needs: \[merge-and-push\]/); + assert.match(job, /needs: \[reuse_canary_image, merge-and-push\]/); assert.match(job, /group: docker-canary-channel-promotion/); assert.match(job, /cancel-in-progress: false/); }); diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index ed52dfa700d..ec7c71000e5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,6 +33,35 @@ concurrency: cancel-in-progress: false jobs: + reuse_canary_image: + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && startsWith(github.ref, 'refs/tags/canary/v') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + packages: write + attestations: read + outputs: + reused: ${{ steps.reuse.outputs.reused }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + - name: Login to GitHub Container Registry + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Reuse the verified master image when present + id: reuse + env: + GH_TOKEN: ${{ github.token }} + run: node scripts/reuse-canary-image.mjs + # Multi-arch by native runner, not QEMU. # # This was one job building linux/amd64,linux/arm64 together on an x86 @@ -56,6 +85,8 @@ jobs: # digest; `merge` assembles the manifest list. Both architectures remain # available to self-hosted installations and downstream image composers. build-and-push: + needs: [reuse_canary_image] + if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} strategy: # Independent legs: one architecture failing should still publish # nothing, but it must not also hide the other's logs behind a @@ -277,6 +308,7 @@ jobs: # tags. Nothing is publicly tagged until this runs, so a half-published # multi-arch image is not a state anything can pull. merge-and-push: + if: ${{ !cancelled() && needs.build-and-push.result == 'success' }} needs: build-and-push runs-on: ubuntu-latest timeout-minutes: 20 @@ -407,11 +439,8 @@ jobs: # promotion is gated on its own completed pushes). Every interleaving # converges the Docker channel onto the npm channel. promote_canary_channel: - if: startsWith(github.ref, 'refs/tags/canary/v') - # merge-and-push, not build-and-push: the per-arch legs push untagged - # digests, and the production `sha-*` tags this promotion retags only - # exist once the manifest merge has named them. - needs: [merge-and-push] + if: ${{ !cancelled() && startsWith(github.ref, 'refs/tags/canary/v') && (needs.merge-and-push.result == 'success' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'true')) }} + needs: [reuse_canary_image, merge-and-push] runs-on: ubuntu-latest timeout-minutes: 10 permissions: diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b1b73c1f085..dae203d6216 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -202,7 +202,7 @@ jobs: run: node "$CI_NODE_TEST" ./scripts/__tests__/e2e-shard.test.mjs - name: Test release verify workflow wiring - run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs + run: node "$CI_NODE_TEST" ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/__tests__/eval-schedule-monitor.test.mjs ./scripts/__tests__/prepare-runner-release-image.test.mjs ./scripts/cloud-source-verification.test.mjs ./scripts/standard-image-contract.test.mjs ./scripts/reuse-canary-image.test.mjs - name: Test standalone package build concurrency run: node "$CI_NODE_TEST" ./scripts/__tests__/build-standalone-concurrency.test.mjs diff --git a/doc/RELEASING.md b/doc/RELEASING.md index dbf40956a50..70e28fe798f 100644 --- a/doc/RELEASING.md +++ b/doc/RELEASING.md @@ -86,7 +86,7 @@ internal traceability tag. moves them to the versioned name - canaries, nightlies, and betas never create GitHub Releases - canaries, nightlies, and betas never require changelog generation -- Docker `:latest` moves only on stable releases; master builds publish +- Docker `:latest` moves only on stable releases; canary dispatches promote `:canary`, nightly builds `:nightly`, and beta builds `:beta` ## TL;DR @@ -310,12 +310,12 @@ The workflow: ## Docker Image Tags -[`docker.yml`](../.github/workflows/docker.yml) publishes both the self-hosted -image and the `-cloud` variant with the same lane mapping: +[`docker.yml`](../.github/workflows/docker.yml) publishes the standard image: | Build ref | Tags | | --- | --- | -| `master` push | `:canary`, `:sha-` | +| `master` push | `:sha-`, `:sha-` | +| `canary/v*` dispatch | `:sha-`, then `:canary` for the current npm canary | | `nightly/v*` tag | `:nightly`, `:sha-` | | `beta/v*` tag | `:beta`, `:sha-` | | `v*` tag (stable) | `:latest`, `:YYYY.MDD.P`, `:YYYY.MDD`, `:sha-` | @@ -325,6 +325,21 @@ suppresses push-triggered workflow runs for those pushes. The release jobs therefore dispatch `docker.yml` explicitly at the new tag ref; the tag mapping keys off `github.ref` either way. +Canary dispatches first look up the full-SHA master image. Reuse requires a +valid multi-platform index with both `linux/amd64` and `linux/arm64`, a matching +registry digest, and signed provenance from this repository's `docker.yml` +on `master` at the exact source commit. The image is retagged by immutable +digest. This preserves the original build inputs and skips both native builds +and the manifest merge. A missing full-SHA image runs the normal build instead; +registry errors or invalid provenance fail the run. Rerun after the master +image's attestation finishes if the image became visible before its proof. + +Both paths retain the serialized promotion that resolves the current npm +`canary` dist-tag before moving Docker `:canary`. Nightly, beta, stable, and +master builds retain their existing build paths. Downstream image builds can +continue to consume the signed full-SHA master image independently of npm +canary publication. + The Docker build matrix and the release preview image builder configure BuildKit to check `mirror.gcr.io` for Docker Hub images first. This covers both the Dockerfile frontend and base images. BuildKit keeps Docker Hub as diff --git a/scripts/reuse-canary-image.mjs b/scripts/reuse-canary-image.mjs new file mode 100644 index 00000000000..3a43524dd0b --- /dev/null +++ b/scripts/reuse-canary-image.mjs @@ -0,0 +1,34 @@ +import assert from "node:assert/strict"; +import { execFileSync } from "node:child_process"; +import { appendFile } from "node:fs/promises"; +import { pathToFileURL } from "node:url"; +import { resolveStandardImageDigest } from "./standard-image-contract.mjs"; + +export async function reuseCanaryImage(sha, { + resolve = source => resolveStandardImageDigest(source, fetch, { missing: "return-null" }), + run = (command, args) => execFileSync(command, args, { stdio: "inherit" }), +} = {}) { + assert.match(sha ?? "", /^[a-f0-9]{40}$/); + const digest = await resolve(sha); + if (digest === null) return false; + assert.match(digest, /^sha256:[a-f0-9]{64}$/); + const image = `ghcr.io/paperclipai/paperclip@${digest}`; + await run("gh", ["attestation", "verify", `oci://${image}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]); + await run("docker", ["buildx", "imagetools", "create", "-t", + `ghcr.io/paperclipai/paperclip:sha-${sha.slice(0, 7)}`, image]); + return true; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const reused = await reuseCanaryImage(process.env.GITHUB_SHA); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `reused=${reused}\n`); + const summary = reused + ? "Reused the signed multi-platform master image. Skipped both image builds and the manifest merge." + : "No full-SHA master image exists. Building both platforms for this canary."; + console.log(summary); + if (process.env.GITHUB_STEP_SUMMARY) await appendFile(process.env.GITHUB_STEP_SUMMARY, `${summary}\n`); +} diff --git a/scripts/reuse-canary-image.test.mjs b/scripts/reuse-canary-image.test.mjs new file mode 100644 index 00000000000..26dfbc83c5f --- /dev/null +++ b/scripts/reuse-canary-image.test.mjs @@ -0,0 +1,75 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { reuseCanaryImage } from "./reuse-canary-image.mjs"; + +const sha = "a".repeat(40); +const digest = `sha256:${"b".repeat(64)}`; + +test("reuses only an exact-source, master-signed multi-platform image by digest", async () => { + const calls = []; + const reused = await reuseCanaryImage(sha, { + resolve: async source => { assert.equal(source, sha); return digest; }, + run: async (...args) => calls.push(args), + }); + assert.equal(reused, true); + assert.deepEqual(calls, [ + ["gh", ["attestation", "verify", `oci://ghcr.io/paperclipai/paperclip@${digest}`, + "--repo", "paperclipai/paperclip", "--bundle-from-oci", + "--signer-workflow", "paperclipai/paperclip/.github/workflows/docker.yml", + "--source-ref", "refs/heads/master", "--source-digest", sha, + "--signer-digest", sha, "--deny-self-hosted-runners"]], + ["docker", ["buildx", "imagetools", "create", "-t", + "ghcr.io/paperclipai/paperclip:sha-aaaaaaa", `ghcr.io/paperclipai/paperclip@${digest}`]], + ]); +}); + +test("a missing master image enables the fallback without retagging", async () => { + assert.equal(await reuseCanaryImage(sha, { + resolve: async () => null, + run: () => assert.fail("A missing image must not be promoted"), + }), false); +}); + +test("registry and provenance failures stop promotion rather than bypass verification", async () => { + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => { throw new Error("HTTP 403"); }, + run: () => assert.fail("A registry failure must not be promoted"), + }), /HTTP 403/); + let commands = 0; + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => digest, + run: command => { + assert.equal(command, "gh"); + commands++; + throw new Error("Untrusted source or signature"); + }, + }), /Untrusted source/); + assert.equal(commands, 1); +}); + +test("invalid source refs and digests cannot reach external commands", async () => { + for (const invalid of ["master", "a".repeat(7), "a".repeat(40) + "\n"]) { + await assert.rejects(reuseCanaryImage(invalid, { + resolve: () => assert.fail("Invalid source accepted"), + })); + } + await assert.rejects(reuseCanaryImage(sha, { + resolve: async () => "mutable:tag", + run: () => assert.fail("Invalid digest accepted"), + })); +}); + +test("canary reuse gates both native builds and still runs convergent promotion", () => { + const workflow = readFileSync(new URL("../.github/workflows/docker.yml", import.meta.url), "utf8"); + const reuse = workflow.split(" reuse_canary_image:\n")[1].split(" build-and-push:\n")[0]; + assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/); + assert.match(reuse, /node scripts\/reuse-canary-image.mjs/); + const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0]; + assert.match(build, /needs: \[reuse_canary_image\]/); + assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/); + assert.match(build, /platform: linux\/amd64/); + assert.match(build, /platform: linux\/arm64/); + const promotion = workflow.split(" promote_canary_channel:\n")[1]; + assert.match(promotion, /!cancelled\(\).*needs.merge-and-push.result == 'success'.*outputs.reused == 'true'/); +}); diff --git a/scripts/standard-image-contract.mjs b/scripts/standard-image-contract.mjs index 62a8249e2da..5e0ae3cc24c 100644 --- a/scripts/standard-image-contract.mjs +++ b/scripts/standard-image-contract.mjs @@ -21,6 +21,7 @@ export function standardImageDigest(bytes) { /** Read registry bytes directly; CLI stdout formatting is not artifact content. */ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { sleep = ms => new Promise(resolve => setTimeout(resolve, ms)), + missing = "error", } = {}) { assert.match(sha ?? "", /^[a-f0-9]{40}$/); const request = async (url, headers, consume, allowNotFound = false) => { @@ -35,6 +36,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } if (!response.ok) { await response.body?.cancel(); + if (allowNotFound && response.status === 404 && missing === "return-null") return null; const transient = [408, 425, 429].includes(response.status) || response.status >= 500 || (allowNotFound && response.status === 404); assert.ok(transient && attempt < 5, `Registry lookup failed: HTTP ${response.status}`); const retryAfter = Number(response.headers.get("retry-after")); @@ -53,7 +55,7 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { }; const { token } = await request("https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip:pull", {}, response => response.json()); assert.ok(typeof token === "string" && token, "Missing public pull token"); - const { raw, expected } = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { + const manifest = await request(`https://ghcr.io/v2/paperclipai/paperclip/manifests/sha-${sha}`, { authorization: `Bearer ${token}`, accept: "application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json", }, async response => { assert.ok(response.body, "Standard image index has no body"); @@ -71,6 +73,8 @@ export async function resolveStandardImageDigest(sha, fetchImpl = fetch, { } finally { await reader.cancel(); } return { raw: Buffer.concat(chunks), expected: response.headers.get("docker-content-digest") }; }, true); + if (manifest === null) return null; + const { raw, expected } = manifest; const digest = standardImageDigest(raw); assert.equal(digest, expected, "Registry digest does not match the index bytes"); return digest; diff --git a/scripts/standard-image-contract.test.mjs b/scripts/standard-image-contract.test.mjs index af2d4462977..d61f3fb862d 100644 --- a/scripts/standard-image-contract.test.mjs +++ b/scripts/standard-image-contract.test.mjs @@ -77,3 +77,17 @@ test("permanent registry errors fail immediately and transient errors exhaust a assert.equal(calls, expectedCalls); } }); + +test("optional lookup falls back only for a missing manifest", async () => { + for (const status of [401, 403, 404, 503]) { + let calls = 0; + const lookup = resolveStandardImageDigest("a".repeat(40), async url => { + if (url.includes("/token?")) return Response.json({ token: "public-fixture" }); + calls++; + return new Response(null, { status }); + }, { missing: "return-null", sleep: async () => {} }); + if (status === 404) assert.equal(await lookup, null); + else await assert.rejects(lookup, /Registry lookup failed/); + assert.equal(calls, status === 503 ? 5 : 1); + } +});