diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e61d8ad3077..eb78a7e3d23 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -33,14 +33,21 @@ concurrency: cancel-in-progress: false jobs: - runner_image_report: - name: Report the Runner image of this commit - if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' + runner_image: + name: Resolve the Runner image of this commit + if: vars.RUNNER_IMAGE_CONSUME_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' continue-on-error: true runs-on: ubuntu-latest - timeout-minutes: 8 + timeout-minutes: 70 permissions: contents: read + actions: read + attestations: read + outputs: + mode: ${{ steps.verify.outputs.mode || 'inline' }} + build_contexts: ${{ steps.verify.outputs.build_contexts }} + labels: ${{ steps.verify.outputs.labels }} + build_args: ${{ steps.verify.outputs.build_args }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -70,12 +77,48 @@ jobs: timeout-minutes: 5 run: node scripts/runner-image-resolve.mjs inputs --dir "$RUNNER_TEMP/runner-image-resolve" - - name: Report the Runner an image build would use - timeout-minutes: 2 + - name: Find the published Runner image of these inputs + id: resolve + timeout-minutes: 55 + env: + WAIT: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }} + GITHUB_TOKEN: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.token || '' }} run: | set -euo pipefail - node scripts/runner-image-resolve.mjs resolve --dir "$RUNNER_TEMP/runner-image-resolve" - echo "Report only: no build read this result." >> "$GITHUB_STEP_SUMMARY" + if [ "$WAIT" = true ]; then + node scripts/runner-image-resolve.mjs resolve --dir "$RUNNER_TEMP/runner-image-resolve" --wait + else + node scripts/runner-image-resolve.mjs resolve --dir "$RUNNER_TEMP/runner-image-resolve" + fi + + - name: Verify the attestation of the Runner image + id: verify + if: steps.resolve.outputs.mode == 'published' + timeout-minutes: 5 + env: + GH_TOKEN: ${{ github.token }} + IMAGE: ${{ steps.resolve.outputs.image }} + LABELS: ${{ steps.resolve.outputs.labels }} + BUILD_ARGS: ${{ steps.resolve.outputs.build_args }} + run: | + set -euo pipefail + if ! gh attestation verify "oci://$IMAGE" \ + --repo paperclipai/paperclip \ + --signer-workflow paperclipai/paperclip/.github/workflows/runner-image.yml \ + --source-ref refs/heads/master; then + echo "::warning title=Runner image not verified::The attestation of $IMAGE did not verify. The image build makes its own Runner." + echo "The attestation of \`$IMAGE\` did not verify: the image build makes its own Runner." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + { + echo "mode=published" + echo "build_contexts=runner-artifacts=docker-image://$IMAGE" + echo "build_args=$BUILD_ARGS" + echo "labels<> "$GITHUB_OUTPUT" + echo "The attestation of \`$IMAGE\` verified: the image build copies this Runner." >> "$GITHUB_STEP_SUMMARY" source_verification: name: Watch exact-source verification @@ -153,7 +196,7 @@ jobs: # digest; `merge` assembles the manifest list. Both architectures remain # available to self-hosted installations and downstream image composers. build-and-push: - needs: [reuse_canary_image] + needs: [reuse_canary_image, runner_image] if: ${{ !cancelled() && (needs.reuse_canary_image.result == 'skipped' || (needs.reuse_canary_image.result == 'success' && needs.reuse_canary_image.outputs.reused == 'false')) }} strategy: # Independent legs: one architecture failing should still publish @@ -235,6 +278,7 @@ jobs: node-version: 24 - name: Refresh lockfile for Docker build context + if: needs.runner_image.outputs.mode != 'published' run: | set -euo pipefail pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile @@ -286,7 +330,7 @@ jobs: # Build-before-merge. When the merge queue already built and probed this # exact commit (docker-merge-group.yml, with no registry credentials), - # push those bytes instead of rebuilding. scripts/merge-group-image.mjs + # push those bytes, unless a published Runner image is used. The script # requires a successful merge-group run of this repository for this SHA # and checks the record's commit, tree, architecture and archive digest. # Any missing or mismatched proof, or any failure here, leaves @@ -294,7 +338,7 @@ jobs: # before. The probe below and the exact-source gate apply either way. - name: Find the merge-queue build of this commit id: find - if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' + if: needs.runner_image.outputs.mode != 'published' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' continue-on-error: true env: GITHUB_TOKEN: ${{ github.token }} @@ -413,6 +457,8 @@ jobs: PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }} PAPERCLIP_BUILD_COMMIT=${{ github.sha }} CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }} + ${{ needs.runner_image.outputs.build_args }} + build-contexts: ${{ needs.runner_image.outputs.build_contexts }} platforms: ${{ matrix.platform }} # By digest, not by tag: two runners cannot each push the same tag # and end up with a manifest list. Each leg publishes an untagged @@ -428,7 +474,9 @@ jobs: # have each leg overwrite the other's cache on every build. cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }} cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=max - labels: ${{ steps.meta.outputs.labels }} + labels: | + ${{ steps.meta.outputs.labels }} + ${{ needs.runner_image.outputs.labels }} # Exactly one of the two paths produced this leg's digest. - name: Select image digest diff --git a/doc/DOCKER.md b/doc/DOCKER.md index cc46a428ffb..81af60d3b6e 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -92,7 +92,9 @@ identity rules and how to consume the image. The application image takes its Runner from the `runner-artifacts` stage. It has no Runner build of its own, no Rust toolchain and no C compiler. Every -build of the `production` or `cloud` target builds that stage itself, once: +build of the `production` or `cloud` target builds that stage itself, once, +unless it is given a published Runner image in its place (see "The Runner of +the application image" below): - The `build` stage copies the Runner's build output and its build stamp into `packages/paperclip-runner`. It sets `PAPERCLIP_RUNNER_VENDOR_REQUIRE_CURRENT=1`, @@ -128,6 +130,29 @@ lockfile in the checkout, then build again. Do not commit the result (see pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile ``` +### The Runner of the application image + +An application image either makes its own Runner in its build (inline) or +copies the published Runner image of the same inputs. A local build, a fork, +a pull request and the merge-queue build are always inline. +`.github/workflows/docker.yml` in this repository copies the published image +when the repository variable `RUNNER_IMAGE_CONSUME_ENABLED` is exactly `true` +and the image of the commit's inputs is published and its attestation +verifies. On a `master` push it waits for the commit's own run of +`runner-image.yml`, at most 40 minutes from the push. In every other case the +build is inline, with a notice: no state of the Runner image fails an +application image build. A build that copies the published image builds the +committed `pnpm-lock.yaml`: the workflow does not refresh it first, and the +`deps` stage resolves the other packages' manifests inside the image. + +An image that copied the published Runner has the labels +`io.paperclip.runner.image` (`ghcr.io/paperclipai/paperclip-runner@sha256:`) +and `io.paperclip.runner.inputs-digest`. An image without them made its own +Runner and is as complete. To go back to inline builds everywhere, set the +variable to another value or delete it; nothing needs a revert. See "The +Runner of the application image" in +[`doc/runner-image-contract.md`](runner-image-contract.md). + `.github/workflows/docker.yml` builds the application image on `master` and on tags only. `.github/workflows/docker-app-check.yml` builds the `build` and `runner-provider-pack` stages on a pull request that changes the `Dockerfile`, diff --git a/doc/runner-image-contract.md b/doc/runner-image-contract.md index a6ab0fea197..373b4293ff1 100644 --- a/doc/runner-image-contract.md +++ b/doc/runner-image-contract.md @@ -4,7 +4,8 @@ The Runner image is the Runner built alone, as a plain file tree with no entrypoint and no operating system. An image build copies paths out of it instead of compiling the Runner again. The application image of this repository takes its Runner from the stage that builds it (see "The -application image"); no build reads the published image yet. Its name is +application image"), or copies the published image when a repository variable +allows it (see "The Runner of the application image"). Its name is `ghcr.io/paperclipai/paperclip-runner`, and one build makes two variants: - full, the `runner-artifacts` target of the `Dockerfile`: every path below; @@ -180,9 +181,10 @@ The `production` and `cloud` targets of the `Dockerfile` read only the `COPY --from=runner-provider-pack /provider-pack-transport/ /opt/paperclip-runner/` in the image history (`retained-provider-pack-oci.ts`). Do not change it. -No workflow replaces the stage yet, so every application image build runs -the Runner stages itself: an inline Runner. An inline Runner is not this -image, and an application image says so in three ways: +Unless a workflow replaces the stage (see "The Runner of the application +image"), an application image build runs the Runner stages itself: an inline +Runner. An inline Runner is not this image, and an application image says so +in three ways: - its pack names the build commit of the application image as `payload.runnerSourceRevision`, the value of `PAPERCLIP_BUILD_COMMIT`; @@ -192,8 +194,9 @@ image, and an application image says so in three ways: A build without `PAPERCLIP_BUILD_COMMIT` has no pack, as before. `build` also declares the build argument -`PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST`, which no build sets yet. It is for -a build that replaces the stage with a published image. When the argument is +`PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST`. It is for a build that replaces the +stage with a published image, and `docker.yml` sets it together with the +build context. When the argument is set, `scripts/check-runner-image-expectation.mjs` reads `/runner-image.json` of the `runner-artifacts` the build was given, and the build fails unless that file has `publication: true` and names exactly those inputs. An inline @@ -203,12 +206,11 @@ The check runs before the server build vendors the Runner. ### Choosing the Runner for an image build `scripts/runner-image-resolve.mjs` decides whether a build of a checkout can -copy the published image. No build uses its answer yet: on a `master` push, -the `runner_image_report` job of `.github/workflows/docker.yml` runs it and -writes the answer to the job summary, and nothing reads that job. The job -runs on the committed files, holds no credential (the Runner package is -public, so the lookup is anonymous) and cannot fail the run. The script has -two commands, so that the one that builds reads no registry credential: +copy the published image. The `runner_image` job of +`.github/workflows/docker.yml` runs it for the application image; see "The +Runner of the application image". The Runner package is public, so the lookup +needs no credential. The script has two commands, so that the one that builds +reads no credential: ```sh node scripts/runner-image-resolve.mjs inputs --dir "$dir" @@ -276,6 +278,94 @@ no wait. last time. Without an image the answer is `inline`, with the reason of that lookup and what the wait saw. +### The Runner of the application image + +The application image, `ghcr.io/paperclipai/paperclip`, gets its Runner in one +of two ways. The recipe is the same in both, so the Runner behaves the same. + +- **Inline.** The image build runs the `runner-artifacts` stage itself. +- **Published.** The build receives + `--build-context runner-artifacts=docker-image://ghcr.io/paperclipai/paperclip-runner@sha256:`. + The stage is then not built, and the image copies the published Runner. + +Only `.github/workflows/docker.yml` in this repository can choose +"published". A local `docker build`, a fork, the pull-request check +(`docker-app-check.yml`) and the merge-queue build +(`docker-merge-group.yml`) are always inline. + +**Labels.** An image that copied the published Runner has two labels, on the +image of each platform: + +| Label | Value | +| --- | --- | +| `io.paperclip.runner.image` | `ghcr.io/paperclipai/paperclip-runner@sha256:` | +| `io.paperclip.runner.inputs-digest` | The inputs digest of that Runner image. | + +The workflow writes them, not the `Dockerfile`, and only together with the +build context. An image without them made its own Runner. Nothing else +follows from their absence: such an image is complete and supported. Its pack +has no `payload.runnerInputsDigest`, and the pack's `runnerSourceRevision` is +the commit of the image. In an image with the labels, `runnerSourceRevision` +is the commit that published the Runner image, which can be older than the +commit of the application image. + +**The variable.** The repository variable `RUNNER_IMAGE_CONSUME_ENABLED` +turns this on. Only the exact value `true` does. With any other value, or no +variable, the `runner_image` job does not run and every build is inline, as +before the job existed. + +**The job.** `runner_image` runs once for a workflow run, and both +architecture builds need it, so both always get the same answer. + +1. `inputs`, on the committed files, with no credential. +2. `resolve`. For a `master` push it runs with `--wait` and a token that can + read workflow runs. For a tag or a manual run it does not wait: no run of + `runner-image.yml` belongs to that event. +3. For a `published` answer, the job verifies the attestation of the full + index, with the command under "Create-only". Only this step writes the + outputs of the job, so no build receives a digest that did not verify. An + image that does not verify fails this step with a warning. The job is then + red, the run is not, and the builds are inline. +4. The builds add the outputs to their own values: the build context, the two + labels and the build argument `PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST`. With + empty outputs a build has exactly the arguments, contexts and labels it + had without the job. + +The job holds no registry credential and cannot write anything. It cannot +fail the run or stop a build: every failure, a timeout included, leaves the +outputs empty and the builds inline. The 40 minutes of the wait end inside +the limit of its step (55 minutes) and of the job (70 minutes), so the script +ends the wait, with a notice, before the workflow would stop it. + +**The check in the build.** With the build argument, the build fails unless +its `runner-artifacts` is the publication of those inputs (see "The +application image"). A wrong or missing build context then fails the build, +and no image carries labels for a Runner it does not have. The resolver +answers `unsupported` for a `Dockerfile` that does not declare the argument. + +**The lockfile.** The published Runner was built from the committed +`pnpm-lock.yaml`, and its build stamp covers that whole file. A build that +copies it must therefore build the committed lockfile. `docker.yml` refreshes +the lockfile on the runner before an inline build, because the inline Runner +stages install frozen. It does not refresh it when the answer is `published`: +a refresh rewrites the file whenever the manifest of any package is ahead of +it, also a package the Runner does not read, and the server build would then +refuse the published Runner as built from other inputs. The resolver answers +`published` only when the committed lockfile matches the Runner's own +manifests, so the `runner-lockfile` stage passes, and the `deps` stage +resolves the manifests of the other packages inside the image, as it does for +a local `docker build`. A build that is given a published Runner and a +refreshed lockfile fails at the step that vendors the Runner. + +**Merge-queue images.** `docker.yml` can push the image that the merge queue +built for a commit instead of building it. That image made its own Runner. A +run whose `runner_image` job answered `published` does not look for it and +builds from source, so one commit never has both kinds. + +**Rollback.** Set `RUNNER_IMAGE_CONSUME_ENABLED` to another value, or delete +it. The next run of `docker.yml` builds inline. No revert is needed, and +images that exist do not change. + ## Publication `.github/workflows/runner-image.yml` publishes on a `master` push in this diff --git a/scripts/merge-group-image.test.mjs b/scripts/merge-group-image.test.mjs index ce82abc5534..5654c65e415 100644 --- a/scripts/merge-group-image.test.mjs +++ b/scripts/merge-group-image.test.mjs @@ -179,7 +179,7 @@ test("master promotes only after proof and otherwise builds exactly as before", const step = name => { const found = steps.find(item => item.startsWith(` - name: ${name}\n`)); assert.ok(found, name); return found; }; const order = name => steps.indexOf(step(name)); assert.match(step("Find the merge-queue build of this commit"), - /if: github.repository == 'paperclipai\/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs\/heads\/master'/); + /if: needs.runner_image.outputs.mode != 'published' && github.repository == 'paperclipai\/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs\/heads\/master'/); for (const name of ["Find the merge-queue build of this commit", "Download the merge-queue image", "Promote the merge-queue image"]) { assert.match(step(name), /continue-on-error: true/, `${name} must fall back to the build`); } diff --git a/scripts/reuse-canary-image.test.mjs b/scripts/reuse-canary-image.test.mjs index 26dfbc83c5f..226589fa24f 100644 --- a/scripts/reuse-canary-image.test.mjs +++ b/scripts/reuse-canary-image.test.mjs @@ -66,7 +66,7 @@ test("canary reuse gates both native builds and still runs convergent promotion" assert.match(reuse, /github.repository_id == '1170821064'.*startsWith\(github.ref, 'refs\/tags\/canary\/v'\)/); assert.match(reuse, /node scripts\/reuse-canary-image.mjs/); const build = workflow.split(" build-and-push:\n")[1].split(" merge-and-push:\n")[0]; - assert.match(build, /needs: \[reuse_canary_image\]/); + assert.match(build, /needs: \[reuse_canary_image, runner_image\]/); assert.match(build, /!cancelled\(\).*result == 'skipped'.*result == 'success'.*outputs.reused == 'false'/); assert.match(build, /platform: linux\/amd64/); assert.match(build, /platform: linux\/arm64/); diff --git a/server/scripts/prepare-runner-vendor.test.mjs b/server/scripts/prepare-runner-vendor.test.mjs index 3f2d894c1b3..f85dc9057f5 100644 --- a/server/scripts/prepare-runner-vendor.test.mjs +++ b/server/scripts/prepare-runner-vendor.test.mjs @@ -313,6 +313,47 @@ describe("prepareRunnerVendor", () => { expect(readBuildInputsStamp(stampPath)).not.toBeNull(); }); + it("with PAPERCLIP_RUNNER_VENDOR_REQUIRE_CURRENT=1, accepts a Runner built elsewhere for the committed lockfile and refuses it once the lockfile is refreshed for another package", () => { + const committed = [ + "lockfileVersion: '9.0'", + "importers:", + " packages/paperclip-runner:", + " dependencies:", + " ajv: { specifier: ^8.20.0, version: 8.20.0 }", + " ui:", + " dependencies:", + " react: { specifier: ^19.2.7, version: 19.2.8 }", + "", + ].join("\n"); + const built = makeRepo(); + writeFileSync(path.join(built.repoRoot, "pnpm-lock.yaml"), committed); + recordCurrentBuild(built); + + const { repoRoot, packageRoot } = makeRepo(); + writeFileSync(path.join(repoRoot, "pnpm-lock.yaml"), committed); + writeTree(repoRoot, { + "packages/paperclip-runner/node_modules/.cache/paperclip-runner-build-inputs.json": readFileSync(buildInputsStampPath(built.packageRoot), "utf8"), + }); + const env = { [REQUIRE_CURRENT_VARIABLE]: "1" }; + let runs = 0; + const prepare = (lines) => prepareRunnerVendor({ repoRoot, env, run: () => { runs += 1; return 0; }, log: (line) => lines.push(line) }); + + const reused = []; + expect(prepare(reused)).toBe(0); + expect(reused[1]).toMatch(/,action:reuse,reason:inputs_unchanged,/); + + writeFileSync(path.join(repoRoot, "pnpm-lock.yaml"), committed.replace("specifier: ^19.2.7", "specifier: ^19.2.8")); + const refused = []; + expect(prepare(refused)).toBe(1); + expect(refused[0]).toContain("pnpm-lock.yaml included"); + expect(refused[1]).toMatch(/,action:fail,reason:inputs_changed,/); + + writeFileSync(path.join(repoRoot, "pnpm-lock.yaml"), committed); + expect(prepare([])).toBe(0); + expect(runs).toBe(0); + expect(readBuildInputsStamp(buildInputsStampPath(packageRoot))).not.toBeNull(); + }); + it.skipIf(process.platform === "win32")( "end to end: the first server build rebuilds the Runner, the second reuses it, a change or a lost output rebuilds again", async () => { diff --git a/server/src/__tests__/docker-build-stamp.test.ts b/server/src/__tests__/docker-build-stamp.test.ts index 2bb94d73700..656ef563837 100644 --- a/server/src/__tests__/docker-build-stamp.test.ts +++ b/server/src/__tests__/docker-build-stamp.test.ts @@ -175,8 +175,9 @@ describe("Application image Runner", () => { for (const [name, source] of [["standard", workflow], ["preview", previewWorkflow], ["merge group", mergeGroupWorkflow], ["pull request", appCheckWorkflow]]) { expect(source, name).toContain("PAPERCLIP_BUILD_COMMIT="); expect(source, name).not.toMatch(/PAPERCLIP_RUNNER_(?:SOURCE_REVISION|IMAGE_PUBLICATION)/); - expect(source, name).not.toMatch(/runner-artifacts|build-contexts?\b/); + if (name !== "standard") expect(source, name).not.toMatch(/runner-artifacts|build-contexts?\b/); } + expect(workflow.match(/runner-artifacts|build-contexts?\b/g)).toEqual(["runner-artifacts", "build-contexts"]); }); }); diff --git a/server/src/__tests__/docker-runner-workflows.test.ts b/server/src/__tests__/docker-runner-workflows.test.ts index f61ac5a69ac..33183e2abcd 100644 --- a/server/src/__tests__/docker-runner-workflows.test.ts +++ b/server/src/__tests__/docker-runner-workflows.test.ts @@ -1,7 +1,10 @@ -import { readFileSync } from "node:fs"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; +import { RUNNER_IMAGE_WAIT_MINUTES } from "../../../scripts/runner-image-resolve.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); const read = (name: string) => readFileSync(path.join(repoRoot, ".github", "workflows", name), "utf8"); @@ -23,40 +26,203 @@ function expectPinnedAndExpressionFree(text: string) { for (const script of shell) expect(script).not.toContain("${{"); } -describe("the report-only Runner image job in docker.yml", () => { +const named = (job: string, name: string) => { + const found = steps(job).filter((step) => step.startsWith(` - name: ${name}\n`)); + expect(found, name).toHaveLength(1); + return found[0]; +}; +const shell = (step: string) => scripts(`${step}\n`)[0].trimEnd(); +const condition = (text: string, indent: number) => new RegExp(`^ {${indent}}if: (.+)$`, "m").exec(text)?.[1] ?? ""; +const minutes = (text: string, indent: number) => Number(new RegExp(`^ {${indent}}timeout-minutes: (\\d+)$`, "m").exec(text)![1]); +function runStep(step: string, env: Record, stubs: Record) { + const directory = mkdtempSync(path.join(os.tmpdir(), "docker-runner-workflows-")); + try { + for (const [name, body] of Object.entries(stubs)) writeFileSync(path.join(directory, name), `#!/bin/sh\n${body}\n`, { mode: 0o755 }); + for (const file of ["output", "summary", "calls"]) writeFileSync(path.join(directory, file), ""); + const run = spawnSync("bash", ["--noprofile", "--norc", "-eo", "pipefail", "-c", shell(step)], { + cwd: directory, + encoding: "utf8", + env: { + PATH: `${directory}${path.delimiter}${process.env.PATH}`, RUNNER_TEMP: directory, CALLS: path.join(directory, "calls"), + GITHUB_OUTPUT: path.join(directory, "output"), GITHUB_STEP_SUMMARY: path.join(directory, "summary"), ...env, + }, + }); + const text = (file: string) => readFileSync(path.join(directory, file), "utf8"); + return { status: run.status, stdout: run.stdout, stderr: run.stderr, output: text("output"), calls: text("calls").trimEnd().split("\n").filter(Boolean) }; + } finally { + rmSync(directory, { recursive: true, force: true }); + } +} +function outputs(text: string) { + const values: Record = {}; + const lines = text.split("\n"); + for (let index = 0; index < lines.length; index += 1) { + const block = /^([a-z_]+)<<(.+)$/.exec(lines[index]); + if (block) { + const end = lines.indexOf(block[2], index + 1); + values[block[1]] = lines.slice(index + 1, end).join("\n"); + index = end; + } else if (lines[index].includes("=")) { + values[lines[index].slice(0, lines[index].indexOf("="))] = lines[index].slice(lines[index].indexOf("=") + 1); + } + } + return values; +} +function listInputs(step: string, source: string, resolved: Record) { + const rendered = `${step}\n`.replace(new RegExp(`\\$\\{\\{ ${source.replaceAll(".", "\\.")}\\.([a-z_]+) \\}\\}`, "g"), (_match, name: string) => + (resolved[name] ?? "").split("\n").join(`\n${" ".repeat(12)}`)); + const list = (input: string) => { + const found = new RegExp(`^ {10}${input}: (?:\\|\\n((?: {12}.*\\n)+)|(.*)\\n)`, "m").exec(rendered); + return (found?.[1] ?? found?.[2] ?? "").split("\n").map((line) => line.trim()).filter(Boolean); + }; + return { buildArgs: list("build-args"), buildContexts: list("build-contexts"), labels: list("labels") }; +} +const IMAGE = `ghcr.io/paperclipai/paperclip-runner@sha256:${"c".repeat(64)}`; +const INPUTS = "1".repeat(64); +const LABELS = `io.paperclip.runner.image=${IMAGE}\nio.paperclip.runner.inputs-digest=${INPUTS}`; +const BUILD_ARGS = `PAPERCLIP_RUNNER_EXPECT_INPUTS_DIGEST=${INPUTS}`; +const published = { mode: "published", build_contexts: `runner-artifacts=docker-image://${IMAGE}`, labels: LABELS, build_args: BUILD_ARGS }; +const canonical = "github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064'"; +const VERIFY = "Verify the attestation of the Runner image"; + +describe("the Runner image an application image build copies in docker.yml", () => { const workflow = read("docker.yml"); const jobs = jobsOf(workflow); - const report = jobs.runner_image_report; + const resolver = jobs.runner_image; + const legs = jobs["build-and-push"]; + const masterPush = "github.event_name == 'push' && github.ref == 'refs/heads/master'"; + const buildInputs = (resolved: Record) => listInputs(named(legs, "Build and push by digest"), "needs.runner_image.outputs", resolved); + const inlineBuild = { + buildArgs: [ + "PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}", + "PAPERCLIP_BUILD_COMMIT=${{ github.sha }}", + "CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}", + ], + buildContexts: [], + labels: ["${{ steps.meta.outputs.labels }}"], + }; - it("holds no credential, can only read the repository, and cannot write to a registry or sign", () => { + it("runs only when the repository variable has the enabling value, in the canonical repository, and not otherwise", () => { + expect(condition(resolver, 4)).toBe(`vars.RUNNER_IMAGE_CONSUME_ENABLED == 'true' && ${canonical}`); + expect(workflow.match(/RUNNER_IMAGE_CONSUME_ENABLED/g)).toHaveLength(1); + expect(Object.keys(jobs).filter((name) => jobs[name].includes("runner-image-resolve.mjs"))).toEqual(["runner_image"]); + for (const name of ["docker-merge-group.yml", "docker-app-check.yml"]) { + expect(read(name)).not.toMatch(/runner-image-resolve|RUNNER_IMAGE_CONSUME_ENABLED|build-contexts|--build-context|io\.paperclip\.runner|runner_image/); + } + }); + + it("gives a build with no answer exactly the build arguments, contexts and labels it had before", () => { + expect(buildInputs({})).toEqual(inlineBuild); + expect(buildInputs({ mode: "inline" })).toEqual(inlineBuild); + }); + + it("gives both architectures the one answer: the context, the labels and the expected inputs of the verified image", () => { + expect(/^ {4}needs: \[(.+)\]$/m.exec(legs)![1].split(", ")).toContain("runner_image"); + const reads = [...legs.matchAll(/needs\.runner_image\.[a-z_.]+/g)].map((match) => match[0]); + expect(new Set(reads)).toEqual(new Set(["mode", "build_args", "build_contexts", "labels"].map((name) => `needs.runner_image.outputs.${name}`))); + for (const line of legs.split("\n").filter((entry) => entry.includes("needs.runner_image"))) expect(line).not.toContain("matrix."); + expect(buildInputs(published)).toEqual({ + buildArgs: [...inlineBuild.buildArgs, BUILD_ARGS], + buildContexts: [`runner-artifacts=docker-image://${IMAGE}`], + labels: [...inlineBuild.labels, ...LABELS.split("\n")], + }); + }); + + it("hands a build only what the attestation step wrote, and that step fails and writes nothing for an image that does not verify", () => { + const declared = /\n {4}outputs:\n((?: {6}.+\n)+)/.exec(resolver)![1].trimEnd().split("\n").map((line) => line.trim()); + expect(declared).toEqual([ + "mode: ${{ steps.verify.outputs.mode || 'inline' }}", + "build_contexts: ${{ steps.verify.outputs.build_contexts }}", + "labels: ${{ steps.verify.outputs.labels }}", + "build_args: ${{ steps.verify.outputs.build_args }}", + ]); + const verify = named(resolver, VERIFY); + expect(verify).toContain("\n id: verify\n"); + expect(condition(verify, 8)).toBe("steps.resolve.outputs.mode == 'published'"); + const env = { IMAGE, LABELS, BUILD_ARGS, GH_TOKEN: "token-for-the-test" }; + const gh = (status: number) => ({ gh: `printf '%s\\n' "$*" >> "$CALLS"\nexit ${status}` }); + + const verified = runStep(verify, env, gh(0)); + expect(verified.status, verified.stderr).toBe(0); + expect(verified.calls).toEqual([ + `attestation verify oci://${IMAGE} --repo paperclipai/paperclip --signer-workflow paperclipai/paperclip/.github/workflows/runner-image.yml --source-ref refs/heads/master`, + ]); + expect(outputs(verified.output)).toEqual(published); + expect(verified.stdout).not.toContain("::warning"); + + for (const status of [1, 4]) { + const refused = runStep(verify, env, gh(status)); + expect(refused.status, refused.stderr).toBe(1); + expect(refused.calls).toHaveLength(1); + expect(refused.output).toBe(""); + expect(refused.stdout).toContain("::warning title=Runner image not verified::"); + } + expect(verify).not.toMatch(/continue-on-error/); + expect(resolver).toContain("\n continue-on-error: true\n"); + expect(condition(legs, 4)).not.toMatch(/runner_image|success\(\)/); + }); + + it("builds the committed lockfile when it copies the published Runner, and refreshes it for every other build", () => { + const refresh = named(legs, "Refresh lockfile for Docker build context"); + expect(condition(refresh, 8)).toBe("needs.runner_image.outputs.mode != 'published'"); + expect(legs.match(/pnpm install/g)).toHaveLength(1); + expect(refresh).toContain("pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile"); + const order = ["Refresh lockfile for Docker build context", "Build and push by digest"].map((name) => steps(legs).indexOf(named(legs, name))); + expect(order[0]).toBeLessThan(order[1]); + }); + + it("waits only for a master push, and gives the token to the lookup only then", () => { + const resolve = named(resolver, "Find the published Runner image of these inputs"); + expect(resolve).toContain("\n id: resolve\n"); + expect(resolve).toContain(` WAIT: \${{ ${masterPush} }}\n`); + expect(resolve).toContain(` GITHUB_TOKEN: \${{ ${masterPush} && github.token || '' }}\n`); + const node = { node: `printf '%s\\n' "$*" >> "$CALLS"` }; + const command = (wait: string) => runStep(resolve, { WAIT: wait }, node).calls; + const waiting = command("true"); + expect(waiting).toHaveLength(1); + expect(waiting[0]).toMatch(/^scripts\/runner-image-resolve\.mjs resolve --dir \S+\/runner-image-resolve --wait$/); + for (const wait of ["false", ""]) { + const once = command(wait); + expect(once).toHaveLength(1); + expect(once[0]).toMatch(/^scripts\/runner-image-resolve\.mjs resolve --dir \S+\/runner-image-resolve$/); + } + }); + + it("holds no credential where it builds, can write nothing, and reads only what the wait and the verification need", () => { expect(workflow).toMatch(/^permissions: \{\}$/m); - expect(permissions(report)).toEqual(["contents: read"]); - expect(report).not.toMatch(credential); - expect(report).not.toMatch(/github\.actor|REGISTRY_USERNAME/); - expect(report).not.toMatch(registryWrite); - expect(report).toContain("persist-credentials: false"); - expectPinnedAndExpressionFree(report); + expect(permissions(resolver)).toEqual(["contents: read", "actions: read", "attestations: read"]); + expect(resolver.slice(0, resolver.indexOf("\n steps:\n"))).not.toMatch(credential); + expect(steps(resolver).filter((step) => credential.test(step)).map((step) => /- name: (.+)/.exec(step)![1])).toEqual([ + "Find the published Runner image of these inputs", + "Verify the attestation of the Runner image", + ]); + const building = steps(resolver).filter((step) => /runner-image-resolve\.mjs inputs|docker buildx|setup-buildx/.test(step)); + expect(building).toHaveLength(2); + for (const step of building) expect(step).not.toMatch(/\n {8}env:\n/); + expect(resolver).not.toMatch(/github\.actor|REGISTRY_USERNAME|REGISTRY_PASSWORD/); + expect(resolver).not.toMatch(registryWrite); + expect(resolver).toContain("persist-credentials: false"); + expectPinnedAndExpressionFree(resolver); }); - it("bounds the step that builds and the lookup", () => { - for (const command of ["runner-image-resolve.mjs inputs", "runner-image-resolve.mjs resolve"]) { - const running = steps(report).filter((step) => step.includes(command)); - expect(running).toHaveLength(1); - expect(running[0]).toMatch(/^ {8}timeout-minutes: \d+$/m); + it("does not promote a merge-queue image for a commit that copies a published Runner", () => { + const find = named(legs, "Find the merge-queue build of this commit"); + expect(condition(find, 8)).toBe(`needs.runner_image.outputs.mode != 'published' && ${canonical} && ${masterPush}`); + for (const name of ["Download the merge-queue image", "Promote the merge-queue image"]) { + expect(condition(named(legs, name), 8)).toBe("steps.find.outputs.run_id != ''"); } + expect(legs.match(/steps\.find\.outputs\.run_id/g)).toHaveLength(3); + expect(condition(named(legs, "Build and push by digest"), 8)).toBe("steps.promote.outputs.digest == ''"); }); - it("cannot fail the run or feed another job", () => { - expect(report).toContain("\n continue-on-error: true\n"); - expect(report).toMatch(/^ {4}timeout-minutes: \d+$/m); - expect(report).not.toMatch(/^ {4}(?:needs|outputs):/m); - expect(workflow).not.toContain("needs.runner_image_report"); - for (const [name, job] of Object.entries(jobs)) { - if (name !== "runner_image_report") expect(/^ {4}needs: .*$/m.exec(job)?.[0] ?? "").not.toContain("runner_image_report"); - } - expect(/^ {4}if: (.+)$/m.exec(report)![1]).toBe( - "github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master'", - ); + it("cannot fail or block an image build, and ends its own wait before any time limit of the workflow", () => { + expect(resolver).toContain("\n continue-on-error: true\n"); + expect(condition(legs, 4).startsWith("${{ !cancelled() && ")).toBe(true); + expect(condition(legs, 4)).not.toContain("runner_image"); + const limits = ["Compute the Runner inputs of this commit", "Find the published Runner image of these inputs", "Verify the attestation of the Runner image"] + .map((name) => minutes(named(resolver, name), 8)); + expect(limits[1]).toBeGreaterThanOrEqual(RUNNER_IMAGE_WAIT_MINUTES + 15); + expect(minutes(resolver, 4)).toBeGreaterThanOrEqual(limits.reduce((sum, limit) => sum + limit, 0) + 5); }); });