diff --git a/.github/workflows/runner-image.yml b/.github/workflows/runner-image.yml index c8855371fad..c98a189724e 100644 --- a/.github/workflows/runner-image.yml +++ b/.github/workflows/runner-image.yml @@ -1,13 +1,29 @@ name: Runner image -# Builds and checks the Runner image: the `runner-artifacts` target of the -# Dockerfile, a plain file tree that other image builds copy the Runner from, -# and its `runner-artifacts-slim` variant without the pack's transport -# archives. doc/runner-image-contract.md is the contract. +# Builds the Runner once for each distinct set of its inputs and publishes it +# as ghcr.io/paperclipai/paperclip-runner, a multi-platform image that other +# image builds copy from, in two variants: `inputs-` (full) and +# `inputs--slim` (without the pack's transport archives). +# doc/runner-image-contract.md is the contract. # -# This workflow builds and checks only. It has no access to a registry and -# publishes nothing. +# Every master push computes the digest from the commit's own build context, +# with the committed pnpm-lock.yaml. The build runs only when no image has +# that digest yet. When the committed lockfile is behind the manifests the +# Runner build reads, the run says so, publishes nothing and stays green: the +# repository's lockfile refresh brings the commit that publishes. +# +# Pull requests that touch the Runner's inputs, its recipe or this pipeline, +# and manual runs, always build and check the image and never publish it. +# They resolve the lockfile first, because a contributor cannot edit it. +# +# This is a workflow of its own, not a job in docker.yml: docker.yml has no +# pull-request trigger and holds one concurrency slot per ref, and the Runner +# image needs its own signer identity so its attestation cannot stand in for +# the application image's. on: + push: + branches: + - master pull_request: # Every input of scripts/runner-image-identity.mjs, .dockerignore (it # decides which of those files the build receives), and the pipeline's @@ -30,27 +46,40 @@ on: - scripts/runner-image-identity.mjs - scripts/runner-image-layout.mjs - scripts/runner-image-oci.mjs + - scripts/runner-image-publish.mjs + - scripts/standard-image-contract.mjs - docker/runner-image/** workflow_dispatch: -# Least privilege: nothing at the workflow level, and read-only contents in -# each job. +# Least privilege: nothing at the workflow level. Only the publish job, which +# runs no build, can write to the registry or sign. permissions: {} -# A newer push to a pull request replaces its older run. +# A newer push to a pull request replaces its older run. A master run has a +# group of its own, so no other run cancels it here. The publish job has a +# second group, one for each digest; see there for what that group can cancel. concurrency: group: runner-image-${{ github.event.pull_request.number || github.sha }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +env: + IMAGE: ghcr.io/paperclipai/paperclip-runner + jobs: identity: name: Resolve the Runner image identity + # A fork's own pushes do not run this workflow. Pull requests and manual + # runs are not affected. + if: github.event_name != 'push' || (github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064') runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read + packages: read outputs: + lock: ${{ steps.lock.outputs.state }} digest: ${{ steps.identity.outputs.digest }} + exists: ${{ steps.lookup.outputs.exists }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 @@ -63,7 +92,7 @@ jobs: version: 9.15.4 run_install: false - # No dependency cache: a restored cache must not reach the image. + # No dependency cache: a restored cache must not reach a published image. - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: @@ -73,7 +102,13 @@ jobs: # A contributor cannot edit pnpm-lock.yaml, so a pull request that # changes the Runner's dependencies is built against a lockfile resolved # here. The image build installs from it frozen. + # + # Never on a master push. The lockfile refresh owns pnpm-lock.yaml on + # master and holds new package versions until they are old enough, so + # the published image installs from the committed file and its digest + # is a function of the commit alone. - name: Resolve the lockfile + if: github.event_name != 'push' run: | set -euo pipefail pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile @@ -84,7 +119,10 @@ jobs: echo "$changed" exit 1 fi - echo "This run built from a \`pnpm-lock.yaml\` resolved for it${changed:+ (it differs from the committed file)}." >> "$GITHUB_STEP_SUMMARY" + { + echo "This run built from a \`pnpm-lock.yaml\` resolved for it${changed:+ (it differs from the committed file)}." + echo "A master push publishes only from the committed lockfile." + } >> "$GITHUB_STEP_SUMMARY" - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 @@ -93,11 +131,40 @@ jobs: [registry."docker.io"] mirrors = ["mirror.gcr.io"] + # The files the Runner build receives, after .dockerignore: the root + # manifest and the manifests of the Runner and the eval kernel are the + # only ones in it. + - name: Export the Runner build context + timeout-minutes: 10 + run: docker buildx build --target runner-artifacts-context --output "type=local,dest=$RUNNER_TEMP/runner-image-context" . + + # The image build installs with a frozen lockfile. This asks the same + # question first, offline and without installing, so that a lockfile + # that is merely behind is told apart from a broken build. pnpm can + # rewrite the lockfile it checks. It checks the exported copy above, + # which no build reads: the builds read the checkout. + - name: Check the lockfile against the Runner's manifests + id: lock + run: node scripts/runner-image-publish.mjs lock-state --context "$RUNNER_TEMP/runner-image-context" + + # On master this is expected between a dependency change and the next + # lockfile refresh. It is not a failure, and there is nothing to build. + - name: Report a lockfile that is behind the Runner's manifests + if: steps.lock.outputs.state == 'behind' + run: | + set -euo pipefail + message="pnpm-lock.yaml is behind the Runner's manifests at $GITHUB_SHA; no Runner image for this commit; the next lockfile refresh will publish one." + echo "::notice title=No Runner image for this commit::$message" + echo "$message" >> "$GITHUB_STEP_SUMMARY" + # A run that resolved its own lockfile cannot be behind it. + test "$GITHUB_EVENT_NAME" = "push" + # Computed inside the build, from the files the build context supplies # after .dockerignore. A digest computed from the checkout would cover # files the image build never receives. - name: Compute the identity in the build context id: identity + if: steps.lock.outputs.state == 'current' timeout-minutes: 10 run: | set -euo pipefail @@ -115,8 +182,10 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" # The legs must build from the file the identity was computed from, not - # from a second resolution. + # from a second resolution. Only for runs that resolved one: a master + # push builds from the committed file, which every job checks out. - name: Keep the resolved lockfile for the build legs + if: steps.lock.outputs.state == 'current' && github.event_name != 'push' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-image-lockfile @@ -124,14 +193,34 @@ jobs: if-no-files-found: error retention-days: 1 + # An image that exists is never rebuilt on master. It exists when its + # full tag does: that tag is written last. An inconclusive answer + # builds: the publish job decides again, strictly, before it writes a + # tag. Other events build whether or not the image exists. + - name: Look for the published image + id: lookup + if: steps.lock.outputs.state == 'current' && github.event_name == 'push' + env: + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + DIGEST: ${{ steps.identity.outputs.digest }} + run: node scripts/runner-image-publish.mjs publication-state --inputs-digest "$DIGEST" + # Multi-arch by native runner, not QEMU, as in docker.yml. GitHub-hosted # runners: the protected release fleets admit only docker.yml on master. # - # These legs hold no credential and use no registry build cache. The install - # runs dependency lifecycle scripts. + # These legs cannot write to the registry on any event, and they read and + # write no registry build cache. The install runs dependency lifecycle + # scripts; the only token in the job can read packages, and only the lookup + # step receives it. + # + # A master push builds only when the image does not exist. A pull request + # and a manual run always build, so a change to the smoke matrix or to the + # pipeline is exercised even when the Runner's inputs did not change. build: name: Build and check (${{ matrix.arch }}) needs: identity + if: needs.identity.outputs.lock == 'current' && (github.event_name != 'push' || (github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && needs.identity.outputs.exists != 'true')) strategy: fail-fast: false matrix: @@ -146,7 +235,10 @@ jobs: timeout-minutes: 60 permissions: contents: read + packages: read env: + # Only a canonical master push keeps its image for publication. + KEEP_IMAGE: ${{ github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' }} DIGEST: ${{ needs.identity.outputs.digest }} ARCH: ${{ matrix.arch }} PLATFORM: ${{ matrix.platform }} @@ -162,16 +254,33 @@ jobs: node-version: 24 package-manager-cache: false + # Another master run can have published these inputs since the identity + # job looked, most often the run of the push just before this one. Then + # this leg has nothing to do. Master only: other events always build. + # An inconclusive answer builds. + - name: Look again for the published image + id: published + if: github.event_name == 'push' + env: + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: node scripts/runner-image-publish.mjs publication-state --inputs-digest "$DIGEST" + - name: Download the resolved lockfile + if: github.event_name != 'push' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-image-lockfile path: ${{ runner.temp }}/runner-image-lockfile + # Build from the lockfile the identity was computed from, not from a + # second resolution that a registry change could make different. - name: Use the resolved lockfile + if: github.event_name != 'push' run: cp "$RUNNER_TEMP/runner-image-lockfile/pnpm-lock.yaml" pnpm-lock.yaml - name: Free runner disk + if: steps.published.outputs.exists != 'true' run: | set -euo pipefail sudo apt-get clean || true @@ -181,6 +290,7 @@ jobs: df -h - name: Set up Docker Buildx + if: steps.published.outputs.exists != 'true' uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 with: buildkitd-config-inline: | @@ -199,9 +309,11 @@ jobs: # # This is the publication form of the build, the only one whose pack # names the image (`runnerInputsDigest`), so the check below can bind - # the tree to the digest the identity job resolved. Nothing here - # publishes it: the archives and trees stay on this runner. + # the tree to the digest the identity job resolved. This job cannot + # publish it. On a canonical master push the archives are the images + # the publish job pushes, byte for byte. - name: Build both variants of the Runner image + if: steps.published.outputs.exists != 'true' timeout-minutes: 30 run: | set -euo pipefail @@ -215,6 +327,7 @@ jobs: --build-arg PAPERCLIP_RUNNER_IMAGE_PUBLICATION=1 \ --provenance=false --sbom=false \ --output "type=oci,dest=$RUNNER_TEMP/runner-image/$variant.oci.tar" \ + --metadata-file "$RUNNER_TEMP/runner-image/$variant.metadata.json" \ . done @@ -223,6 +336,7 @@ jobs: # not a second export of the build, and each blob is checked against # its digest as it is read. - name: Unpack both images + if: steps.published.outputs.exists != 'true' timeout-minutes: 10 run: | set -euo pipefail @@ -239,6 +353,7 @@ jobs: # them as a sandbox does and compares the result with the pack; that # needs about as much free disk as the pack is large. - name: Check the layout contract and pack integrity + if: steps.published.outputs.exists != 'true' timeout-minutes: 10 run: | set -euo pipefail @@ -254,6 +369,7 @@ jobs: # then the transport layers, with nothing else in them. No file may be # in two layers. - name: Check the layers of both variants + if: steps.published.outputs.exists != 'true' timeout-minutes: 10 run: | set -euo pipefail @@ -269,6 +385,7 @@ jobs: # one run covers both, and the slim tree is the smaller one to load. # This command has one export, a few small files. - name: Run the daemon on every consumer base image + if: steps.published.outputs.exists != 'true' timeout-minutes: 20 run: | docker buildx build \ @@ -278,3 +395,273 @@ jobs: --output "type=local,dest=$RUNNER_TEMP/runner-image/smoke" \ docker/runner-image ls -l "$RUNNER_TEMP/runner-image/smoke" + + # Written only after every check passed. The publish job accepts nothing + # without it. The two archives become one OCI layout that holds both + # variants and stores the layers they share once, so the upload is the + # size of the full image, not of both. The command reads the layout + # back and checks every blob against its digest. + # + # On every event, so that a pull request exercises the hand-off on + # real build output. Only the upload below is for a master push. + - name: Record the verified images + if: steps.published.outputs.exists != 'true' + timeout-minutes: 10 + run: | + node scripts/runner-image-publish.mjs record \ + --arch "$ARCH" \ + --slim-archive "$RUNNER_TEMP/runner-image/slim.oci.tar" \ + --slim-metadata "$RUNNER_TEMP/runner-image/slim.metadata.json" \ + --full-archive "$RUNNER_TEMP/runner-image/full.oci.tar" \ + --full-metadata "$RUNNER_TEMP/runner-image/full.metadata.json" \ + --tree "$RUNNER_TEMP/runner-image/full" \ + --layout "$RUNNER_TEMP/runner-image/images.oci.tar" \ + --out "$RUNNER_TEMP/runner-image/record.json" + + - name: Upload the verified images + if: env.KEEP_IMAGE == 'true' && steps.published.outputs.exists != 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: runner-image-${{ matrix.arch }} + path: | + ${{ runner.temp }}/runner-image/images.oci.tar + ${{ runner.temp }}/runner-image/record.json + if-no-files-found: error + # Layers are already compressed. + compression-level: 0 + retention-days: 1 + + - name: Report an image another run published first + if: steps.published.outputs.exists == 'true' + run: echo "\`$IMAGE:inputs-$DIGEST\` and its slim variant already exist. This leg built nothing." >> "$GITHUB_STEP_SUMMARY" + + # Pushes the verified bytes, names them and attests them. Only a master push + # in the canonical repository reaches this job; it builds nothing and runs no + # dependency code. + # + # It writes two tags, `inputs--slim` and then `inputs-`. The + # full tag is last, so it marks a complete publication: when it exists, both + # exist, and the full index names its slim index by digest. A run that stops + # between the two tags leaves a slim tag alone. The inputs are then not + # published; the next run for them builds again, replaces that slim tag with + # its own, and writes the full tag. It never keeps another run's slim image. + publish: + name: Publish and attest + needs: [identity, build] + if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master' && needs.identity.outputs.lock == 'current' && needs.identity.outputs.exists != 'true' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + id-token: write + attestations: write + # One publication per digest at a time, so the strict check below and the + # tag writes cannot interleave with another run for the same inputs. + # GitHub keeps one waiting job for a group. When three runs for one digest + # reach this job, the second is cancelled as the third arrives. That does + # no harm: the third publishes the digest if the first did not. + concurrency: + group: runner-image-publish-${{ needs.identity.outputs.digest }} + cancel-in-progress: false + env: + DIGEST: ${{ needs.identity.outputs.digest }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + package-manager-cache: false + + # A build leg that found the image published uploaded nothing. Ask + # before the downloads, so that run ends here and not on a missing + # artifact. + - name: Look for an image another run published + id: early + env: + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: node scripts/runner-image-publish.mjs publication-state --inputs-digest "$DIGEST" + + - name: Download the verified amd64 images + if: steps.early.outputs.exists != 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-image-amd64 + path: ${{ runner.temp }}/runner-image/amd64 + + - name: Download the verified arm64 images + if: steps.early.outputs.exists != 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: runner-image-arm64 + path: ${{ runner.temp }}/runner-image/arm64 + + - name: Accept only this run's verified images + id: legs + if: steps.early.outputs.exists != 'true' + run: | + node scripts/runner-image-publish.mjs verify-legs \ + --dir "$RUNNER_TEMP/runner-image" \ + --inputs-digest "$DIGEST" \ + --source-revision "$GITHUB_SHA" + + - name: Login to GitHub Container Registry + if: steps.early.outputs.exists != 'true' + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + if: steps.early.outputs.exists != 'true' + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + + # The downloads take a while. Ask once more immediately before the + # pushes, so that a run that lost the race does not upload its images. + - name: Look again immediately before pushing + id: prepush + if: steps.early.outputs.exists != 'true' + env: + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: node scripts/runner-image-publish.mjs publication-state --inputs-digest "$DIGEST" + + # By digest and untagged. --preserve-digests fails rather than re-encode, + # so the registry holds exactly the images the build legs verified. Each + # leg's layout names its two images `slim` and `full`. + - name: Push the verified images by digest + if: steps.prepush.outputs.exists == 'false' + timeout-minutes: 15 + env: + SLIM_AMD64: ${{ steps.legs.outputs.slim_amd64 }} + SLIM_ARM64: ${{ steps.legs.outputs.slim_arm64 }} + FULL_AMD64: ${{ steps.legs.outputs.full_amd64 }} + FULL_ARM64: ${{ steps.legs.outputs.full_arm64 }} + run: | + set -euo pipefail + command -v skopeo >/dev/null || { echo "skopeo is not installed on this runner"; exit 1; } + echo "Pushing with $(skopeo --version)" + layout="$RUNNER_TEMP/runner-image" + skopeo copy --preserve-digests "oci-archive:$layout/amd64/images.oci.tar:slim" "docker://$IMAGE@$SLIM_AMD64" + skopeo copy --preserve-digests "oci-archive:$layout/arm64/images.oci.tar:slim" "docker://$IMAGE@$SLIM_ARM64" + skopeo copy --preserve-digests "oci-archive:$layout/amd64/images.oci.tar:full" "docker://$IMAGE@$FULL_AMD64" + skopeo copy --preserve-digests "oci-archive:$layout/arm64/images.oci.tar:full" "docker://$IMAGE@$FULL_ARM64" + + # The tags are create-only. Another run can have published these inputs + # since the lookups above. Only a clear answer goes on: `absent`, or + # `incomplete` (a slim tag alone, which this run replaces). `published` + # skips the rest. An inconclusive answer fails, and so does a full tag + # whose slim image is missing or is another image. + - name: Confirm the images are not published + id: gate + if: steps.prepush.outputs.exists == 'false' + env: + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: node scripts/runner-image-publish.mjs publication-state --inputs-digest "$DIGEST" --strict + + # Candidate tags first, so a final tag appears only after its digest is + # attested. Candidate tags are internal and may be overwritten. + - name: Create the slim multi-platform candidate + if: steps.gate.outputs.publish == 'true' + env: + AMD64: ${{ steps.legs.outputs.slim_amd64 }} + ARM64: ${{ steps.legs.outputs.slim_arm64 }} + run: | + set -euo pipefail + docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.source=https://github.com/paperclipai/paperclip" \ + --annotation "index:org.opencontainers.image.revision=$GITHUB_SHA" \ + --annotation "index:io.paperclip.runner.inputs-digest=$DIGEST" \ + --annotation "index:io.paperclip.runner.variant=slim" \ + -t "$IMAGE:candidate-$DIGEST-slim" \ + "$IMAGE@$AMD64" "$IMAGE@$ARM64" + + - name: Verify the slim candidate names exactly the verified images + id: slim + if: steps.gate.outputs.publish == 'true' + env: + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: node scripts/runner-image-publish.mjs verify-index "candidate-$DIGEST-slim" --variant slim --dir "$RUNNER_TEMP/runner-image" + + # The full index names the slim index of this same run by digest, so a + # consumer can tell that the two belong together. + - name: Create the full multi-platform candidate + if: steps.gate.outputs.publish == 'true' + env: + AMD64: ${{ steps.legs.outputs.full_amd64 }} + ARM64: ${{ steps.legs.outputs.full_arm64 }} + SLIM_INDEX: ${{ steps.slim.outputs.digest }} + run: | + set -euo pipefail + docker buildx imagetools create \ + --annotation "index:org.opencontainers.image.source=https://github.com/paperclipai/paperclip" \ + --annotation "index:org.opencontainers.image.revision=$GITHUB_SHA" \ + --annotation "index:io.paperclip.runner.inputs-digest=$DIGEST" \ + --annotation "index:io.paperclip.runner.variant=full" \ + --annotation "index:io.paperclip.runner.slim-digest=$SLIM_INDEX" \ + -t "$IMAGE:candidate-$DIGEST" \ + "$IMAGE@$AMD64" "$IMAGE@$ARM64" + + - name: Verify the full candidate names exactly the verified images and its slim index + id: full + if: steps.gate.outputs.publish == 'true' + env: + SLIM_INDEX: ${{ steps.slim.outputs.digest }} + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: | + node scripts/runner-image-publish.mjs verify-index "candidate-$DIGEST" --variant full \ + --slim-digest "$SLIM_INDEX" --dir "$RUNNER_TEMP/runner-image" + + - name: Attest the slim Runner image + if: steps.gate.outputs.publish == 'true' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-name: ghcr.io/paperclipai/paperclip-runner + subject-digest: ${{ steps.slim.outputs.digest }} + push-to-registry: true + + - name: Attest the full Runner image + if: steps.gate.outputs.publish == 'true' + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-name: ghcr.io/paperclipai/paperclip-runner + subject-digest: ${{ steps.full.outputs.digest }} + push-to-registry: true + + # Slim first, full last. Writing the slim tag replaces the one a run + # that stopped here left behind. The last command checks the pair as + # every later run will see it. + - name: Publish the tags + if: steps.gate.outputs.publish == 'true' + env: + SLIM_INDEX: ${{ steps.slim.outputs.digest }} + FULL_INDEX: ${{ steps.full.outputs.digest }} + RUNNER_IMAGE_REGISTRY_USERNAME: ${{ github.actor }} + RUNNER_IMAGE_REGISTRY_PASSWORD: ${{ github.token }} + run: | + set -euo pipefail + layout="$RUNNER_TEMP/runner-image" + docker buildx imagetools create -t "$IMAGE:inputs-$DIGEST-slim" "$IMAGE@$SLIM_INDEX" + node scripts/runner-image-publish.mjs verify-index "inputs-$DIGEST-slim" --variant slim --dir "$layout" --expect "$SLIM_INDEX" + docker buildx imagetools create -t "$IMAGE:inputs-$DIGEST" "$IMAGE@$FULL_INDEX" + node scripts/runner-image-publish.mjs verify-index "inputs-$DIGEST" --variant full --slim-digest "$SLIM_INDEX" --dir "$layout" --expect "$FULL_INDEX" + node scripts/runner-image-publish.mjs publication-state --inputs-digest "$DIGEST" --strict | grep -Fx "state=published" + { + echo "Published from \`$GITHUB_SHA\`:" + echo "- full: \`$IMAGE:inputs-$DIGEST\` at \`$IMAGE@$FULL_INDEX\`" + echo "- slim: \`$IMAGE:inputs-$DIGEST-slim\` at \`$IMAGE@$SLIM_INDEX\`" + } >> "$GITHUB_STEP_SUMMARY" + + - name: Report images another run published first + if: steps.early.outputs.exists == 'true' || steps.prepush.outputs.exists == 'true' || steps.gate.outputs.state == 'published' + run: echo "\`$IMAGE:inputs-$DIGEST\` and its slim variant already exist. Another run published first; this run published nothing." >> "$GITHUB_STEP_SUMMARY" diff --git a/doc/DOCKER.md b/doc/DOCKER.md index b208d6df269..771f800779b 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -84,7 +84,8 @@ package's build output, the remote provider pack and its transport archives. The `runner-artifacts-slim` target is the same image without the transport archives. Neither depends on the server or UI build, and no other target depends on them yet. `.github/workflows/runner-image.yml` builds and checks -both on pull requests, for Linux AMD64 and ARM64. See +both on pull requests, for Linux AMD64 and ARM64, and publishes them from +`master` as `ghcr.io/paperclipai/paperclip-runner`. See [`doc/runner-image-contract.md`](runner-image-contract.md) for the layout, the identity rules and how to consume the image. diff --git a/doc/runner-image-contract.md b/doc/runner-image-contract.md index f6263aedba5..3a535ff5b8a 100644 --- a/doc/runner-image-contract.md +++ b/doc/runner-image-contract.md @@ -10,9 +10,6 @@ two variants: - slim, the `runner-artifacts-slim` target: the same without the two paths marked "full only", the provider pack's transport archives. -Nothing publishes it yet: `.github/workflows/runner-image.yml` builds and -checks it. - ## Layout Every path a consumer may copy (`RUNNER_IMAGE_PATHS` in @@ -40,8 +37,9 @@ is the same file in both variants: - `inputsDigest` (the identity below) and `buildInputsDigest` (the digest in the Runner's build stamp); -- `sourceRevision`: the commit given to the build, also in the pack as - `payload.runnerSourceRevision`; +- `sourceRevision`: the commit that built the image, also in the pack as + `payload.runnerSourceRevision`. An image is published once for its + inputs, so later commits with the same inputs keep this revision; - `publication`: `true` only for a build with the build argument `PAPERCLIP_RUNNER_IMAGE_PUBLICATION=1`. Only then does the pack carry the identity, as `payload.runnerInputsDigest`. Another image build that runs @@ -70,8 +68,10 @@ lists the dependency files first, so their names do not depend on `dist`. ## Identity -Both variants are named by one digest, `inputs-`: the variant is not -part of the identity. `scripts/runner-image-identity.mjs` computes it from: +Both variants are named by one digest: the variant is not part of the +identity. The full image has the tag `inputs-` and the slim image +`inputs--slim`. `scripts/runner-image-identity.mjs` computes the +digest from: - every file under the paths the build copies from the build context (`RUNNER_IMAGE_CONTEXT_INPUTS`, which include `pnpm-lock.yaml`): path, @@ -144,6 +144,41 @@ its own, as in this repository, can take a built image or tree in its place with `--build-context =`. The stage is then not built. +## Publication + +`.github/workflows/runner-image.yml` publishes on a `master` push in this +repository and nowhere else. Other runs build and check, and publish nothing. + +- **Lockfile.** A `master` push builds from the committed `pnpm-lock.yaml`, + so the digest is a function of the commit. When that file is behind the + Runner's manifests, the run writes a notice, publishes nothing and stays + green. The commit that merges the lockfile refresh publishes the image. +- **Once.** The build runs only when `inputs-` does not exist. +- **Verified bytes.** The build legs cannot write to the registry. The + publish job pushes the images they checked by digest and joins each + variant into a multi-platform index under a `candidate-` tag. It attests + both indexes, and only then writes the two `inputs-` tags. `candidate-` + tags are internal: one can name an index that is not attested, and the + next run overwrites it. Do not consume them. +- **Order.** It writes `inputs--slim` first and `inputs-` + last, so the full tag marks a complete publication. The full index names + its slim index in the annotation `io.paperclip.runner.slim-digest`. +- **Pairing.** Resolve the full tag first. Take the slim digest from that + annotation of the full index, and pull the slim image by that digest. Never + resolve `inputs--slim` by tag on its own: while a publication is + incomplete, that tag can name the slim image of another run. +- **Recovery.** A run that stops between the two tags leaves a slim tag + alone. Those inputs are not published. The next run for them builds again, + replaces that tag with its own slim image and then writes the full tag. A + full tag whose slim tag is missing or names another image fails every run + for those inputs. An operator must then delete both package versions and + re-run the workflow run of a `master` push with those inputs. +- **Create-only.** The workflow never moves a tag of a complete publication, + but the registry does not enforce that. Pin the index digest and verify it: + `gh attestation verify oci://ghcr.io/paperclipai/paperclip-runner@sha256: --repo paperclipai/paperclip --signer-workflow paperclipai/paperclip/.github/workflows/runner-image.yml --source-ref refs/heads/master` +- **First publication.** GitHub creates the package private. A maintainer + must make it public once. + ## Known limits - A build without `PAPERCLIP_RUNNER_SOURCE_REVISION` or @@ -157,3 +192,6 @@ built. does not change the identity. - The archive check does not tell a directory with its own archive member from one created only as a file's parent. +- A lockfile that the lockfile refresh cannot bring up to the Runner's + manifests leaves every `master` run green with only a notice, for as long + as it lasts. No image is published in that time, and nothing fails. diff --git a/scripts/runner-image-oci.mjs b/scripts/runner-image-oci.mjs index f698920eaa1..45a1538ca7f 100644 --- a/scripts/runner-image-oci.mjs +++ b/scripts/runner-image-oci.mjs @@ -225,13 +225,18 @@ async function readTarStream(source, visit) { } /** - * The one image in an OCI archive: its manifest descriptor and its layers. A + * One image in an OCI archive: its manifest descriptor and its layers. A * member is found by its exact name (`index.json`, `blobs/sha256/`), as * buildx writes them. The manifest is checked against its digest here, and - * each layer when `readLayer` reads it. + * each layer when `readLayer` or `verifyBlobs` reads it. + * + * An archive from buildx holds one image. A layout that holds several names + * each with the annotation `org.opencontainers.image.ref.name`; `ref` then + * says which one to read. * @param {string} archive + * @param {{ ref?: string | null }} [options] */ -export function readOciArchive(archive) { +export function readOciArchive(archive, { ref = null } = {}) { const members = indexTarFile(archive); const located = (name) => { const member = members.get(name); @@ -255,12 +260,23 @@ export function readOciArchive(archive) { }; const sha256 = (data) => `sha256:${createHash("sha256").update(data).digest("hex")}`; const index = JSON.parse(bytes("index.json").toString("utf8")); - if (!Array.isArray(index.manifests) || index.manifests.length !== 1) throw new Error(`${archive} must hold exactly one image manifest`); - const { mediaType, digest, size } = index.manifests[0]; + if (!Array.isArray(index.manifests)) throw new Error(`${archive} has no list of manifests`); + const named = ref === null ? index.manifests : index.manifests.filter((entry) => entry.annotations?.["org.opencontainers.image.ref.name"] === ref); + if (named.length !== 1) { + throw new Error(ref === null ? `${archive} must hold exactly one image manifest` : `${archive} must hold exactly one image named ${ref}`); + } + const { mediaType, digest, size } = named[0]; const manifestBytes = bytes(blobName(digest)); if (sha256(manifestBytes) !== digest) throw new Error(`${archive}: the manifest does not have the digest the index names`); const manifest = JSON.parse(manifestBytes.toString("utf8")); if (!Array.isArray(manifest.layers)) throw new Error(`${archive} holds an index, not one platform's image; build without attestations`); + /** The digest of a blob's bytes as the archive holds them. */ + const blobDigest = async (name) => { + const { offset, size: length } = located(name); + const hash = createHash("sha256"); + if (length > 0) for await (const chunk of createReadStream(archive, { start: offset, end: offset + length - 1 })) hash.update(chunk); + return `sha256:${hash.digest("hex")}`; + }; for (const layer of manifest.layers) { if (!GZIP_LAYER.test(layer.mediaType ?? "")) throw new Error(`${archive}: layer ${layer.digest} is ${layer.mediaType}, not a gzip tar`); located(blobName(layer.digest)); @@ -270,6 +286,12 @@ export function readOciArchive(archive) { descriptor: { mediaType, digest, size }, /** @type {{ mediaType: string, digest: string, size: number }[]} */ layers: manifest.layers.map((layer) => ({ mediaType: layer.mediaType, digest: layer.digest, size: layer.size })), + /** Fails unless the config and every layer of the image are in the archive with their digests. */ + async verifyBlobs() { + for (const blob of [manifest.config, ...manifest.layers]) { + if (await blobDigest(blobName(blob?.digest)) !== blob.digest) throw new Error(`${archive}: blob ${blob.digest} does not have its digest`); + } + }, /** Visits every entry of one layer and fails when the blob does not have its digest. */ async readLayer(layer, visit) { const { offset, size: bytesInArchive } = located(blobName(layer.digest)); diff --git a/scripts/runner-image-publish.mjs b/scripts/runner-image-publish.mjs new file mode 100644 index 00000000000..48974f95404 --- /dev/null +++ b/scripts/runner-image-publish.mjs @@ -0,0 +1,533 @@ +#!/usr/bin/env node +// Publication checks for the Runner image (.github/workflows/runner-image.yml). +// +// The build legs run without write access to the registry. Each one records +// the two variants it built and verified, slim and full; the publish job +// pushes exactly those bytes, names each variant under one multi-platform +// tag and attests both. +// +// lock-state whether pnpm-lock.yaml matches the manifests in an exported +// `runner-artifacts-context` tree: `current` or `behind`. It +// runs the image build's own frozen install as a lockfile-only, +// offline check. Any other failure is an error. pnpm can +// rewrite the lockfile of the tree it checks, so the command +// refuses a checkout +// record a build leg, after its checks passed: joins the OCI archives +// of the two variants into one OCI layout, in which their +// shared layers are stored once, and binds it to the image +// digests, the architecture, the inputs digest and the source +// revision +// verify-legs the publish job, before any push: accepts only one record per +// architecture, for this run's inputs digest and commit, whose +// layout still has the recorded bytes +// publication-state whether these inputs are published. The full tag +// `inputs-` is written last, so it alone says +// `published`; a slim tag without it is `incomplete` and the +// next publication replaces it. A full tag whose slim tag is +// missing or names another image is an error. --strict also +// fails on `unknown` instead of guessing +// verify-index the pushed tag names exactly the recorded images of one +// variant; the full index must also name its slim index +// +// Registry credentials come from RUNNER_IMAGE_REGISTRY_USERNAME and +// RUNNER_IMAGE_REGISTRY_PASSWORD when both are set; lookups are anonymous +// otherwise. An unknown flag, a repeated flag or a flag with an empty value is +// an error. + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createReadStream, existsSync } from "node:fs"; +import { appendFile, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { RUNNER_IMAGE_NAME, RUNNER_IMAGE_PLATFORMS } from "./runner-image-identity.mjs"; +import { RUNNER_IMAGE_PATHS, RUNNER_IMAGE_VARIANTS } from "./runner-image-layout.mjs"; +import { readOciArchive } from "./runner-image-oci.mjs"; +import { assertIndexUsesVerifiedImages, standardImageDigest } from "./standard-image-contract.mjs"; + +export const RUNNER_IMAGE_RECORD_SCHEMA = "paperclip.runner-image-leg.v2"; +/** On the full index: the digest of the slim index published with it. */ +export const SLIM_DIGEST_ANNOTATION = "io.paperclip.runner.slim-digest"; +/** The tag of each variant. The full tag is written last and marks a complete publication. */ +export const runnerImageTags = (inputsDigest) => ({ slim: `inputs-${inputsDigest}-slim`, full: `inputs-${inputsDigest}` }); +const ARCHITECTURES = RUNNER_IMAGE_PLATFORMS.map((platform) => platform.split("/")[1]); +const DIGEST = /^sha256:[0-9a-f]{64}$/; +const HEX_64 = /^[0-9a-f]{64}$/; +const REVISION = /^[0-9a-f]{40}$/; +const MANIFEST_TYPES = [ + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.v2+json", +].join(", "); + +async function sha256File(file) { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(file)) hash.update(chunk); + return `sha256:${hash.digest("hex")}`; +} + +function tar(args, options = {}) { + const result = spawnSync("tar", args, { maxBuffer: 64 * 1024 * 1024, ...options }); + assert.equal(result.status, 0, `tar ${args[0]} failed: ${String(result.stderr).trim().slice(-300)}`); + return result.stdout; +} + +/** + * What a build leg hands to the publish job: one OCI layout with both + * variants, named `slim` and `full`, and a record read from the tree the leg + * verified. The layout stores the layers the variants share once. + * @param {{ + * architecture: string, + * archives: { slim: string, full: string }, + * metadataFiles: { slim: string, full: string }, + * tree: string, + * layout: string, + * }} input + */ +export async function createLegRecord({ architecture, archives, metadataFiles, tree, layout }) { + assert.ok(ARCHITECTURES.includes(architecture), `Unsupported architecture ${architecture}`); + const manifest = JSON.parse(await readFile(path.join(tree, RUNNER_IMAGE_PATHS.manifest), "utf8")); + assert.equal(manifest.platform?.architecture, architecture, "The verified tree is for another architecture"); + assert.match(manifest.inputsDigest?.digest ?? "", HEX_64); + assert.match(manifest.sourceRevision ?? "", REVISION); + assert.equal(manifest.publication, true, "Only a publication build can be recorded"); + /** @type {Record} */ + const images = {}; + for (const variant of RUNNER_IMAGE_VARIANTS) { + const image = readOciArchive(archives[variant]); + images[variant] = { descriptor: image.descriptor, layers: image.layers.map((layer) => layer.digest) }; + const reported = JSON.parse(await readFile(metadataFiles[variant], "utf8"))["containerimage.digest"]; + assert.match(reported ?? "", DIGEST, `The ${variant} build reported no image digest`); + assert.equal(images[variant].descriptor.digest, reported, `The ${variant} archive is not the image its build reported`); + } + // The same blobs in the same order: the full image is the slim image plus layers. + assert.deepEqual(images.full.layers.slice(0, images.slim.layers.length), images.slim.layers, + "The full image does not start with the layers of the slim image"); + assert.ok(images.full.layers.length > images.slim.layers.length, "The full image has no layer of its own"); + + const directory = await mkdtemp(path.join(tmpdir(), "runner-image-layout-")); + try { + for (const variant of RUNNER_IMAGE_VARIANTS) tar(["-xf", archives[variant], "-C", directory]); + await writeFile(path.join(directory, "index.json"), JSON.stringify({ + schemaVersion: 2, + mediaType: "application/vnd.oci.image.index.v1+json", + manifests: RUNNER_IMAGE_VARIANTS.map((variant) => ({ + ...images[variant].descriptor, + annotations: { "org.opencontainers.image.ref.name": variant }, + })), + })); + tar(["-cf", layout, "-C", directory, "blobs", "index.json", "oci-layout"]); + } finally { + await rm(directory, { recursive: true, force: true }); + } + // Read the layout back the way the publish job's push does: each image by + // its name, with every blob it needs under its digest. + for (const variant of RUNNER_IMAGE_VARIANTS) { + const image = readOciArchive(layout, { ref: variant }); + assert.equal(image.descriptor.digest, images[variant].descriptor.digest, `The layout names another image as ${variant}`); + await image.verifyBlobs(); + } + return { + schema: RUNNER_IMAGE_RECORD_SCHEMA, + architecture, + images: { slim: images.slim.descriptor.digest, full: images.full.descriptor.digest }, + inputsDigest: manifest.inputsDigest.digest, + sourceRevision: manifest.sourceRevision, + layoutSha256: await sha256File(layout), + }; +} + +/** + * The image digest of each variant and architecture, as `_`, + * accepted only when every leg built this run's inputs at this run's commit + * and its layout is unchanged. + * @param {{ directory: string, inputsDigest: string, sourceRevision: string, layoutName?: string }} input + * @returns {Promise>} + */ +export async function verifyLegRecords({ directory, inputsDigest, sourceRevision, layoutName = "images.oci.tar" }) { + assert.match(inputsDigest ?? "", HEX_64); + assert.match(sourceRevision ?? "", REVISION); + /** @type {Record} */ + const digests = {}; + for (const architecture of ARCHITECTURES) { + const legDirectory = path.join(directory, architecture); + const record = JSON.parse(await readFile(path.join(legDirectory, "record.json"), "utf8")); + assert.equal(record.schema, RUNNER_IMAGE_RECORD_SCHEMA, `${architecture}: unknown record schema`); + assert.equal(record.architecture, architecture, `${architecture}: the record is for another architecture`); + assert.equal(record.inputsDigest, inputsDigest, `${architecture}: the image was built from other inputs`); + assert.equal(record.sourceRevision, sourceRevision, `${architecture}: the image was built at another commit`); + for (const variant of RUNNER_IMAGE_VARIANTS) { + assert.match(record.images?.[variant] ?? "", DIGEST, `${architecture}: the record has no ${variant} image digest`); + digests[`${variant}_${architecture}`] = record.images[variant]; + } + assert.notEqual(record.images.slim, record.images.full, `${architecture}: the record names one image for both variants`); + assert.equal(await sha256File(path.join(legDirectory, layoutName)), record.layoutSha256, + `${architecture}: the image layout does not match its record`); + } + return digests; +} + +function repositoryOf(image) { + const match = /^ghcr\.io\/([a-z0-9][a-z0-9._/-]*)$/.exec(image); + assert.ok(match, `Not a ghcr.io image name: ${image}`); + return match[1]; +} + +async function request(url, headers, fetchImpl, sleep) { + for (let attempt = 1; ; attempt += 1) { + let response; + try { + response = await fetchImpl(url, { headers, redirect: "error", signal: AbortSignal.timeout(30_000) }); + } catch (error) { + if (attempt === 5) throw error; + await sleep(attempt * 1000); + continue; + } + const transient = [408, 425, 429].includes(response.status) || response.status >= 500; + if (!transient || attempt === 5) return response; + await response.body?.cancel(); + await sleep(attempt * 1000); + } +} + +/** + * Reads a manifest. `state` is `present` with the registry bytes, `absent` + * when the registry says the manifest does not exist, and `unknown` for every + * other answer: a refused token, a package that is private to this caller, or + * a registry error that outlasted the retries. + * @param {string} reference + * @param {{ + * image?: string, + * credentials?: { username: string, password: string } | null, + * fetchImpl?: (url: string, init: { headers: Record, redirect: "error", signal: AbortSignal }) => Promise, + * sleep?: (ms: number) => Promise, + * }} [options] + * @returns {Promise<{ state: "present", bytes: Buffer, digest: string } | { state: "absent" } | { state: "unknown", detail: string }>} + */ +export async function readRunnerImageManifest(reference, { + image = RUNNER_IMAGE_NAME, + credentials = null, + fetchImpl = fetch, + sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), +} = {}) { + const repository = repositoryOf(image); + /** @type {Record} */ + const basic = credentials ? { authorization: `Basic ${Buffer.from(`${credentials.username}:${credentials.password}`).toString("base64")}` } : {}; + // A registry that cannot be reached, or that answers with something other + // than a token, leaves the question open. It is not an answer. + let token; + try { + const tokenResponse = await request(`https://ghcr.io/token?service=ghcr.io&scope=repository:${repository}:pull`, basic, fetchImpl, sleep); + if (!tokenResponse.ok) { + await tokenResponse.body?.cancel(); + return { state: "unknown", detail: `token request: HTTP ${tokenResponse.status}` }; + } + ({ token } = await tokenResponse.json()); + } catch (error) { + return { state: "unknown", detail: `token request failed: ${error instanceof Error ? error.message : String(error)}` }; + } + if (typeof token !== "string" || !token) return { state: "unknown", detail: "token request returned no token" }; + let response; + let bytes; + try { + response = await request(`https://ghcr.io/v2/${repository}/manifests/${reference}`, + { authorization: `Bearer ${token}`, accept: MANIFEST_TYPES }, fetchImpl, sleep); + if (response.status === 404) { + await response.body?.cancel(); + return { state: "absent" }; + } + if (!response.ok) { + await response.body?.cancel(); + return { state: "unknown", detail: `manifest request: HTTP ${response.status}` }; + } + bytes = Buffer.from(await response.arrayBuffer()); + } catch (error) { + return { state: "unknown", detail: `manifest request failed: ${error instanceof Error ? error.message : String(error)}` }; + } + assert.ok(bytes.length <= 1024 * 1024, "Registry manifest exceeds its size limit"); + const digest = `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + const reported = response.headers.get("docker-content-digest"); + assert.ok(!reported || reported === digest, "Registry digest does not match the manifest bytes"); + return { state: "present", bytes, digest }; +} + +/** + * The digest of the pushed index, accepted only when it names exactly the + * recorded images. `slimDigest` is given for the full index, which must name + * the slim index published with it, and for no other. + * @param {Buffer} bytes + * @param {Record} legDigests one image digest for each architecture + * @param {{ slimDigest?: string | null }} [options] + */ +export function verifyPublishedIndex(bytes, legDigests, { slimDigest = null } = {}) { + assert.deepEqual(Object.keys(legDigests).sort(), [...ARCHITECTURES].sort(), "Expected one recorded image for each architecture"); + const digest = standardImageDigest(bytes); + assertIndexUsesVerifiedImages(bytes, new Set(Object.values(legDigests))); + const index = JSON.parse(bytes.toString("utf8")); + // Exactly the two images, and nothing beside them: no attestation manifest, + // no entry without a platform, no other operating system. + assert.equal(index.manifests.length, ARCHITECTURES.length, `Expected exactly ${ARCHITECTURES.length} manifests in the index, found ${index.manifests.length}`); + for (const entry of index.manifests) { + const architecture = entry.platform?.architecture; + // BuildKit records linux/arm64 with or without the variant `v8`; both are + // the one arm64 platform. No other variant is one of the two platforms. + const variant = entry.platform?.variant; + assert.ok(entry.platform?.os === "linux" && ARCHITECTURES.includes(architecture) + && (variant === undefined || (architecture === "arm64" && variant === "v8")), + `The index names an unexpected platform: ${JSON.stringify(entry.platform ?? null)}`); + assert.equal(entry.digest, legDigests[architecture], `linux/${architecture} is not the image its build leg recorded`); + } + assert.equal(index.annotations?.[SLIM_DIGEST_ANNOTATION] ?? null, slimDigest, + slimDigest === null ? "Only the full index may name a slim index" : "The full index does not name the slim index published with it"); + return digest; +} + +/** + * Whether the images of these inputs are published. The publish job writes + * the slim tag, then the full tag, so the full tag is the completion marker: + * + * - `absent`: neither tag exists; + * - `incomplete`: only the slim tag exists. A publication stopped between the + * two tags. Nothing may use that slim image; the next publication replaces + * the tag with its own slim image before it writes the full tag; + * - `published`: the full tag exists and names, by digest, the image the slim + * tag points at; + * - `broken`: the full tag exists, but the slim tag is missing or points at + * another image. No run can repair that: the full tag is never moved; + * - `unknown`: the registry gave no clear answer. + * @param {string} inputsDigest + * @param {(reference: string) => ReturnType} read + * @returns {Promise<{ state: "absent" | "incomplete" | "published" | "broken" | "unknown", detail: string, full?: string, slim?: string }>} + */ +export async function runnerPublicationState(inputsDigest, read) { + assert.match(inputsDigest ?? "", HEX_64); + const tags = runnerImageTags(inputsDigest); + const full = await read(tags.full); + if (full.state === "unknown") return { state: "unknown", detail: `${tags.full}: ${full.detail}` }; + const slim = await read(tags.slim); + if (slim.state === "unknown") return { state: "unknown", detail: `${tags.slim}: ${slim.detail}` }; + if (full.state === "absent") { + return slim.state === "absent" + ? { state: "absent", detail: `${tags.full} does not exist` } + : { state: "incomplete", detail: `${tags.slim} exists without ${tags.full}: an earlier publication did not finish`, slim: slim.digest }; + } + let named = null; + try { + named = JSON.parse(full.bytes.toString("utf8")).annotations?.[SLIM_DIGEST_ANNOTATION] ?? null; + } catch { + // Not JSON: it names no slim image. + } + if (slim.state === "present" && slim.digest === named) { + return { state: "published", detail: `${tags.full} and ${tags.slim} exist`, full: full.digest, slim: slim.digest }; + } + return { + state: "broken", + detail: `${tags.full} exists and names the slim image ${named ?? "(none)"}, but ${tags.slim} ${slim.state === "present" ? `is ${slim.digest}` : "does not exist"}`, + full: full.digest, + }; +} + +/** pnpm reports a lockfile that the manifests have moved past with one of these codes. */ +const LOCK_BEHIND_CODES = ["ERR_PNPM_OUTDATED_LOCKFILE", "ERR_PNPM_LOCKFILE_CONFIG_MISMATCH"]; + +/** + * Whether pnpm-lock.yaml in an exported `runner-artifacts-context` tree + * matches the manifests in that tree. This is the image build's frozen, + * filtered install with --lockfile-only and --offline: it downloads nothing + * and installs nothing. pnpm can still write `pnpm-lock.yaml` in that tree + * again, so `context` must be a copy no build reads: the workflow gives it an + * export of the build context, and a checkout is refused. `behind` means the + * manifests the Runner build reads were changed after the lockfile was. Every + * other failure throws. + * @param {string} context + * @param {{ run?: (command: string, args: string[], options: object) => { status: number | null, stdout: string, stderr: string, error?: Error } }} [options] + * @returns {{ state: "current" | "behind", detail: string }} + */ +export function runnerLockState(context, { run = (command, args, options) => spawnSync(command, args, options) } = {}) { + assert.ok(existsSync(path.join(context, "pnpm-lock.yaml")), `${context} has no pnpm-lock.yaml`); + assert.ok(existsSync(path.join(context, "packages", "paperclip-runner", "package.json")), `${context} is not a Runner build context`); + assert.ok(!existsSync(path.join(context, ".git")), `${context} is a checkout: pnpm can rewrite its pnpm-lock.yaml. Check an exported build context`); + const result = run("pnpm", ["install", "--frozen-lockfile", "--lockfile-only", "--offline", "--ignore-scripts", + "--filter", "@paperclipai/paperclip-runner..."], { cwd: context, encoding: "utf8" }); + if (result.error) throw result.error; + const output = `${result.stdout ?? ""}${result.stderr ?? ""}`; + if (result.status === 0) return { state: "current", detail: "pnpm-lock.yaml matches the Runner's manifests" }; + const code = LOCK_BEHIND_CODES.find((candidate) => output.includes(candidate)); + if (code) { + const line = output.split("\n").find((candidate) => candidate.includes(code)) ?? code; + return { state: "behind", detail: line.trim().slice(0, 500) }; + } + throw new Error(`The lockfile check failed for another reason (exit ${result.status}):\n${output.trim().slice(-2000)}`); +} + +const COMMAND_FLAGS = Object.freeze({ + "lock-state": { context: "value" }, + record: { + arch: "value", "slim-archive": "value", "slim-metadata": "value", "full-archive": "value", "full-metadata": "value", + tree: "value", layout: "value", out: "value", + }, + "verify-legs": { dir: "value", "inputs-digest": "value", "source-revision": "value" }, + "publication-state": { "inputs-digest": "value", strict: "boolean" }, + "verify-index": { dir: "value", variant: "value", expect: "value", "slim-digest": "value" }, +}); +const REQUIRED_FLAGS = Object.freeze({ + "lock-state": ["context"], + record: ["arch", "slim-archive", "slim-metadata", "full-archive", "full-metadata", "tree", "layout", "out"], + "verify-legs": ["dir", "inputs-digest", "source-revision"], + "publication-state": ["inputs-digest"], + "verify-index": ["dir", "variant"], +}); + +/** + * Splits a command line into positional arguments and the flags the command + * knows. Throws on an unknown command, an unknown or repeated flag, a missing + * or empty value, and a missing required flag. + * @param {string | undefined} command + * @param {string[]} argv + */ +export function parsePublishArguments(command, argv) { + if (!command || !Object.hasOwn(COMMAND_FLAGS, command)) { + throw new Error("Usage: runner-image-publish.mjs lock-state|record|verify-legs|publication-state|verify-index ..."); + } + const known = COMMAND_FLAGS[command]; + /** @type {string[]} */ + const positional = []; + /** @type {Record} */ + const flags = {}; + for (let index = 0; index < argv.length; index += 1) { + const argument = argv[index]; + if (!argument.startsWith("--")) { + positional.push(argument); + continue; + } + const name = argument.slice(2); + if (!Object.hasOwn(known, name)) throw new Error(`${command} does not take ${argument}`); + if (Object.hasOwn(flags, name)) throw new Error(`${argument} is given twice`); + if (known[name] === "boolean") { + flags[name] = true; + continue; + } + const value = argv[index += 1]; + if (value === undefined || value === "" || value.startsWith("--")) throw new Error(`${argument} needs a value`); + flags[name] = value; + } + for (const name of REQUIRED_FLAGS[command]) { + if (!Object.hasOwn(flags, name)) throw new Error(`${command} needs --${name}`); + } + const takesReference = command === "verify-index"; + if (positional.length !== (takesReference ? 1 : 0)) { + throw new Error(takesReference ? `${command} takes exactly one tag or digest` : `${command} takes no positional argument`); + } + if (takesReference && !/^(?:[A-Za-z0-9_][A-Za-z0-9._-]{0,127}|sha256:[0-9a-f]{64})$/.test(positional[0])) { + throw new Error(`${command}: not a tag or digest: ${positional[0]}`); + } + return { positional, flags }; +} + +/** + * Runs one command. Returns the values it reports; throws when the command's + * check does not hold. + * @param {string[]} argv + * @param {{ env?: NodeJS.ProcessEnv, read?: typeof readRunnerImageManifest, lockState?: typeof runnerLockState, log?: (line: string) => void }} [options] + * @returns {Promise>} + */ +export async function runPublishCommand(argv, { + env = process.env, + read = readRunnerImageManifest, + lockState = runnerLockState, + log = console.log, +} = {}) { + const [command, ...rest] = argv; + const { positional, flags } = parsePublishArguments(command, rest); + const value = (name) => /** @type {string} */ (flags[name]); + if (command === "lock-state") { + const result = lockState(path.resolve(value("context"))); + log(result.detail); + return { state: result.state }; + } + if (command === "record") { + const record = await createLegRecord({ + architecture: value("arch"), + archives: { slim: value("slim-archive"), full: value("full-archive") }, + metadataFiles: { slim: value("slim-metadata"), full: value("full-metadata") }, + tree: value("tree"), + layout: value("layout"), + }); + await writeFile(value("out"), `${JSON.stringify(record, null, 2)}\n`); + return { slim: record.images.slim, full: record.images.full }; + } + if (command === "verify-legs") { + return verifyLegRecords({ + directory: value("dir"), inputsDigest: value("inputs-digest"), sourceRevision: value("source-revision"), + }); + } + const credentials = environmentCredentials(env); + if (command === "publication-state") { + const tags = runnerImageTags(value("inputs-digest")); + const result = await runnerPublicationState(value("inputs-digest"), (reference) => read(reference, { credentials })); + // Only an operator can repair this, so it fails every run that meets it. + assert.notEqual(result.state, "broken", `${result.detail}. No run can repair this: the workflow never moves ${tags.full}. ` + + `Delete the package versions tagged ${tags.full} and ${tags.slim}, then re-run the workflow run of a master push with these inputs.`); + if (result.state === "unknown") { + const message = `Could not determine whether ${RUNNER_IMAGE_NAME}:${tags.full} is published (${result.detail}).`; + // A create-only tag is never written on a guess. + assert.ok(flags.strict !== true, message); + log(`::notice title=Runner image lookup inconclusive::${message}`); + } + if (result.state === "incomplete") log(`::notice title=Runner image publication incomplete::${result.detail}.`); + return { + state: result.state, + exists: result.state === "published", + // What a publish job may do next: write the tags, slim first. + publish: result.state === "absent" || result.state === "incomplete", + }; + } + const reference = positional[0]; + const variant = value("variant"); + assert.ok(RUNNER_IMAGE_VARIANTS.includes(variant), "--variant must be slim or full"); + if (Object.hasOwn(flags, "slim-digest")) assert.match(value("slim-digest"), DIGEST, "--slim-digest must be a sha256 digest"); + assert.equal(Object.hasOwn(flags, "slim-digest"), variant === "full", "--slim-digest is given for the full index, and only for it"); + const result = await read(reference, { credentials }); + assert.equal(result.state, "present", `${RUNNER_IMAGE_NAME}:${reference} is ${result.state}`); + assert.ok(result.state === "present"); + const digest = verifyPublishedIndex(result.bytes, await legDigests(value("dir"), variant), + { slimDigest: variant === "full" ? value("slim-digest") : null }); + if (Object.hasOwn(flags, "expect")) { + assert.match(value("expect"), DIGEST, "--expect must be a sha256 digest"); + assert.equal(digest, value("expect"), `${reference} resolves to ${digest}, expected ${value("expect")}`); + } + return { digest }; +} + +async function output(values) { + const lines = Object.entries(values).map(([key, value]) => `${key}=${value}`); + for (const line of lines) console.log(line); + if (process.env.GITHUB_OUTPUT) await appendFile(process.env.GITHUB_OUTPUT, `${lines.join("\n")}\n`); +} + +function environmentCredentials(env = process.env) { + const username = env.RUNNER_IMAGE_REGISTRY_USERNAME; + const password = env.RUNNER_IMAGE_REGISTRY_PASSWORD; + return username && password ? { username, password } : null; +} + +async function legDigests(directory, variant) { + /** @type {Record} */ + const digests = {}; + for (const architecture of ARCHITECTURES) { + digests[architecture] = JSON.parse(await readFile(path.join(directory, architecture, "record.json"), "utf8")).images?.[variant]; + } + return digests; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) { + try { + await output(await runPublishCommand(process.argv.slice(2))); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + } +} diff --git a/server/src/__tests__/runner-image-publish.test.ts b/server/src/__tests__/runner-image-publish.test.ts new file mode 100644 index 00000000000..f7eb9a74e67 --- /dev/null +++ b/server/src/__tests__/runner-image-publish.test.ts @@ -0,0 +1,548 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { afterAll, describe, expect, it } from "vitest"; +import { + SLIM_DIGEST_ANNOTATION, + createLegRecord, + parsePublishArguments, + readRunnerImageManifest, + runPublishCommand, + runnerImageTags, + runnerLockState, + runnerPublicationState, + verifyLegRecords, + verifyPublishedIndex, +} from "../../../scripts/runner-image-publish.mjs"; +import { readOciArchive } from "../../../scripts/runner-image-oci.mjs"; + +/** + * The checks the Runner image workflow runs around publication. The image has + * two variants under two tags, and consumers verify their attestations. So + * the bytes published must be the ones the build legs checked, a tag must + * never be written on a guess, and a publication that stopped half way must + * neither block its inputs nor leave a full image paired with another run's + * slim image. + */ + +const INPUTS = "1".repeat(64); +const REVISION = "a".repeat(40); +const TAGS = runnerImageTags(INPUTS); +const sha256 = (bytes: Buffer | string) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +const scratch = mkdtempSync(path.join(tmpdir(), "runner-image-publish-")); +afterAll(() => rmSync(scratch, { recursive: true, force: true })); +let sequence = 0; + +/** What `docker buildx build --output type=oci --metadata-file` writes for one image with these layers. */ +function ociArchive(directory: string, name: string, layers: string[]) { + const layout = path.join(directory, `${name}-layout`); + mkdirSync(path.join(layout, "blobs", "sha256"), { recursive: true }); + const blob = (bytes: string) => { + writeFileSync(path.join(layout, "blobs", "sha256", sha256(bytes).slice(7)), bytes); + return { digest: sha256(bytes), size: bytes.length }; + }; + const manifest = blob(JSON.stringify({ + schemaVersion: 2, config: blob(`config of ${name}`), + layers: layers.map((layer) => ({ mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", ...blob(layer) })), + })); + writeFileSync(path.join(layout, "index.json"), JSON.stringify({ + schemaVersion: 2, manifests: [{ mediaType: "application/vnd.oci.image.manifest.v1+json", ...manifest }], + })); + writeFileSync(path.join(layout, "oci-layout"), '{"imageLayoutVersion":"1.0.0"}'); + const archive = path.join(directory, `${name}.oci.tar`); + // The members buildx writes. `tar -C layout .` would name them ./index.json and ./blobs/... + expect(spawnSync("tar", ["-cf", archive, "-C", layout, "blobs", "index.json", "oci-layout"]).status).toBe(0); + writeFileSync(path.join(directory, `${name}.metadata.json`), JSON.stringify({ "containerimage.digest": manifest.digest })); + return manifest.digest; +} + +/** What one build leg uploads: one layout with both variants, and the record of what it verified. */ +async function leg(directory: string, architecture: string, options: { + inputsDigest?: string; sourceRevision?: string; publication?: boolean; slim?: string[]; full?: string[]; +} = {}) { + const { inputsDigest = INPUTS, sourceRevision = REVISION, publication = true } = options; + const slim = options.slim ?? [`large ${architecture}`, `rest ${architecture}`]; + const full = options.full ?? [...slim, `archives ${architecture}`, `overlay ${architecture}`]; + const root = path.join(directory, architecture); + mkdirSync(path.join(root, "tree"), { recursive: true }); + const images = { slim: ociArchive(root, "slim", slim), full: ociArchive(root, "full", full) }; + writeFileSync(path.join(root, "tree", "runner-image.json"), JSON.stringify({ + inputsDigest: { digest: inputsDigest }, sourceRevision, publication, platform: { os: "linux", architecture }, + })); + const record = await createLegRecord({ + architecture, + archives: { slim: path.join(root, "slim.oci.tar"), full: path.join(root, "full.oci.tar") }, + metadataFiles: { slim: path.join(root, "slim.metadata.json"), full: path.join(root, "full.metadata.json") }, + tree: path.join(root, "tree"), + layout: path.join(root, "images.oci.tar"), + }); + writeFileSync(path.join(root, "record.json"), JSON.stringify(record)); + return { record, images }; +} + +async function legs(options: Record = {}) { + const directory = path.join(scratch, String(sequence += 1)); + const images: Record = {}; + for (const architecture of ["amd64", "arm64"]) ({ images: images[architecture] } = await leg(directory, architecture, options[architecture])); + return { directory, images }; +} + +describe("verified build legs", () => { + it("accepts both variants of each architecture for this run's inputs and commit", async () => { + const { directory, images } = await legs(); + await expect(verifyLegRecords({ directory, inputsDigest: INPUTS, sourceRevision: REVISION })).resolves.toEqual({ + slim_amd64: images.amd64.slim, full_amd64: images.amd64.full, slim_arm64: images.arm64.slim, full_arm64: images.arm64.full, + }); + }); + + it("hands over one layout that names both variants and stores their shared layers once", async () => { + const { directory, images } = await legs(); + const layout = path.join(directory, "amd64", "images.oci.tar"); + const read = (member: string) => spawnSync("tar", ["-xOf", layout, member], { encoding: "utf8" }).stdout; + expect(JSON.parse(read("index.json")).manifests.map((entry: { digest: string; annotations: Record }) => + [entry.annotations["org.opencontainers.image.ref.name"], entry.digest])).toEqual([["slim", images.amd64.slim], ["full", images.amd64.full]]); + const blobs = spawnSync("tar", ["-tf", layout], { encoding: "utf8" }).stdout.split("\n").filter((name) => /^blobs\/sha256\/[0-9a-f]{64}$/.test(name)); + // Two manifests, two configs, and four layers: the two shared ones are not stored twice. + expect(blobs).toHaveLength(8); + expect(new Set(blobs).size).toBe(8); + expect(JSON.parse(read(`blobs/sha256/${images.amd64.full.slice(7)}`)).layers.slice(0, 2)) + .toEqual(JSON.parse(read(`blobs/sha256/${images.amd64.slim.slice(7)}`)).layers); + // Each image can be read alone by its name, with every blob under its digest: what a push does. + for (const variant of ["slim", "full"] as const) { + const image = readOciArchive(layout, { ref: variant }); + expect(image.descriptor.digest).toBe(images.amd64[variant]); + await expect(image.verifyBlobs()).resolves.toBeUndefined(); + } + expect(() => readOciArchive(layout)).toThrow(/must hold exactly one image manifest/); + expect(() => readOciArchive(layout, { ref: "candidate" })).toThrow(/must hold exactly one image named candidate/); + }); + + it("does not record an archive that lacks a blob of its image, or whose members have other names", async () => { + const directory = path.join(scratch, String(sequence += 1)); + await leg(directory, "amd64"); + const root = path.join(directory, "amd64"); + const input = { + architecture: "amd64", + archives: { slim: path.join(root, "slim.oci.tar"), full: path.join(root, "full.oci.tar") }, + metadataFiles: { slim: path.join(root, "slim.metadata.json"), full: path.join(root, "full.metadata.json") }, + tree: path.join(root, "tree"), + layout: path.join(root, "again.oci.tar"), + }; + // The full archive without one of its layers. + const layer = readOciArchive(input.archives.full).layers[3].digest; + rmSync(path.join(root, "full-layout", "blobs", "sha256", layer.slice(7))); + expect(spawnSync("tar", ["-cf", input.archives.full, "-C", path.join(root, "full-layout"), "blobs", "index.json", "oci-layout"]).status).toBe(0); + await expect(createLegRecord(input)).rejects.toThrow(/has no member blobs\/sha256\//); + // An archive of `.`: the names a registry tool does not find. + expect(spawnSync("tar", ["-cf", input.archives.slim, "-C", path.join(root, "slim-layout"), "."]).status).toBe(0); + await expect(createLegRecord(input)).rejects.toThrow(/has no member index\.json/); + }); + + it("refuses a leg that built other inputs or another commit", async () => { + const otherInputs = await legs({ arm64: { inputsDigest: "2".repeat(64) } }); + await expect(verifyLegRecords({ directory: otherInputs.directory, inputsDigest: INPUTS, sourceRevision: REVISION })) + .rejects.toThrow(/arm64: the image was built from other inputs/); + const otherCommit = await legs({ amd64: { sourceRevision: "b".repeat(40) } }); + await expect(verifyLegRecords({ directory: otherCommit.directory, inputsDigest: INPUTS, sourceRevision: REVISION })) + .rejects.toThrow(/amd64: the image was built at another commit/); + }); + + it("refuses a layout that changed after its leg recorded it, a record without a variant, and a missing leg", async () => { + const { directory } = await legs(); + appendFileSync(path.join(directory, "amd64", "images.oci.tar"), "tampered"); + await expect(verifyLegRecords({ directory, inputsDigest: INPUTS, sourceRevision: REVISION })) + .rejects.toThrow(/amd64: the image layout does not match its record/); + const oneVariant = await legs(); + const recordPath = path.join(oneVariant.directory, "arm64", "record.json"); + const record = JSON.parse(readFileSync(recordPath, "utf8")); + writeFileSync(recordPath, JSON.stringify({ ...record, images: { full: record.images.full } })); + await expect(verifyLegRecords({ directory: oneVariant.directory, inputsDigest: INPUTS, sourceRevision: REVISION })) + .rejects.toThrow(/arm64: the record has no slim image digest/); + const missing = await legs(); + rmSync(path.join(missing.directory, "arm64"), { recursive: true }); + await expect(verifyLegRecords({ directory: missing.directory, inputsDigest: INPUTS, sourceRevision: REVISION })).rejects.toThrow(); + }); + + it.each<[string, object, RegExp]>([ + ["a full image that does not start with the slim image's layers", { full: ["another large", "rest amd64", "archives", "overlay"] }, /does not start with the layers of the slim image/], + ["a full image with the slim layers in another order", { full: ["rest amd64", "large amd64", "archives", "overlay"] }, /does not start with the layers of the slim image/], + ["a full image that is the slim image", { full: ["large amd64", "rest amd64"] }, /has no layer of its own/], + ["a build that is not a publication build", { publication: false }, /Only a publication build can be recorded/], + ])("does not record %s", async (_label, options, expected) => { + await expect(leg(path.join(scratch, String(sequence += 1)), "amd64", options)).rejects.toThrow(expected); + }); + + it("does not record a tree that is for another architecture, or an archive that is not the image the build reported", async () => { + const directory = path.join(scratch, String(sequence += 1)); + await leg(directory, "amd64"); + const root = path.join(directory, "amd64"); + const input = { + architecture: "amd64", + archives: { slim: path.join(root, "slim.oci.tar"), full: path.join(root, "full.oci.tar") }, + metadataFiles: { slim: path.join(root, "slim.metadata.json"), full: path.join(root, "full.metadata.json") }, + tree: path.join(root, "tree"), + layout: path.join(root, "again.oci.tar"), + }; + await expect(createLegRecord({ ...input, architecture: "arm64" })).rejects.toThrow(/another architecture/); + await expect(createLegRecord({ ...input, archives: { slim: input.archives.full, full: input.archives.full } })) + .rejects.toThrow(/The slim archive is not the image its build reported/); + }); +}); + +describe("registry lookups", () => { + const index = Buffer.from(JSON.stringify({ schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json", manifests: [] })); + const respond = (status: number, body: Buffer | string = "", headers: Record = {}) => + new Response(typeof body === "string" ? body : new Uint8Array(body), { status, headers }); + function registry(manifest: () => Response, token: () => Response = () => respond(200, JSON.stringify({ token: "t" }))) { + const requests: { url: string; headers: Record }[] = []; + const fetchImpl = async (url: string, init: { headers: Record }) => { + requests.push({ url, headers: init.headers }); + return url.includes("/token?") ? token() : manifest(); + }; + return { requests, options: { fetchImpl, sleep: async () => {} } }; + } + + it("reports a tag that exists, with the digest of the registry bytes", async () => { + const { requests, options } = registry(() => respond(200, index, { "docker-content-digest": sha256(index) })); + const result = await readRunnerImageManifest(`inputs-${INPUTS}`, options); + expect(result).toMatchObject({ state: "present", digest: sha256(index) }); + expect(requests.map((request) => request.url)).toEqual([ + "https://ghcr.io/token?service=ghcr.io&scope=repository:paperclipai/paperclip-runner:pull", + `https://ghcr.io/v2/paperclipai/paperclip-runner/manifests/inputs-${INPUTS}`, + ]); + expect(requests[0].headers.authorization).toBeUndefined(); + expect(requests[1].headers.authorization).toBe("Bearer t"); + }); + + it("reports a missing tag only when the registry says it is missing", async () => { + await expect(readRunnerImageManifest("inputs-x", registry(() => respond(404)).options)).resolves.toEqual({ state: "absent" }); + }); + + it.each<[string, () => Response, (() => Response) | undefined]>([ + ["the token request is refused", () => respond(200, index), () => respond(403)], + ["the token response has no token", () => respond(200, index), () => respond(200, "{}")], + ["the manifest request is unauthorized", () => respond(401), undefined], + ["the manifest request is denied", () => respond(403), undefined], + ["the registry keeps failing", () => respond(503), undefined], + ])("does not guess when %s", async (_label, manifest, token) => { + const result = await readRunnerImageManifest("inputs-x", registry(manifest, token).options); + expect(result.state).toBe("unknown"); + }); + + it("retries a transient registry error", async () => { + let calls = 0; + const { options } = registry(() => ((calls += 1) < 3 ? respond(502) : respond(404))); + await expect(readRunnerImageManifest("inputs-x", options)).resolves.toEqual({ state: "absent" }); + expect(calls).toBe(3); + }); + + it.each<[string, () => Promise]>([ + ["the registry cannot be reached", async () => { throw new TypeError("fetch failed"); }], + ["the token response is not JSON", async () => respond(200, "gateway")], + ])("does not guess, and does not crash, when %s", async (_label, token) => { + let calls = 0; + const fetchImpl = async () => { calls += 1; return token(); }; + const result = await readRunnerImageManifest("inputs-x", { fetchImpl, sleep: async () => {} }); + expect(result.state).toBe("unknown"); + expect(calls).toBeGreaterThanOrEqual(1); + }); + + it("does not guess when the manifest request cannot be completed", async () => { + let manifests = 0; + const fetchImpl = async (url: string) => { + if (url.includes("/token?")) return respond(200, JSON.stringify({ token: "t" })); + manifests += 1; + throw new TypeError("fetch failed"); + }; + await expect(readRunnerImageManifest("inputs-x", { fetchImpl, sleep: async () => {} })).resolves.toMatchObject({ state: "unknown" }); + expect(manifests).toBe(5); + }); + + it("authenticates with the credentials it is given and rejects bytes that do not match their digest", async () => { + const { requests, options } = registry(() => respond(200, index, { "docker-content-digest": sha256("other") })); + await expect(readRunnerImageManifest("inputs-x", { ...options, credentials: { username: "u", password: "p" } })) + .rejects.toThrow(/does not match the manifest bytes/); + expect(requests[0].headers.authorization).toBe(`Basic ${Buffer.from("u:p").toString("base64")}`); + }); +}); + +describe("published index", () => { + const legDigests = { amd64: sha256("manifest amd64"), arm64: sha256("manifest arm64") }; + const entry = (architecture: string, digest: string) => ({ + mediaType: "application/vnd.oci.image.manifest.v1+json", digest, size: 500, platform: { os: "linux", architecture }, + }); + const bytes = (manifests: object[]) => Buffer.from(JSON.stringify({ + schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json", manifests, + })); + + it("accepts an index of exactly the two verified images and returns its digest", () => { + const index = bytes([entry("amd64", legDigests.amd64), entry("arm64", legDigests.arm64)]); + expect(verifyPublishedIndex(index, legDigests)).toBe(sha256(index)); + }); + + it("requires the full index, and no other, to name its slim index", () => { + const slimDigest = sha256("slim index"); + const manifests = [entry("amd64", legDigests.amd64), entry("arm64", legDigests.arm64)]; + const full = Buffer.from(JSON.stringify({ + schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json", manifests, annotations: { [SLIM_DIGEST_ANNOTATION]: slimDigest }, + })); + expect(verifyPublishedIndex(full, legDigests, { slimDigest })).toBe(sha256(full)); + expect(() => verifyPublishedIndex(full, legDigests, { slimDigest: sha256("an earlier run's slim index") })) + .toThrow(/The full index does not name the slim index published with it/); + expect(() => verifyPublishedIndex(bytes(manifests), legDigests, { slimDigest })).toThrow(/does not name the slim index/); + expect(() => verifyPublishedIndex(full, legDigests)).toThrow(/Only the full index may name a slim index/); + }); + + const both = [entry("amd64", legDigests.amd64), entry("arm64", legDigests.arm64)]; + const extra = (platform: object | undefined) => ({ + mediaType: "application/vnd.oci.image.manifest.v1+json", digest: sha256("extra"), size: 500, ...(platform ? { platform } : {}), + }); + + it.each<[string, object[]]>([ + ["an image no leg verified", [entry("amd64", legDigests.amd64), entry("arm64", sha256("unverified"))]], + ["the images under the wrong architectures", [entry("amd64", legDigests.arm64), entry("arm64", legDigests.amd64)]], + ["one architecture only", [entry("amd64", legDigests.amd64)]], + ["one architecture twice", [entry("amd64", legDigests.amd64), entry("amd64", legDigests.amd64), entry("arm64", legDigests.arm64)]], + ["an attestation manifest beside the images", [...both, extra({ os: "unknown", architecture: "unknown" })]], + ["an entry without a platform", [...both, extra(undefined)]], + ["an image for another operating system", [...both, extra({ os: "windows", architecture: "amd64" })]], + ["an image for another architecture", [...both, extra({ os: "linux", architecture: "riscv64" })]], + ["a variant on the amd64 image", [{ ...entry("amd64", legDigests.amd64), platform: { os: "linux", architecture: "amd64", variant: "v8" } }, + entry("arm64", legDigests.arm64)]], + ["another arm64 variant", [entry("amd64", legDigests.amd64), + { ...entry("arm64", legDigests.arm64), platform: { os: "linux", architecture: "arm64", variant: "v7" } }]], + ])("refuses an index with %s", (_label, manifests) => { + expect(() => verifyPublishedIndex(bytes(manifests), legDigests)).toThrow(); + }); + + it("accepts linux/arm64 with the variant v8, which some BuildKit versions record", () => { + const index = bytes([entry("amd64", legDigests.amd64), + { ...entry("arm64", legDigests.arm64), platform: { os: "linux", architecture: "arm64", variant: "v8" } }]); + expect(verifyPublishedIndex(index, legDigests)).toBe(sha256(index)); + }); + + it("needs a recorded image for each architecture, and no other", () => { + expect(() => verifyPublishedIndex(bytes(both), { amd64: legDigests.amd64 })).toThrow(/one recorded image for each architecture/); + expect(() => verifyPublishedIndex(bytes(both), { ...legDigests, riscv64: sha256("extra") })).toThrow(/one recorded image for each architecture/); + }); +}); + +describe("lockfile state", () => { + function context() { + const directory = path.join(scratch, String(sequence += 1)); + mkdirSync(path.join(directory, "packages", "paperclip-runner"), { recursive: true }); + writeFileSync(path.join(directory, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n"); + writeFileSync(path.join(directory, "packages", "paperclip-runner", "package.json"), "{}\n"); + return directory; + } + const answer = (status: number | null, stderr = "", extra: object = {}) => { + const calls: { command: string; args: string[]; options: { cwd?: string } }[] = []; + const run = (command: string, args: string[], options: object) => { + calls.push({ command, args, options }); + return { status, stdout: "", stderr, ...extra }; + }; + return { calls, run }; + }; + + it("asks pnpm the image build's own question, offline and without installing", () => { + const directory = context(); + const { calls, run } = answer(0); + expect(runnerLockState(directory, { run }).state).toBe("current"); + expect(calls).toHaveLength(1); + expect(calls[0].command).toBe("pnpm"); + expect(calls[0].options.cwd).toBe(directory); + expect(calls[0].args).toEqual(["install", "--frozen-lockfile", "--lockfile-only", "--offline", "--ignore-scripts", + "--filter", "@paperclipai/paperclip-runner..."]); + }); + + it.each([ + [" ERR_PNPM_OUTDATED_LOCKFILE Cannot install with \"frozen-lockfile\" because pnpm-lock.yaml is not up to date with /packages/paperclip-runner/package.json"], + [" ERR_PNPM_LOCKFILE_CONFIG_MISMATCH Cannot proceed with the frozen installation. The current \"patchedDependencies\" configuration doesn't match the value found in the lockfile"], + ])("reports a lockfile the manifests have moved past as behind: %s", (stderr) => { + const result = runnerLockState(context(), { run: answer(1, stderr).run }); + expect(result.state).toBe("behind"); + expect(result.detail).toContain("ERR_PNPM_"); + }); + + it.each<[string, number | null, string, object]>([ + ["a broken lockfile", 1, " ERR_PNPM_BROKEN_LOCKFILE The lockfile is broken", {}], + ["a failure pnpm does not name", 1, "Killed", {}], + ["a pnpm that was killed", null, "", {}], + ["a pnpm that cannot be started", null, "", { error: new Error("spawn pnpm ENOENT") }], + ])("fails for %s instead of calling it behind", (_label, status, stderr, extra) => { + expect(() => runnerLockState(context(), { run: answer(status, stderr, extra).run })).toThrow(); + }); + + it("fails for a directory that is not a Runner build context", () => { + const directory = context(); + rmSync(path.join(directory, "pnpm-lock.yaml")); + expect(() => runnerLockState(directory, { run: answer(0).run })).toThrow(/no pnpm-lock\.yaml/); + }); + + it("refuses a checkout, whose lockfile pnpm could rewrite, and does not start pnpm there", () => { + const directory = context(); + mkdirSync(path.join(directory, ".git")); + const { calls, run } = answer(0); + expect(() => runnerLockState(directory, { run })).toThrow(/is a checkout: pnpm can rewrite its pnpm-lock\.yaml/); + expect(calls).toEqual([]); + }); +}); + +describe("publication state", () => { + type Manifest = Awaited>; + const index = (label: string, slimDigest?: string) => { + const bytes = Buffer.from(JSON.stringify({ schemaVersion: 2, label, ...(slimDigest ? { annotations: { [SLIM_DIGEST_ANNOTATION]: slimDigest } } : {}) })); + return { state: "present" as const, bytes, digest: sha256(bytes) }; + }; + const registry = (tags: Record) => async (reference: string): Promise => tags[reference] ?? { state: "absent" }; + const slim = index("slim of this run"); + const full = index("full of this run", slim.digest); + const unknown = { state: "unknown" as const, detail: "manifest request: HTTP 503" }; + + it("is published only when the full tag exists and names the image the slim tag points at", async () => { + await expect(runnerPublicationState(INPUTS, registry({ [TAGS.slim]: slim, [TAGS.full]: full }))) + .resolves.toMatchObject({ state: "published", slim: slim.digest, full: full.digest }); + await expect(runnerPublicationState(INPUTS, registry({}))).resolves.toMatchObject({ state: "absent" }); + }); + + it("treats a slim tag without the full tag as a publication that did not finish", async () => { + // A run stopped between the two tags. The inputs are not published, so + // the next run builds, and its publish job replaces the slim tag. + const orphan = index("slim of the run that stopped"); + await expect(runnerPublicationState(INPUTS, registry({ [TAGS.slim]: orphan }))) + .resolves.toMatchObject({ state: "incomplete", slim: orphan.digest }); + await expect(runPublishCommand(["publication-state", "--inputs-digest", INPUTS, "--strict"], { env: {}, read: registry({ [TAGS.slim]: orphan }), log: () => {} })) + .resolves.toEqual({ state: "incomplete", exists: false, publish: true }); + }); + + it.each<[string, Record]>([ + ["the slim tag is missing", { [TAGS.full]: full }], + ["the slim tag points at another run's image", { [TAGS.full]: full, [TAGS.slim]: index("slim of another run") }], + ["the full index names no slim image", { [TAGS.full]: index("full without the annotation"), [TAGS.slim]: slim }], + ["the full tag is not an index", { [TAGS.full]: { state: "present", bytes: Buffer.from("not JSON"), digest: sha256("not JSON") }, [TAGS.slim]: slim }], + ])("is broken, and fails every run, when the full tag exists but %s", async (_label, tags) => { + await expect(runnerPublicationState(INPUTS, registry(tags))).resolves.toMatchObject({ state: "broken" }); + // Also without --strict: no run can repair it, and the message says what an operator must do. + await expect(runPublishCommand(["publication-state", "--inputs-digest", INPUTS], { env: {}, read: registry(tags), log: () => {} })) + .rejects.toThrow(/No run can repair this.*Delete the package versions tagged inputs-1{64} and inputs-1{64}-slim, then re-run/); + }); + + it.each<[string, Record]>([ + ["the full tag", { [TAGS.full]: unknown }], + ["the slim tag, with the full tag absent", { [TAGS.slim]: unknown }], + ["the slim tag, with the full tag present", { [TAGS.full]: full, [TAGS.slim]: unknown }], + ])("does not guess when the registry gives no clear answer for %s", async (_label, tags) => { + await expect(runnerPublicationState(INPUTS, registry(tags))).resolves.toMatchObject({ state: "unknown" }); + }); + + it("lets a run whose own slim tag is already written finish, and never pairs a full image with another run's slim", async () => { + // What the publish job does, against a registry a stopped run left behind. + const tags: Record = { [TAGS.slim]: index("slim of the run that stopped") }; + const read = registry(tags); + const gate = await runPublishCommand(["publication-state", "--inputs-digest", INPUTS, "--strict"], { env: {}, read, log: () => {} }); + expect(gate.publish).toBe(true); + // It writes its own slim tag over the orphan, then the full tag that names that slim. + tags[TAGS.slim] = slim; + tags[TAGS.full] = full; + await expect(runPublishCommand(["publication-state", "--inputs-digest", INPUTS, "--strict"], { env: {}, read, log: () => {} })) + .resolves.toEqual({ state: "published", exists: true, publish: false }); + // Had it kept the orphan, the pair would not hold. + tags[TAGS.slim] = index("slim of the run that stopped"); + await expect(runPublishCommand(["publication-state", "--inputs-digest", INPUTS, "--strict"], { env: {}, read, log: () => {} })).rejects.toThrow(/No run can repair this/); + }); +}); + +describe("command line", () => { + const unknown = async () => ({ state: "unknown" as const, detail: "token request: HTTP 403" }); + const absent = async () => ({ state: "absent" as const }); + const quiet = { env: {}, log: () => {} }; + const state = ["publication-state", "--inputs-digest", INPUTS]; + + it("builds on an inconclusive lookup and fails on one when it is strict", async () => { + const lines: string[] = []; + await expect(runPublishCommand(state, { env: {}, read: unknown, log: (line) => lines.push(line) })) + .resolves.toEqual({ state: "unknown", exists: false, publish: false }); + expect(lines.join("\n")).toMatch(/::notice title=Runner image lookup inconclusive::/); + await expect(runPublishCommand([...state, "--strict"], { ...quiet, read: unknown })) + .rejects.toThrow(/Could not determine whether .*paperclip-runner:inputs-1{64} is published \(inputs-1{64}: token request: HTTP 403\)/); + await expect(runPublishCommand([...state, "--strict"], { ...quiet, read: absent })) + .resolves.toEqual({ state: "absent", exists: false, publish: true }); + }); + + it("stays inconclusive, not failed, when the registry cannot be reached", async () => { + const offline = (reference: string) => readRunnerImageManifest(reference, { + fetchImpl: async () => { throw new TypeError("fetch failed"); }, sleep: async () => {}, + }); + await expect(runPublishCommand(state, { ...quiet, read: offline })).resolves.toEqual({ state: "unknown", exists: false, publish: false }); + await expect(runPublishCommand([...state, "--strict"], { ...quiet, read: offline })).rejects.toThrow(/Could not determine/); + }); + + it("passes the registry credentials from the environment, and none when one is missing", async () => { + const seen: unknown[] = []; + const read = async (_reference: string, options?: { credentials?: unknown }) => { seen.push(options?.credentials); return { state: "absent" as const }; }; + await runPublishCommand(state, { env: { RUNNER_IMAGE_REGISTRY_USERNAME: "u", RUNNER_IMAGE_REGISTRY_PASSWORD: "p" }, read, log: () => {} }); + await runPublishCommand(state, { env: { RUNNER_IMAGE_REGISTRY_USERNAME: "u" }, read, log: () => {} }); + expect(new Set(seen.map((credentials) => JSON.stringify(credentials)))).toEqual(new Set(['{"username":"u","password":"p"}', "null"])); + }); + + it("verifies the index of each variant against the recorded legs and against the expected digest", async () => { + const { directory, images } = await legs(); + const indexOf = (variant: "slim" | "full", annotations?: Record) => Buffer.from(JSON.stringify({ + schemaVersion: 2, mediaType: "application/vnd.oci.image.index.v1+json", + manifests: ["amd64", "arm64"].map((architecture) => ({ + mediaType: "application/vnd.oci.image.manifest.v1+json", digest: images[architecture][variant], size: 500, + platform: { os: "linux", architecture }, + })), + ...(annotations ? { annotations } : {}), + })); + const slim = indexOf("slim"); + const full = indexOf("full", { [SLIM_DIGEST_ANNOTATION]: sha256(slim) }); + const serve = (bytes: Buffer) => async () => ({ state: "present" as const, bytes, digest: sha256(bytes) }); + const verify = (...args: string[]) => ["verify-index", ...args, "--dir", directory]; + await expect(runPublishCommand(verify("candidate-x-slim", "--variant", "slim"), { ...quiet, read: serve(slim) })).resolves.toEqual({ digest: sha256(slim) }); + await expect(runPublishCommand(verify("candidate-x", "--variant", "full", "--slim-digest", sha256(slim), "--expect", sha256(full)), { ...quiet, read: serve(full) })) + .resolves.toEqual({ digest: sha256(full) }); + // The slim index is not the full one, and the other way round. + await expect(runPublishCommand(verify("inputs-x", "--variant", "full", "--slim-digest", sha256(slim)), { ...quiet, read: serve(slim) })).rejects.toThrow(); + await expect(runPublishCommand(verify("inputs-x-slim", "--variant", "slim"), { ...quiet, read: serve(full) })).rejects.toThrow(); + await expect(runPublishCommand(verify("inputs-x", "--variant", "full"), { ...quiet, read: serve(full) })) + .rejects.toThrow(/--slim-digest is given for the full index, and only for it/); + await expect(runPublishCommand(verify("inputs-x-slim", "--variant", "slim", "--slim-digest", sha256(slim)), { ...quiet, read: serve(slim) })) + .rejects.toThrow(/--slim-digest is given for the full index, and only for it/); + await expect(runPublishCommand(verify("inputs-x-slim", "--variant", "slim", "--expect", sha256("another index")), { ...quiet, read: serve(slim) })) + .rejects.toThrow(/inputs-x-slim resolves to sha256:[0-9a-f]{64}, expected sha256:[0-9a-f]{64}/); + await expect(runPublishCommand(verify("inputs-x-slim", "--variant", "slim", "--expect", "latest"), { ...quiet, read: serve(slim) })) + .rejects.toThrow(/--expect must be a sha256 digest/); + await expect(runPublishCommand(verify("inputs-x-slim", "--variant", "both"), { ...quiet, read: serve(slim) })).rejects.toThrow(/--variant must be slim or full/); + await expect(runPublishCommand(verify("inputs-x-slim", "--variant", "slim"), { ...quiet, read: absent })).rejects.toThrow(/is absent/); + }); + + it("reports the lockfile state and fails when the check itself fails", async () => { + await expect(runPublishCommand(["lock-state", "--context", "/context"], { ...quiet, lockState: () => ({ state: "behind", detail: "ERR_PNPM_OUTDATED_LOCKFILE" }) })) + .resolves.toEqual({ state: "behind" }); + await expect(runPublishCommand(["lock-state", "--context", "/context"], { ...quiet, lockState: () => { throw new Error("broken"); } })) + .rejects.toThrow("broken"); + }); + + it.each<[string, string[], RegExp]>([ + ["an empty --expect", ["verify-index", "inputs-x", "--dir", "d", "--variant", "slim", "--expect", ""], /--expect needs a value/], + ["an --expect with no value", ["verify-index", "inputs-x", "--dir", "d", "--variant", "slim", "--expect"], /--expect needs a value/], + ["an index check without a variant", ["verify-index", "inputs-x", "--dir", "d"], /verify-index needs --variant/], + ["a misspelt flag", [...state, "--strikt"], /publication-state does not take --strikt/], + ["a flag of another command", [...state, "--expect", "sha256:x"], /publication-state does not take --expect/], + ["a repeated flag", ["verify-index", "inputs-x", "--dir", "a", "--dir", "b"], /--dir is given twice/], + ["a missing required flag", ["verify-legs", "--dir", "d", "--inputs-digest", INPUTS], /verify-legs needs --source-revision/], + ["an empty required value", ["verify-legs", "--dir", "d", "--inputs-digest", "", "--source-revision", REVISION], /--inputs-digest needs a value/], + ["a lookup without the inputs digest", ["publication-state", "--strict"], /publication-state needs --inputs-digest/], + ["a lookup by tag", ["publication-state", "inputs-x", "--inputs-digest", INPUTS], /takes no positional argument/], + ["a missing tag", ["verify-index", "--dir", "d", "--variant", "slim"], /takes exactly one tag or digest/], + ["an empty tag", ["verify-index", "", "--dir", "d", "--variant", "slim"], /not a tag or digest/], + ["a record of one variant", ["record", "--arch", "amd64", "--full-archive", "f", "--full-metadata", "m", "--tree", "t", "--layout", "l", "--out", "o"], /record needs --slim-archive/], + ["an unknown command", ["tag-state", "inputs-x"], /Usage/], + ["no command", [], /Usage/], + ])("rejects %s", (_label, argv, expected) => { + expect(() => parsePublishArguments(argv[0], argv.slice(1))).toThrow(expected); + }); +}); diff --git a/server/src/__tests__/runner-image-workflow.test.ts b/server/src/__tests__/runner-image-workflow.test.ts index a1f9e565e68..fba9724ad60 100644 --- a/server/src/__tests__/runner-image-workflow.test.ts +++ b/server/src/__tests__/runner-image-workflow.test.ts @@ -6,9 +6,9 @@ import { RUNNER_IMAGE_CONTEXT_INPUTS } from "../../../scripts/runner-image-ident /** * Security properties of the Runner image workflow. The build runs dependency - * lifecycle scripts, so what the jobs may do matters more than how their - * steps are worded: these tests pin permissions, credentials and triggers, - * not step names or order. + * lifecycle scripts and the workflow publishes a signed image, so what each + * job may do matters more than how its steps are worded: these tests pin + * permissions, credentials and gating, not step names or order. */ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); @@ -25,17 +25,89 @@ const permissions = (job: string) => const scripts = [...workflow.matchAll(/^( +)run: (\|\n((?:\1 {2,}.*\n|\n)+)|.+\n)/gm)].map((match) => match[3] ?? match[2]); describe("Runner image workflow", () => { - it("grants nothing at the workflow level and only read access to contents in each job", () => { + const repository = "github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064'"; + const canonical = `${repository} && github.event_name == 'push' && github.ref == 'refs/heads/master'`; + const jobCondition = (job: string) => /^ {4}if: (.+)$/m.exec(job)?.[1] ?? ""; + /** The steps of a job, without the job's own header. */ + const steps = (job: string) => job.slice(job.indexOf("\n steps:\n")).split(/\n(?= {6}- )/).slice(1); + const credential = /github\.token|secrets\.|GITHUB_TOKEN|REGISTRY_PASSWORD/; + + it("lets only the publish job write to the registry or sign", () => { expect(workflow).toMatch(/^permissions: \{\}$/m); - expect(Object.keys(jobs)).toEqual(["identity", "build"]); - for (const job of Object.values(jobs)) expect(permissions(job)).toEqual(["contents: read"]); + expect(Object.keys(jobs)).toEqual(["identity", "build", "publish"]); + expect(permissions(jobs.identity)).toEqual(["contents: read", "packages: read"]); + expect(permissions(jobs.build)).toEqual(["contents: read", "packages: read"]); + expect(permissions(jobs.publish)).toEqual(["contents: read", "packages: write", "id-token: write", "attestations: write"]); + for (const job of [jobs.identity, jobs.build]) { + expect(job).not.toMatch(/docker\/login-action|skopeo|imagetools create|actions\/attest|secrets\.|--push|push=true/); + } + expect(workflow).not.toMatch(/pull_request_target|workflow_run/); }); - it("has no credential, no registry access and no way to publish", () => { - expect(workflow).not.toMatch(/github\.token|secrets\.|GITHUB_TOKEN|docker\/login-action|packages:|id-token|attestations/); - expect(workflow).not.toMatch(/--push|push=true|skopeo|imagetools|type=image|type=registry|cache-to|cache-from|type=gha/); - const triggers = workflow.slice(workflow.indexOf("\non:\n"), workflow.indexOf("\npermissions:")); - expect([...triggers.matchAll(/^ {2}([a-z_]+):/gm)].map((match) => match[1])).toEqual(["pull_request", "workflow_dispatch"]); + it("publishes, and keeps the images for publication, only for a master push in the canonical repository", () => { + expect(jobCondition(jobs.publish).startsWith(`${canonical} && `)).toBe(true); + expect(jobs.publish).toContain(" needs: [identity, build]\n"); + expect(jobs.build).toContain(`KEEP_IMAGE: \${{ ${canonical} }}`); + // The one step that hands the images to the publish job is behind that switch. + const uploading = steps(jobs.build).filter((step) => step.includes("upload-artifact")); + expect(uploading).toHaveLength(1); + expect(uploading[0]).toMatch(/^ {8}if: env\.KEEP_IMAGE == 'true' && /m); + // The record is written on every event, so a pull request exercises it. It needs no credential. + const recording = steps(jobs.build).filter((step) => step.includes("runner-image-publish.mjs record")); + expect(recording).toHaveLength(1); + expect(recording[0]).not.toMatch(/KEEP_IMAGE/); + expect(recording[0]).not.toMatch(credential); + // A fork's own pushes run nothing; no job overrides a skipped or failed dependency. + expect(jobCondition(jobs.identity)).toBe(`github.event_name != 'push' || (${repository})`); + expect(workflow).not.toMatch(/always\(\)|!cancelled\(\)|failure\(\)|continue-on-error/); + }); + + it("keeps the build legs free of credentials and of any registry build cache", () => { + // No job-level token: the block before the steps holds none, in any job. + for (const job of Object.values(jobs)) expect(job.slice(0, job.indexOf("\n steps:\n"))).not.toMatch(credential); + // The steps that build, check and smoke the image never receive one. + const working = steps(jobs.build).filter((step) => /docker buildx build|runner-image-layout\.mjs|pnpm |upload-artifact/.test(step)); + expect(working.length).toBeGreaterThanOrEqual(4); + for (const step of working) expect(step).not.toMatch(credential); + // The one token the legs can see only reads packages, and only a lookup gets it. + for (const step of steps(jobs.build).filter((entry) => credential.test(entry))) { + expect(step).toContain("runner-image-publish.mjs publication-state"); + expect(step).not.toContain("--strict"); + } + expect(workflow).not.toMatch(/cache-to|cache-from|type=registry|type=gha/); + }); + + it("writes the full tag last, after a strict lookup and after both images are attested", () => { + // The full tag marks a complete publication, so nothing may follow it + // that can fail to write a tag, and no tag is written on a guess. + const publish = jobs.publish; + const order = [ + "publication-state --inputs-digest \"$DIGEST\" --strict\n", + "subject-digest: ${{ steps.slim.outputs.digest }}", + "subject-digest: ${{ steps.full.outputs.digest }}", + 'imagetools create -t "$IMAGE:inputs-$DIGEST-slim" "$IMAGE@$SLIM_INDEX"', + 'imagetools create -t "$IMAGE:inputs-$DIGEST" "$IMAGE@$FULL_INDEX"', + ].map((text) => publish.indexOf(text)); + expect(order.every((position) => position >= 0)).toBe(true); + expect([...order].sort((a, b) => a - b)).toEqual(order); + // The final tags are written in one place each; every other tag is a candidate. + expect([...workflow.matchAll(/-t "\$IMAGE:([a-z]+)-/g)].map((match) => match[1])).toEqual(["candidate", "candidate", "inputs", "inputs"]); + // Every step that writes a tag or signs runs only after the strict lookup allowed it. + const writing = steps(publish).filter((step) => /imagetools create|actions\/attest/.test(step)); + expect(writing).toHaveLength(5); + for (const step of writing) expect(step).toMatch(/^ {8}if: steps\.gate\.outputs\.publish == 'true'$/m); + expect(publish).toContain("group: runner-image-publish-${{ needs.identity.outputs.digest }}\n cancel-in-progress: false"); + }); + + it("never resolves the lockfile on a master push, and neither builds nor publishes when it is behind", () => { + const resolving = steps(jobs.identity).filter((step) => step.includes("pnpm install")); + expect(resolving).toHaveLength(1); + expect(resolving[0]).toMatch(/^ {8}if: github\.event_name != 'push'$/m); + expect(workflow.match(/pnpm install/g)).toHaveLength(1); + expect(jobCondition(jobs.build).startsWith("needs.identity.outputs.lock == 'current' && ")).toBe(true); + expect(jobCondition(jobs.publish)).toContain("needs.identity.outputs.lock == 'current'"); + // Pull requests and manual runs always build; only a push skips an image that exists. + expect(jobCondition(jobs.build)).toContain("github.event_name != 'push' || ("); }); it("pins every action by commit and keeps expressions out of shell scripts", () => { @@ -52,14 +124,30 @@ describe("Runner image workflow", () => { // was cancelled. One export for each command, and a limit on each step, // so a stall fails in minutes. const building = workflow.split(/\n(?= {6}- name: )/).filter((step) => step.includes("docker buildx build")); - expect(building.length).toBeGreaterThanOrEqual(3); + expect(building.length).toBeGreaterThanOrEqual(4); for (const step of building) { expect(step).toMatch(/^ {8}timeout-minutes: \d+$/m); for (const command of step.split("docker buildx build").slice(1)) expect(command.match(/--output /g) ?? []).toHaveLength(1); } // The trees the checks read come from the archives, not from a second export. expect(workflow).toContain("node scripts/runner-image-oci.mjs unpack"); - expect(workflow.match(/type=local/g)).toHaveLength(2); + expect(workflow.match(/type=local/g)).toHaveLength(3); + }); + + it("passes the image build exactly two build arguments", () => { + // Build arguments are not part of the image identity, so a new one would + // change what is published under an unchanged digest. + const image = steps(jobs.build).filter((step) => step.includes("--target \"$target\"")); + expect(image).toHaveLength(1); + expect([...image[0].matchAll(/--build-arg (\S+)/g)].map((match) => match[1])).toEqual([ + '"PAPERCLIP_RUNNER_SOURCE_REVISION=$GITHUB_SHA"', + "PAPERCLIP_RUNNER_IMAGE_PUBLICATION=1", + ]); + expect(image[0]).toContain("target=runner-artifacts\n"); + expect(image[0]).toContain('if [ "$variant" = slim ]; then target=runner-artifacts-slim; fi'); + // No other command passes a build argument, and the identity is computed without one. + expect(workflow.match(/--build-arg /g)).toHaveLength(2); + expect(workflow).not.toMatch(/build-args:|BUILDX_BUILD_ARG|--build-arg-file/); }); it("runs for a pull request that changes any input of the image identity", () => { @@ -68,7 +156,7 @@ describe("Runner image workflow", () => { || (pattern.endsWith("/**") && (file === pattern.slice(0, -3) || file.startsWith(pattern.slice(0, -2))))); // .dockerignore is not in the identity, but it decides which files the build receives. for (const input of ["Dockerfile", ".dockerignore", ".github/workflows/runner-image.yml", "docker/runner-image/Dockerfile.smoke", - "scripts/runner-image-oci.mjs", ...RUNNER_IMAGE_CONTEXT_INPUTS]) { + "scripts/runner-image-oci.mjs", "scripts/runner-image-publish.mjs", "scripts/standard-image-contract.mjs", ...RUNNER_IMAGE_CONTEXT_INPUTS]) { expect(covered(input), input).toBe(true); } });