diff --git a/.github/workflows/runner-image.yml b/.github/workflows/runner-image.yml index ccbf4c69460..c8855371fad 100644 --- a/.github/workflows/runner-image.yml +++ b/.github/workflows/runner-image.yml @@ -1,8 +1,9 @@ name: Runner image # Builds and checks the Runner image: the `runner-artifacts` target of the -# Dockerfile, a plain file tree that other image builds copy the Runner from. -# doc/runner-image-contract.md is the contract. +# Dockerfile, a plain file tree that other image builds copy the Runner from, +# and its `runner-artifacts-slim` variant without the pack's transport +# archives. doc/runner-image-contract.md is the contract. # # This workflow builds and checks only. It has no access to a registry and # publishes nothing. @@ -28,6 +29,7 @@ on: - scripts/link-plugin-dev-sdk.mjs - scripts/runner-image-identity.mjs - scripts/runner-image-layout.mjs + - scripts/runner-image-oci.mjs - docker/runner-image/** workflow_dispatch: @@ -96,6 +98,7 @@ jobs: # files the image build never receives. - name: Compute the identity in the build context id: identity + timeout-minutes: 10 run: | set -euo pipefail out="$RUNNER_TEMP/runner-image-identity" @@ -185,44 +188,93 @@ jobs: mirrors = ["mirror.gcr.io"] # The build itself runs the frozen install, the Runner's own build, the - # provider pack integrity check and the layout check. The tree is - # exported for the checks below. + # provider pack integrity check and the layout check. + # + # Both variants come from one build: the second command only reads the + # cache of the first. Each is exported once, as an OCI archive and + # nothing else. A command with two exports, an archive and a tree, + # stalled on these runners when every stage came from the cache, so a + # command has one export, and the step has a time limit: a cold build + # takes about ten minutes. # # This is the publication form of the build, the only one whose pack # names the image (`runnerInputsDigest`), so the check below can bind # the tree to the digest the identity job resolved. Nothing here - # publishes it: the tree stays on this runner. - - name: Build the Runner image + # publishes it: the archives and trees stay on this runner. + - name: Build both variants of the Runner image + timeout-minutes: 30 run: | set -euo pipefail - docker buildx build \ - --target runner-artifacts \ - --platform "$PLATFORM" \ - --build-arg "PAPERCLIP_RUNNER_SOURCE_REVISION=$GITHUB_SHA" \ - --build-arg PAPERCLIP_RUNNER_IMAGE_PUBLICATION=1 \ - --provenance=false --sbom=false \ - --output "type=local,dest=$RUNNER_TEMP/runner-image/tree" \ - . + for variant in full slim; do + target=runner-artifacts + if [ "$variant" = slim ]; then target=runner-artifacts-slim; fi + docker buildx build \ + --target "$target" \ + --platform "$PLATFORM" \ + --build-arg "PAPERCLIP_RUNNER_SOURCE_REVISION=$GITHUB_SHA" \ + --build-arg PAPERCLIP_RUNNER_IMAGE_PUBLICATION=1 \ + --provenance=false --sbom=false \ + --output "type=oci,dest=$RUNNER_TEMP/runner-image/$variant.oci.tar" \ + . + done - # Against the exported tree, bound to this run: the digest the identity - # job resolved, this commit and this leg's architecture. Checks every - # documented path, the build stamp, every pack artifact and tree digest - # in provider-pack.json, and every transport archive. It also extracts - # the archives as a sandbox does and compares the result with the pack; - # that needs about as much free disk as the pack is large. - - name: Check the layout contract and pack integrity + # The trees the checks below read are the layers of those archives, + # written out in order. They are the bytes of the images themselves, + # not a second export of the build, and each blob is checked against + # its digest as it is read. + - name: Unpack both images + timeout-minutes: 10 run: | - node scripts/runner-image-layout.mjs check "$RUNNER_TEMP/runner-image/tree" \ - --expect-inputs-digest "$DIGEST" \ - --expect-source-revision "$GITHUB_SHA" \ - --expect-architecture "$ARCH" + set -euo pipefail + for variant in full slim; do + node scripts/runner-image-oci.mjs unpack "$RUNNER_TEMP/runner-image/$variant.oci.tar" \ + --out "$RUNNER_TEMP/runner-image/$variant" + done + # Against each unpacked tree, bound to this run: the digest the identity + # job resolved, this commit and this leg's architecture. Checks every + # documented path, the build stamp, and every pack artifact and tree + # digest in provider-pack.json. A slim tree must not hold the transport + # archives. A full tree must: the check verifies each one, extracts + # them as a sandbox does and compares the result with the pack; that + # needs about as much free disk as the pack is large. + - name: Check the layout contract and pack integrity + timeout-minutes: 10 + run: | + set -euo pipefail + for variant in full slim; do + node scripts/runner-image-layout.mjs check "$RUNNER_TEMP/runner-image/$variant" \ + --variant "$variant" \ + --expect-inputs-digest "$DIGEST" \ + --expect-source-revision "$GITHUB_SHA" \ + --expect-architecture "$ARCH" + done + + # The full image must be the slim image's layers, as the same blobs, + # then the transport layers, with nothing else in them. No file may be + # in two layers. + - name: Check the layers of both variants + timeout-minutes: 10 + run: | + set -euo pipefail + { + echo '```' + node scripts/runner-image-oci.mjs check-layers \ + --slim "$RUNNER_TEMP/runner-image/slim.oci.tar" \ + --full "$RUNNER_TEMP/runner-image/full.oci.tar" + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + # On the slim tree: the daemon is in a layer both variants share, so + # one run covers both, and the slim tree is the smaller one to load. + # This command has one export, a few small files. - name: Run the daemon on every consumer base image + timeout-minutes: 20 run: | docker buildx build \ --file docker/runner-image/Dockerfile.smoke \ --platform "$PLATFORM" \ - --build-context "runner-artifacts=$RUNNER_TEMP/runner-image/tree" \ + --build-context "runner-artifacts=$RUNNER_TEMP/runner-image/slim" \ --output "type=local,dest=$RUNNER_TEMP/runner-image/smoke" \ docker/runner-image ls -l "$RUNNER_TEMP/runner-image/smoke" diff --git a/Dockerfile b/Dockerfile index f4905a76709..d30284ec19f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -206,8 +206,9 @@ RUN mkdir -p /provider-pack-large \ # stage, so a change outside the Runner's inputs leaves them cached, and the # default targets never build them. # -# Consumers must only `COPY --from=runner-artifacts `. A build can then -# replace the stage with an image built earlier: +# Consumers must only `COPY --from=runner-artifacts `, or from +# `runner-artifacts-slim`, the same tree without the pack's transport +# archives. A build can then replace the stage with an image built earlier: # --build-context runner-artifacts=docker-image://ghcr.io/paperclipai/paperclip-runner@sha256: # # The image identity is a digest of this whole file and of every file under @@ -302,10 +303,11 @@ COPY --from=runner-artifacts-identity /runner-image-identity.json /runner-image- # the Runner build but no provider pack, as in the `runner-provider-pack` stage. ARG PAPERCLIP_RUNNER_SOURCE_REVISION="" ARG PAPERCLIP_BUILD_COMMIT="" -# Set to 1 only by the workflow that builds the image for publication. Only -# then does the pack record `runnerInputsDigest`, the name of the published -# image. Another image build that runs this stage inline, with -# PAPERCLIP_BUILD_COMMIT, builds the same Runner but must not claim to be it. +# Set to 1 only by .github/workflows/runner-image.yml, which builds the +# publication form of the image. Only then does the pack record +# `runnerInputsDigest`, the identity of this image. Another image build that +# runs this stage inline, with PAPERCLIP_BUILD_COMMIT, builds the same Runner +# but must not claim to be this image. ARG PAPERCLIP_RUNNER_IMAGE_PUBLICATION="" # One layer: the install tree and the pnpm store are removed before the layer # is committed, and the pack is granted a+rX in the layer that writes it (see @@ -323,6 +325,12 @@ ARG PAPERCLIP_RUNNER_IMAGE_PUBLICATION="" # The install is frozen: the identity above covers pnpm-lock.yaml as the build # received it, and resolving again here would build from a lockfile the # identity never saw. +# +# The pack's large dependency files get a fixed timestamp before their +# transport archives are written, so each archive has the same bytes while its +# file does. The last commands sort the tree into the layers of the two final +# stages. `dist` is what the Runner's own build writes: its large files change +# with the Runner's source, so they stay with the content that does. RUN set -eu; \ revision="${PAPERCLIP_RUNNER_SOURCE_REVISION:-${PAPERCLIP_BUILD_COMMIT}}"; \ publication="${PAPERCLIP_RUNNER_IMAGE_PUBLICATION}"; \ @@ -340,13 +348,14 @@ RUN set -eu; \ pnpm run build:binary; \ node scripts/record-build-inputs.mjs --finish; \ cd /workspace; \ - mkdir -p /out; \ + mkdir -p /out /out-large /out-archives /out-transport; \ if [ -n "$revision" ]; then \ if [ -n "$publication" ]; then \ PAPERCLIP_RUNNER_INPUTS_DIGEST="$(node -p 'require("/runner-image-identity.json").digest')"; \ export PAPERCLIP_RUNNER_INPUTS_DIGEST; \ fi; \ PAPERCLIP_RUNNER_SOURCE_REVISION="$revision" node packages/paperclip-runner/scripts/build-provider-pack.mjs /out/provider-pack; \ + find /out/provider-pack -path /out/provider-pack/dist -prune -o -type f -size +1048575c -exec touch -d @0 {} +; \ node scripts/create-provider-pack-archive.mjs /out/provider-pack; \ node_modules/.bin/esbuild /out/provider-pack/dist/live/runnerd-codex-transport.js \ --bundle --platform=node --format=esm --log-level=warning \ @@ -354,20 +363,53 @@ RUN set -eu; \ --outfile=/out/bundles/runnerd-codex-transport.mjs; \ node scripts/runner-image-layout.mjs write --out /out --workspace /workspace --identity /runner-image-identity.json --source-revision "$revision" ${publication:+--publication}; \ chmod -R a+rX /out; \ - node scripts/runner-image-layout.mjs check /out --expect-source-revision "$revision" ${publication:+--expect-inputs-digest "$PAPERCLIP_RUNNER_INPUTS_DIGEST"}; \ + node scripts/runner-image-layout.mjs check /out --variant full --expect-source-revision "$revision" ${publication:+--expect-inputs-digest "$PAPERCLIP_RUNNER_INPUTS_DIGEST"}; \ cd / && /out/provider-pack/node_modules/node/bin/node --input-type=module -e 'const transport = await import("/out/bundles/runnerd-codex-transport.mjs"); if (typeof transport.createCapabilityRunnerdCodexTransport !== "function") process.exit(1);'; \ else \ echo "Skipping the provider pack: supply a full PAPERCLIP_RUNNER_SOURCE_REVISION or PAPERCLIP_BUILD_COMMIT to build a complete Runner image"; \ node scripts/runner-image-layout.mjs write --out /out --workspace /workspace --identity /runner-image-identity.json; \ chmod -R a+rX /out; \ - node scripts/runner-image-layout.mjs check /out --allow-unstamped; \ + node scripts/runner-image-layout.mjs check /out --variant full --allow-unstamped; \ fi; \ /out/paperclip-runnerd --build-metadata > /dev/null; \ + cd /out; \ + find provider-pack -path provider-pack/dist -prune -o -type f -size +1048575c \ + -exec sh -c 'mkdir -p "/out-large/$(dirname "$1")" && mv "$1" "/out-large/$1"' _ {} \; ; \ + if [ -f provider-pack.archive.json ]; then \ + mkdir -p /out-archives/provider-pack.archives /out-transport; \ + node -e 'for (const [index, file] of require("/out/provider-pack.archive.json").files.entries()) if (!file.path.startsWith("dist/")) console.log(index)' > /tmp/stable-archives; \ + while read -r index; do mv "provider-pack.archives/$index.tar.zst" /out-archives/provider-pack.archives/; done < /tmp/stable-archives; \ + rm /tmp/stable-archives; \ + mv provider-pack.archives provider-pack.archive.json /out-transport/; \ + fi; \ + find /out-large /out-archives -exec touch -h -d @0 {} +; \ cd /; \ rm -rf /workspace /root/.local/share/pnpm /root/.cache -FROM scratch AS runner-artifacts -COPY --from=runner-artifacts-build /out/ / +# Two variants of one build. Both are plain file trees, and each `COPY --link` +# is one layer, independent of the layers under it. The layers are split by +# how often their content changes, so a new image shares its large layers with +# the one before it; the stage above gives the files of layers 1 and 3 a fixed +# timestamp, so those layers have the same bytes whenever their files do. +# 1. provider pack dependency files of 1 MiB or more (provider binaries and +# Node), which change with a provider version or the Node pin; +# 2. everything else, which changes with every Runner source change. +# `runner-artifacts-slim` is these two layers: the Runner without the pack's +# transport form. `runner-artifacts` is the same two layers, then +# 3. the transport archive of each file in layer 1, as stable as layer 1; +# 4. the other transport archives, the overlay archive and the receipt, +# which change with layer 2. +# Layers 3 and 4 hold the transport archives and their receipt and nothing +# else, so a reader can fetch all of them without the pack. +# `runner-image-oci.mjs check-layers` checks all of this on built images. +# Every path is where it would be in a single layer; consumers copy by path. +FROM scratch AS runner-artifacts-slim +COPY --link --from=runner-artifacts-build /out-large/ / +COPY --link --from=runner-artifacts-build /out/ / + +FROM runner-artifacts-slim AS runner-artifacts +COPY --link --from=runner-artifacts-build /out-archives/ / +COPY --link --from=runner-artifacts-build /out-transport/ / FROM base AS production ARG USER_UID=1000 diff --git a/doc/DOCKER.md b/doc/DOCKER.md index f487b3a0c86..b208d6df269 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -81,9 +81,10 @@ image before rollout. Resolve immutable digests instead of deploying mutable tag The `runner-artifacts` target builds the Runner alone: the daemon, the Runner package's build output, the remote provider pack and its transport archives. -It does not depend on the server or UI build, and no other target depends on -it yet. `.github/workflows/runner-image.yml` builds and checks it on pull -requests, for Linux AMD64 and ARM64. See +The `runner-artifacts-slim` target is the same image without the transport +archives. Neither depends on the server or UI build, and no other target +depends on them yet. `.github/workflows/runner-image.yml` builds and checks +both on pull requests, for Linux AMD64 and ARM64. See [`doc/runner-image-contract.md`](runner-image-contract.md) for the layout, the identity rules and how to consume the image. diff --git a/doc/runner-image-contract.md b/doc/runner-image-contract.md index 9166512b91e..f6263aedba5 100644 --- a/doc/runner-image-contract.md +++ b/doc/runner-image-contract.md @@ -1,10 +1,14 @@ # Runner image contract The Runner image is the Runner built alone, as a plain file tree with no -entrypoint and no operating system: the `runner-artifacts` target of the -`Dockerfile`. An image build copies paths out of it instead of compiling the -Runner again. Its name is `ghcr.io/paperclipai/paperclip-runner`. Nothing in -this repository consumes it yet. +entrypoint and no operating system. An image build copies paths out of it +instead of compiling the Runner again. Nothing in this repository consumes it +yet. Its name is `ghcr.io/paperclipai/paperclip-runner`, and one build makes +two variants: + +- full, the `runner-artifacts` target of the `Dockerfile`: every path below; +- slim, the `runner-artifacts-slim` target: the same without the two paths + marked "full only", the provider pack's transport archives. Nothing publishes it yet: `.github/workflows/runner-image.yml` builds and checks it. @@ -19,8 +23,8 @@ Every path a consumer may copy (`RUNNER_IMAGE_PATHS` in | `/runner-image.json` | Identity and description of this image. | | `/paperclip-runnerd` | The daemon. A stripped release binary, mode `0755`. | | `/provider-pack/` | The remote provider pack, with `provider-pack.json`. | -| `/provider-pack.archive.json` | Receipt of the pack's transport archives. | -| `/provider-pack.archives/` | The transport archives the receipt lists. | +| `/provider-pack.archive.json` | Full only. Receipt of the pack's transport archives. | +| `/provider-pack.archives/` | Full only. The transport archives the receipt lists. | | `/runner-package/package.json` | The Runner package manifest. | | `/runner-package/dist/` | The Runner's complete build output. `dist/bin/paperclip-runnerd` is the same file as `/paperclip-runnerd`. | | `/runner-package/node_modules/.cache/paperclip-runner-build-inputs.json` | The build stamp the Runner's build recorded for that output. | @@ -31,7 +35,8 @@ Every path a consumer may copy (`RUNNER_IMAGE_PATHS` in | `/pnpm-lock.yaml` | The lockfile the image was installed from. | Paths under `/runner-package/` keep their place inside -`packages/paperclip-runner`. `runner-image.json` describes one platform: +`packages/paperclip-runner`. `runner-image.json` describes one platform, and +is the same file in both variants: - `inputsDigest` (the identity below) and `buildInputsDigest` (the digest in the Runner's build stamp); @@ -45,12 +50,28 @@ Paths under `/runner-package/` keep their place inside was compiled on (Debian bookworm, the oldest a consumer runs) and the newest glibc version it needs; - `providerPack`: the pack's digest and target; -- `paths` and `sha256`: the table above, and the digest of every single file. +- `paths` and `sha256`: the table above, and the digest of every single file; +- `fullImageOnly`: the names in `paths` that only the full image has. + +## Layers + +1. The pack's dependency files of 1 MiB or more: provider binaries and Node. +2. Everything else, with `runner-image.json`. +3. Full only: the transport archive of each file in layer 1. +4. Full only: the other transport archives, the overlay archive, the receipt. + +The slim image is layers 1 and 2. The full image is those same two layer +blobs, then layers 3 and 4, which hold transport content and nothing else. +Layers 1 and 3 have the same bytes in every build with the same dependency +files, so a new image shares them with the one before it. A large file under +the pack's `dist` is the Runner's own output: it and its archive are in +layers 2 and 4. An archive is named by its file's place in the receipt, which +lists the dependency files first, so their names do not depend on `dist`. ## Identity -An image is named `inputs-`. `scripts/runner-image-identity.mjs` -computes the digest from: +Both variants are named by one digest, `inputs-`: the variant is not +part of the identity. `scripts/runner-image-identity.mjs` computes it from: - every file under the paths the build copies from the build context (`RUNNER_IMAGE_CONTEXT_INPUTS`, which include `pnpm-lock.yaml`): path, @@ -76,8 +97,16 @@ pull-request runs resolve it first. For a local build, run ## Checks -`node scripts/runner-image-layout.mjs check ` runs in the build and -again on the exported tree. It fails unless: +CI builds each variant as an OCI archive and nothing else. +`node scripts/runner-image-oci.mjs unpack --out ` writes +the tree of an image from its layers, so the tree CI checks is the bytes of +the image. It refuses a whiteout, an absolute or `..` path, a path written +twice and an entry behind a link. + +`node scripts/runner-image-layout.mjs check --variant slim|full` runs +in the build and again on each unpacked tree. The variant has no default: a +slim tree with the transport paths fails, and so does a full tree without +them. The check fails unless: - every documented path exists, matches its digest in `runner-image.json`, and resolves inside the tree; @@ -87,12 +116,15 @@ again on the exported tree. It fails unless: image's revision and platform; - the pack names the inputs digest in a publication build and in no other. `--expect-inputs-digest` fails on any other build; -- every transport archive matches its receipt, and the archives - extract to the pack. The check extracts them as a sandbox does (GNU tar +- in a full image, every transport archive matches its receipt, and the + archives extract to the pack. The check extracts them as a sandbox does (GNU tar with the options in `provider-pack-layers.ts`) and compares paths, content, link targets and all twelve permission bits, after the `a+rX` grant the build makes once the archives are written. +`node scripts/runner-image-oci.mjs check-layers --slim --full ` +checks "Layers" on the two archives, and that no file is in two layers. + The build also runs `paperclip-runnerd --build-metadata` on every base image in `docker/runner-image/Dockerfile.smoke`. Add a base there before a consumer uses it. @@ -107,10 +139,10 @@ FROM debian:trixie-slim COPY --from=runner-artifacts /paperclip-runnerd /usr/local/bin/paperclip-runnerd ``` -A Dockerfile with a `runner-artifacts` stage of its own, as in this -repository, can take a built image or tree in its place with -`--build-context runner-artifacts=`. The stage is then -not built. +A Dockerfile with a `runner-artifacts` or `runner-artifacts-slim` stage of +its own, as in this repository, can take a built image or tree in its place +with `--build-context =`. The stage is then not +built. ## Known limits diff --git a/packages/paperclip-runner/scripts/build-provider-pack.mjs b/packages/paperclip-runner/scripts/build-provider-pack.mjs index e68ec58baf5..50c9b204a5a 100644 --- a/packages/paperclip-runner/scripts/build-provider-pack.mjs +++ b/packages/paperclip-runner/scripts/build-provider-pack.mjs @@ -284,9 +284,12 @@ try { : spawnSync("git", ["diff", "--quiet", "--", "packages/paperclip-runner"], { cwd: workspaceRoot, }).status !== 0; - // Set only by the Runner image build (the `runner-artifacts` Dockerfile - // target): the identity of the inputs the image was built from, which is its - // `inputs-` tag. No check reads it. + // Set only by a publication build of the Runner image (the + // `runner-artifacts` Dockerfile target with + // PAPERCLIP_RUNNER_IMAGE_PUBLICATION=1): the identity of the inputs the image + // was built from, which is its `inputs-` tag. + // scripts/runner-image-layout.mjs checks it against the image, and refuses + // it in a pack from any other build. const runnerInputsDigest = process.env.PAPERCLIP_RUNNER_INPUTS_DIGEST?.trim(); if (runnerInputsDigest && !/^[0-9a-f]{64}$/.test(runnerInputsDigest)) { throw new Error("PAPERCLIP_RUNNER_INPUTS_DIGEST must be 64 hexadecimal characters"); diff --git a/scripts/create-provider-pack-archive.mjs b/scripts/create-provider-pack-archive.mjs index b599ea169ef..684849050de 100644 --- a/scripts/create-provider-pack-archive.mjs +++ b/scripts/create-provider-pack-archive.mjs @@ -62,6 +62,14 @@ export async function createProviderPackLayers(root) { } }; await visit(); + // Archive N holds file N, so a file's archive name is its place in this + // list. The pack's `dist` is the Runner's own build output; every other + // large file is a dependency. The dependencies come first, so that their + // archive names do not change when the Runner's build gains or loses a + // large file. `dist` sorts before `node_modules`: in path order alone, one + // more large file under it renamed every dependency archive. + const own = file => file.path.startsWith('dist/'); + files.splice(0, files.length, ...files.filter(file => !own(file)), ...files.filter(own)); const archive = async (name, paths) => { const target = resolve(directory, name); // Explicit NUL-separated names and no recursion avoid options, duplicate @@ -74,10 +82,20 @@ export async function createProviderPackLayers(root) { }); tar.stdin.on('error', () => {}); // The child/pipeline supplies the causal failure. tar.stdin.end(paths.map(path => `./${path}\0`).join('')); + // The compressor is ended only after it has taken every chunk. When end() + // arrives while the last chunk is still queued, Node finishes the zstd + // stream twice and the archive gets an empty frame at its end. Piping + // leaves that to timing, so the same files gave two different archives. + const compress = createZstdCompress({ params: { [constants.ZSTD_c_compressionLevel]: 3 } }); + const feed = async () => { + for await (const chunk of tar.stdout) { + await new Promise((resolve, reject) => compress.write(chunk, error => error ? reject(error) : resolve())); + } + compress.end(); + }; try { - await Promise.all([completed, pipeline(tar.stdout, - createZstdCompress({ params: { [constants.ZSTD_c_compressionLevel]: 3 } }), createWriteStream(target))]); - } catch (error) { tar.kill(); throw error; } + await Promise.all([completed, feed(), pipeline(compress, createWriteStream(target))]); + } catch (error) { tar.kill(); compress.destroy(); throw error; } const hash = createHash('sha256'); for await (const chunk of createReadStream(target)) hash.update(chunk); return { sha256: hash.digest('hex'), bytes: (await stat(target)).size }; diff --git a/scripts/runner-image-layout.mjs b/scripts/runner-image-layout.mjs index b1098914818..974e2c41f77 100644 --- a/scripts/runner-image-layout.mjs +++ b/scripts/runner-image-layout.mjs @@ -14,15 +14,20 @@ // check verifies a built tree: every documented path, runner-image.json, // the build stamp, the provider pack's integrity and its transport // archives. The image build runs it, and CI runs it again on the -// tree it exported. No checked path may resolve outside the tree. +// tree of each built image (scripts/runner-image-oci.mjs unpacks +// it). No checked path may resolve outside the tree. // It also extracts the transport archives the way a sandbox does and // compares the result with the provider pack, so an archive set that // has the recorded bytes but not the pack's files does not pass. // +// The image has two variants. `slim` is everything but the transport archives +// and their receipt; `full` adds those. `check` must be told which one it +// reads: a slim tree with archives fails, and so does a full tree without. +// // Usage: // node scripts/runner-image-layout.mjs write --out DIR --workspace DIR --identity FILE // [--source-revision SHA] [--publication] -// node scripts/runner-image-layout.mjs check DIR [--allow-unstamped] +// node scripts/runner-image-layout.mjs check DIR --variant slim|full [--allow-unstamped] // [--expect-inputs-digest HEX] [--expect-source-revision SHA] [--expect-architecture ARCH] // An unknown flag, a repeated flag or a flag with an empty value is an error: // an expectation that was not really given must not pass as one that held. @@ -89,6 +94,11 @@ export const RUNNER_IMAGE_PATHS = Object.freeze({ lockfile: "pnpm-lock.yaml", }); +/** The two variants of the image. `full` is `slim` plus the paths below. */ +export const RUNNER_IMAGE_VARIANTS = Object.freeze(["slim", "full"]); +/** Names in RUNNER_IMAGE_PATHS that only the full image has: the pack's transport form. */ +export const RUNNER_IMAGE_FULL_ONLY_PATHS = Object.freeze(["providerPackArchiveReceipt", "providerPackArchives"]); + const DIRECTORY_PATHS = new Set(["providerPack", "providerPackArchives", "runnerPackage", "runnerDist"]); /** Paths that exist only when the build was given a source revision. */ const STAMPED_ONLY_PATHS = new Set([ @@ -311,6 +321,8 @@ export function writeRunnerImageManifest({ libc: { family: "glibc", baseline: libc.baseline, buildVersion: libc.buildVersion, runnerdRequires: libc.runnerdRequires }, providerPack: pack === null ? null : { digest: pack.digest, target: pack.payload?.target ?? null }, paths: { ...RUNNER_IMAGE_PATHS }, + // This file is the same in both variants, so it names what only one has. + fullImageOnly: [...RUNNER_IMAGE_FULL_ONLY_PATHS], sha256, }; const manifestPath = resolveIn(outRoot, RUNNER_IMAGE_PATHS.manifest); @@ -387,6 +399,9 @@ function checkManifestShape(manifest, problems, { allowUnstamped }) { if (canonicalJson(manifest.paths ?? null) !== canonicalJson(RUNNER_IMAGE_PATHS)) { problems.push("paths does not match the documented layout (RUNNER_IMAGE_PATHS)"); } + if (canonicalJson(manifest.fullImageOnly ?? null) !== canonicalJson(RUNNER_IMAGE_FULL_ONLY_PATHS)) { + problems.push("fullImageOnly does not name the paths only the full image has (RUNNER_IMAGE_FULL_ONLY_PATHS)"); + } if (!manifest.sha256 || typeof manifest.sha256 !== "object") problems.push("sha256 must map file paths to their digests"); return stamped; } @@ -579,7 +594,7 @@ function treeEntries(root) { } /** Up to five examples and the count, so one wrong archive does not print the whole pack. */ -function summarize(problems, label, items) { +export function summarize(problems, label, items) { if (items.length === 0) return; const shown = items.slice(0, 5).join(", "); problems.push(`${label} (${items.length}): ${shown}${items.length > 5 ? ", ..." : ""}`); @@ -746,22 +761,27 @@ export async function checkProviderPackArchives(root) { * Verifies the tree at `root` against the layout contract. Returns every * problem found; an empty list means the tree is a valid Runner image. * It reads the transport archives' bytes but does not extract them: - * checkProviderPackArchives does, and the `check` command runs both. + * checkProviderPackArchives does, and the `check` command runs both on a + * full image. `variant` has no default: a slim image must not hold the + * transport archives, and a full image must. * @param {string} root * @param {{ + * variant: string, * allowUnstamped?: boolean, * expectInputsDigest?: string | null, * expectSourceRevision?: string | null, * expectArchitecture?: string | null, - * }} [options] + * }} options * @returns {string[]} */ export function checkRunnerImageLayout(root, { + variant, allowUnstamped = false, expectInputsDigest = null, expectSourceRevision = null, expectArchitecture = null, -} = {}) { +}) { + if (!RUNNER_IMAGE_VARIANTS.includes(variant)) throw new Error("the check must be told the variant: slim or full"); const problems = []; const manifestPath = resolveIn(root, RUNNER_IMAGE_PATHS.manifest); if (!existsSync(manifestPath)) return ["runner-image.json is missing"]; @@ -790,9 +810,14 @@ export function checkRunnerImageLayout(root, { try { stat = lstatSync(absolute); } catch { + if (variant === "slim" && RUNNER_IMAGE_FULL_ONLY_PATHS.includes(name)) continue; problems.push(`documented path ${relative} (${name}) is missing`); continue; } + if (variant === "slim" && RUNNER_IMAGE_FULL_ONLY_PATHS.includes(name)) { + problems.push(`${relative} (${name}) is in a slim image; only the full image has the transport archives`); + continue; + } if (locate(root, relative).state !== "inside") { problems.push(`documented path ${relative} (${name}) resolves outside the image`); continue; @@ -827,7 +852,7 @@ export function checkRunnerImageLayout(root, { if (stamped) { const pack = checkProviderPack(root, manifest, problems); - if (pack) checkArchives(root, pack, problems); + if (pack && variant === "full") checkArchives(root, pack, problems); } else if (manifest.providerPack !== null) { problems.push("an image without a source revision must not record a provider pack"); } @@ -837,6 +862,7 @@ export function checkRunnerImageLayout(root, { const COMMAND_FLAGS = Object.freeze({ write: { out: "value", workspace: "value", identity: "value", "source-revision": "value", publication: "boolean" }, check: { + variant: "value", "allow-unstamped": "boolean", "expect-inputs-digest": "value", "expect-source-revision": "value", @@ -847,11 +873,11 @@ const COMMAND_FLAGS = Object.freeze({ /** * Splits a command line into positional arguments and the flags the command * knows. Throws on an unknown or repeated flag and on a missing or empty value. - * @param {"write" | "check"} command + * @param {string} command * @param {string[]} argv + * @param {Record} [known] the command's flags: "value" or "boolean" */ -export function parseLayoutArguments(command, argv) { - const known = COMMAND_FLAGS[command]; +export function parseLayoutArguments(command, argv, known = COMMAND_FLAGS[command]) { /** @type {string[]} */ const positional = []; /** @type {Record} */ @@ -888,7 +914,7 @@ if (invoked === import.meta.url) { try { const [command, ...rest] = process.argv.slice(2); if (command !== "write" && command !== "check") { - throw new Error("Usage: runner-image-layout.mjs write ... | check DIR [...]"); + throw new Error("Usage: runner-image-layout.mjs write ... | check DIR --variant slim|full [...]"); } const { positional, flags } = parseLayoutArguments(command, rest); if (command === "write") { @@ -909,16 +935,21 @@ if (invoked === import.meta.url) { } else { if (positional.length !== 1) throw new Error("check takes exactly one directory"); const root = path.resolve(positional[0]); + const variant = flags.variant; + if (typeof variant !== "string" || !RUNNER_IMAGE_VARIANTS.includes(variant)) throw new Error("check needs --variant slim or --variant full"); const problems = checkRunnerImageLayout(root, { + variant, allowUnstamped: flags["allow-unstamped"] === true, expectInputsDigest: expectation(flags, "expect-inputs-digest", HEX_64, "64 hexadecimal characters"), expectSourceRevision: expectation(flags, "expect-source-revision", REVISION, "a full Git SHA"), expectArchitecture: expectation(flags, "expect-architecture", /^(?:amd64|arm64)$/, "amd64 or arm64"), }); - // Only a tree without a source revision has no archives; the layout - // check above has already refused that tree unless it was allowed. - let archives = "The tree has no provider pack, so there are no transport archives to extract."; - if (existsSync(resolveIn(root, RUNNER_IMAGE_PATHS.providerPackArchiveReceipt))) { + // A full tree without a source revision has no archives either; the + // layout check above has already refused that tree unless it was allowed. + let archives = variant === "slim" + ? "A slim image has no transport archives." + : "The tree has no provider pack, so there are no transport archives to extract."; + if (variant === "full" && existsSync(resolveIn(root, RUNNER_IMAGE_PATHS.providerPackArchiveReceipt))) { const roundTrip = await checkProviderPackArchives(root); problems.push(...roundTrip.problems); archives = roundTrip.tar === "GNU tar" diff --git a/scripts/runner-image-oci.mjs b/scripts/runner-image-oci.mjs new file mode 100644 index 00000000000..f698920eaa1 --- /dev/null +++ b/scripts/runner-image-oci.mjs @@ -0,0 +1,585 @@ +#!/usr/bin/env node +// OCI archives of the Runner image, as `docker buildx build --output type=oci` +// writes them for one platform. doc/runner-image-contract.md is the contract. +// +// unpack writes the file tree of the image in an archive: its layers, +// in order. The layout check and the smoke matrix then read +// the bytes of the archive, not a second export of the build, +// so what they accept is what a registry would receive. +// check-layers verifies the layers of the two variants: they are split by +// how often their content changes, the full image is the slim +// image plus the transport layers, and nothing else is in +// those. +// +// Usage: +// node scripts/runner-image-oci.mjs unpack IMAGE.oci.tar --out DIR +// node scripts/runner-image-oci.mjs check-layers [--slim SLIM.oci.tar] [--full FULL.oci.tar] +// +// The archive and its layers are read here, not with the system's tar, so +// every platform reads them the same way. Each blob is checked against its +// digest as it is read. The image is `FROM scratch` and every layer only adds +// files, so `unpack` refuses a whiteout, a path that is absolute or has `..`, +// a path written twice, an entry behind a link, and every entry that is not a +// file, a directory or a link. +// +// The image build does not run this file, so it is not an input of the image. + +import { createHash } from "node:crypto"; +import { + chmodSync, + closeSync, + createReadStream, + linkSync, + lstatSync, + mkdirSync, + openSync, + readSync, + readdirSync, + symlinkSync, + utimesSync, + writeSync, +} from "node:fs"; +import path from "node:path"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; +import { pathToFileURL } from "node:url"; +import { createGunzip } from "node:zlib"; +import { RUNNER_IMAGE_PATHS, canonicalJson, parseLayoutArguments, summarize } from "./runner-image-layout.mjs"; + +const BLOCK = 512; +const DIGEST = /^sha256:[0-9a-f]{64}$/; +const GZIP_LAYER = /^application\/vnd\.(?:oci\.image\.layer\.v1\.tar\+gzip|docker\.image\.rootfs\.diff\.tar\.gzip)$/; +const ENTRY_KINDS = Object.freeze({ 0: "file", "\0": "file", 1: "hardlink", 2: "link", 5: "directory" }); + +/** One tar header block, or null for the zero block that ends an archive. */ +function parseTarHeader(block) { + if (block.every((byte) => byte === 0)) return null; + const text = (start, length) => { + const field = block.subarray(start, start + length); + const end = field.indexOf(0); + return field.subarray(0, end === -1 ? length : end).toString("utf8"); + }; + const octal = (start, length, label) => { + const value = text(start, length).trim(); + if (!/^[0-7]*$/.test(value)) throw new Error(`tar header has a ${label} this reader does not support`); + return value === "" ? 0 : Number.parseInt(value, 8); + }; + let sum = 0; + for (let index = 0; index < BLOCK; index += 1) sum += index >= 148 && index < 156 ? 32 : block[index]; + if (sum !== octal(148, 8, "checksum")) throw new Error("tar header checksum does not match"); + // Only the POSIX format has a name prefix; GNU tar keeps other fields there. + const prefix = text(257, 6) === "ustar" && block[263] === 48 ? text(345, 155) : ""; + return { + name: prefix ? `${prefix}/${text(0, 100)}` : text(0, 100), + mode: octal(100, 8, "mode"), + size: octal(124, 12, "size"), + mtime: octal(136, 12, "time"), + type: String.fromCharCode(block[156]), + linkname: text(157, 100), + }; +} + +/** The records of a pax extended header: ` =\n`. */ +function parsePaxRecords(data) { + /** @type {Record} */ + const records = {}; + for (let offset = 0; offset < data.length;) { + const space = data.indexOf(32, offset); + const length = Number.parseInt(data.subarray(offset, space).toString("latin1"), 10); + if (space === -1 || !Number.isSafeInteger(length) || length <= 0 || offset + length > data.length) throw new Error("tar pax header is malformed"); + const record = data.subarray(space + 1, offset + length - 1).toString("utf8"); + const equals = record.indexOf("="); + if (equals === -1) throw new Error("tar pax header is malformed"); + records[record.slice(0, equals)] = record.slice(equals + 1); + offset += length; + } + return records; +} + +/** Header types that only describe the next entry: pax (`x`, and `g` for the whole archive) and GNU long names. */ +const DESCRIBING = new Set(["x", "g", "L", "K"]); + +/** Applies describing headers and returns the entry each other header stands for. */ +function tarEntryReader() { + /** @type {Record} */ + let pax = {}; + /** @type {{ name?: string, linkname?: string }} */ + let gnu = {}; + const cString = (data) => data.subarray(0, data.indexOf(0) === -1 ? data.length : data.indexOf(0)).toString("utf8"); + return { + describe(header, data) { + if (header.type === "x") pax = parsePaxRecords(data); + else if (header.type === "L") gnu.name = cString(data); + else if (header.type === "K") gnu.linkname = cString(data); + }, + entry(header) { + const entry = { + name: pax.path ?? gnu.name ?? header.name, + linkname: pax.linkpath ?? gnu.linkname ?? header.linkname, + size: pax.size === undefined ? header.size : Number(pax.size), + mtime: pax.mtime === undefined ? header.mtime : Number(pax.mtime), + mode: header.mode & 0o7777, + type: header.type, + }; + pax = {}; + gnu = {}; + if (!Number.isSafeInteger(entry.size) || entry.size < 0) throw new Error(`tar entry ${entry.name} has no usable size`); + return entry; + }, + }; +} + +/** Where each member of an uncompressed tar file is, by its exact name. */ +function indexTarFile(file) { + const fd = openSync(file, "r"); + try { + /** @type {Map} */ + const members = new Map(); + const reader = tarEntryReader(); + const block = Buffer.alloc(BLOCK); + for (let offset = 0; readSync(fd, block, 0, BLOCK, offset) === BLOCK;) { + const header = parseTarHeader(block); + if (header === null) break; + offset += BLOCK; + if (DESCRIBING.has(header.type)) { + const data = Buffer.alloc(header.size); + if (readSync(fd, data, 0, header.size, offset) !== header.size) throw new Error(`${file} ends inside a header`); + reader.describe(header, data); + offset += Math.ceil(header.size / BLOCK) * BLOCK; + continue; + } + const entry = reader.entry(header); + if (members.has(entry.name)) throw new Error(`${file} holds ${entry.name} twice`); + members.set(entry.name, { offset, size: entry.size }); + offset += Math.ceil(entry.size / BLOCK) * BLOCK; + } + return members; + } finally { + closeSync(fd); + } +} + +/** + * Calls `visit(entry, content)` for every entry of a tar stream. `content` + * yields the entry's bytes; what `visit` leaves unread is skipped. + * @param {AsyncIterable} source + * @param {(entry: ReturnType["entry"]>, content: () => AsyncGenerator) => Promise | void} visit + */ +async function readTarStream(source, visit) { + const iterator = source[Symbol.asyncIterator](); + let buffered = Buffer.alloc(0); + /** Some bytes, at most `limit`: what is buffered, or the next chunk. Empty at the end of the stream. */ + const some = async (limit) => { + while (buffered.length === 0) { + const next = await iterator.next(); + if (next.done) return buffered; + buffered = next.value; + } + const taken = buffered.subarray(0, limit); + buffered = buffered.subarray(taken.length); + return taken; + }; + /** Exactly `length` bytes, or fewer at the end of the stream. */ + const take = async (length) => { + const parts = []; + for (let missing = length; missing > 0;) { + const part = await some(missing); + if (part.length === 0) break; + parts.push(part); + missing -= part.length; + } + return parts.length === 1 ? parts[0] : Buffer.concat(parts); + }; + const exactly = async (length) => { + const taken = await take(length); + if (taken.length !== length) throw new Error("tar stream ends inside an entry"); + return taken; + }; + const reader = tarEntryReader(); + for (;;) { + const block = await take(BLOCK); + if (block.length === 0) break; + if (block.length !== BLOCK) throw new Error("tar stream ends inside a header"); + const header = parseTarHeader(block); + if (header === null) break; + if (DESCRIBING.has(header.type)) { + reader.describe(header, (await exactly(Math.ceil(header.size / BLOCK) * BLOCK)).subarray(0, header.size)); + continue; + } + const entry = reader.entry(header); + let remaining = entry.size; + const content = async function* () { + while (remaining > 0) { + const chunk = await some(remaining); + if (chunk.length === 0) throw new Error("tar stream ends inside an entry"); + remaining -= chunk.length; + yield chunk; + } + }; + await visit(entry, content); + for await (const _unread of content()) { /* skip what the visitor left */ } + await exactly((BLOCK - (entry.size % BLOCK)) % BLOCK); + } + // Read to the end, so the caller's digest covers the whole blob. + while (!(await iterator.next()).done) { /* trailing zero blocks */ } +} + +/** + * The one image in an OCI archive: its manifest descriptor and its layers. A + * member is found by its exact name (`index.json`, `blobs/sha256/`), as + * buildx writes them. The manifest is checked against its digest here, and + * each layer when `readLayer` reads it. + * @param {string} archive + */ +export function readOciArchive(archive) { + const members = indexTarFile(archive); + const located = (name) => { + const member = members.get(name); + if (!member) throw new Error(`${archive} has no member ${name}`); + return member; + }; + const bytes = (name) => { + const { offset, size } = located(name); + const data = Buffer.alloc(size); + const fd = openSync(archive, "r"); + try { + if (readSync(fd, data, 0, size, offset) !== size) throw new Error(`${archive} ends inside ${name}`); + } finally { + closeSync(fd); + } + return data; + }; + const blobName = (digest) => { + if (!DIGEST.test(digest ?? "")) throw new Error(`${archive} names a blob by something other than a sha256 digest`); + return `blobs/sha256/${digest.slice(7)}`; + }; + const sha256 = (data) => `sha256:${createHash("sha256").update(data).digest("hex")}`; + const index = JSON.parse(bytes("index.json").toString("utf8")); + if (!Array.isArray(index.manifests) || index.manifests.length !== 1) throw new Error(`${archive} must hold exactly one image manifest`); + const { mediaType, digest, size } = index.manifests[0]; + const manifestBytes = bytes(blobName(digest)); + if (sha256(manifestBytes) !== digest) throw new Error(`${archive}: the manifest does not have the digest the index names`); + const manifest = JSON.parse(manifestBytes.toString("utf8")); + if (!Array.isArray(manifest.layers)) throw new Error(`${archive} holds an index, not one platform's image; build without attestations`); + for (const layer of manifest.layers) { + if (!GZIP_LAYER.test(layer.mediaType ?? "")) throw new Error(`${archive}: layer ${layer.digest} is ${layer.mediaType}, not a gzip tar`); + located(blobName(layer.digest)); + } + return { + archive, + descriptor: { mediaType, digest, size }, + /** @type {{ mediaType: string, digest: string, size: number }[]} */ + layers: manifest.layers.map((layer) => ({ mediaType: layer.mediaType, digest: layer.digest, size: layer.size })), + /** Visits every entry of one layer and fails when the blob does not have its digest. */ + async readLayer(layer, visit) { + const { offset, size: bytesInArchive } = located(blobName(layer.digest)); + if (bytesInArchive !== layer.size || bytesInArchive === 0) { + throw new Error(`${archive}: layer ${layer.digest} is ${bytesInArchive} bytes, the manifest says ${layer.size}`); + } + const hash = createHash("sha256"); + const hashing = new Transform({ transform(chunk, _encoding, done) { hash.update(chunk); done(null, chunk); } }); + const gunzip = createGunzip({ chunkSize: 1024 * 1024 }); + const feeding = pipeline(createReadStream(archive, { start: offset, end: offset + bytesInArchive - 1 }), hashing, gunzip); + // A failure below destroys the streams; that rejection is not the cause. + feeding.catch(() => {}); + try { + await readTarStream(gunzip, visit); + } catch (error) { + gunzip.destroy(); + throw error; + } + await feeding; + if (`sha256:${hash.digest("hex")}` !== layer.digest) throw new Error(`${archive}: layer ${layer.digest} does not have its digest`); + }, + }; +} + +/** + * The path of a layer entry relative to the image root, and its kind. Throws + * for an entry an additive layer of a `FROM scratch` image cannot have. + * Returns null for the root directory itself. + */ +function layerEntry(entry) { + const kind = ENTRY_KINDS[entry.type]; + if (!kind) throw new Error(`layer entry ${entry.name} is of type ${JSON.stringify(entry.type)}, not a file, a directory or a link`); + const clean = (name, label) => { + if (name.startsWith("/")) throw new Error(`layer ${label} ${name} is absolute`); + const relative = name.replace(/^(?:\.\/)+/, "").replace(/\/+$/, ""); + if (relative === "" || relative === ".") return null; + const parts = relative.split("/"); + if (relative.includes("\0") || parts.some((part) => part === "" || part === "." || part === "..")) { + throw new Error(`layer ${label} ${name} is not a plain relative path`); + } + if (parts.some((part) => part.startsWith(".wh."))) throw new Error(`layer ${label} ${name} is a whiteout: the image only adds files`); + return relative; + }; + const name = clean(entry.name, "entry"); + if (name === null) { + if (kind !== "directory") throw new Error(`layer entry ${entry.name} replaces the image root`); + return null; + } + // A link's target is text and is never followed here. A hard link names a file of the image. + const target = kind === "hardlink" ? clean(entry.linkname, "hard link target") : kind === "link" ? entry.linkname : null; + if (kind === "hardlink" && target === null) throw new Error(`layer entry ${entry.name} is a hard link to the image root`); + return { name, kind, target, mode: entry.mode, size: entry.size, mtime: entry.mtime }; +} + +/** + * The entries of every layer of the image in an archive, in order. + * @param {string} archive + */ +export async function listOciLayers(archive) { + const image = readOciArchive(archive); + const layers = []; + for (const layer of image.layers) { + /** @type {NonNullable>[]} */ + const entries = []; + await image.readLayer(layer, (raw) => { + const entry = layerEntry(raw); + if (entry) entries.push(entry); + }); + layers.push({ digest: layer.digest, bytes: layer.size, entries }); + } + return { digest: image.descriptor.digest, layers }; +} + +/** + * Writes the file tree of the image in an archive to `destination`, which + * must be empty or absent: every layer in order. Returns the image digest. + * @param {string} archive + * @param {string} destination + */ +export async function unpackOciImage(archive, destination) { + const image = readOciArchive(archive); + mkdirSync(destination, { recursive: true }); + if (readdirSync(destination).length > 0) throw new Error(`${destination} is not empty`); + const stat = (file) => { + try { + return lstatSync(file); + } catch (error) { + if (/** @type {NodeJS.ErrnoException} */ (error).code === "ENOENT") return null; + throw error; + } + }; + /** The place of an entry, after its parents are known to be real directories. */ + const place = (name) => { + let current = destination; + const parts = name.split("/"); + for (const part of parts.slice(0, -1)) { + current = path.join(current, part); + const parent = stat(current); + if (parent === null) mkdirSync(current, { mode: 0o755 }); + else if (!parent.isDirectory()) throw new Error(`layer entry ${name} is behind ${path.relative(destination, current)}, which is not a directory`); + } + return path.join(current, parts.at(-1)); + }; + /** @type {Map} */ + const directoryModes = new Map(); + let files = 0; + for (const layer of image.layers) { + await image.readLayer(layer, async (raw, content) => { + const entry = layerEntry(raw); + if (entry === null) return; + const target = place(entry.name); + const existing = stat(target); + if (entry.kind === "directory") { + if (existing === null) mkdirSync(target, { mode: 0o755 }); + else if (!existing.isDirectory()) throw new Error(`a layer writes ${entry.name} again, as a directory`); + directoryModes.set(target, entry.mode); + return; + } + if (existing !== null) throw new Error(`a layer writes ${entry.name} again: the image only adds files`); + files += 1; + if (entry.kind === "link") { + symlinkSync(/** @type {string} */ (entry.target), target); + } else if (entry.kind === "hardlink") { + const source = place(/** @type {string} */ (entry.target)); + if (!stat(source)?.isFile()) throw new Error(`layer entry ${entry.name} is a hard link to ${entry.target}, which is not a file of the image`); + linkSync(source, target); + } else { + const fd = openSync(target, "wx", 0o600); + try { + for await (const chunk of content()) writeSync(fd, chunk); + } finally { + closeSync(fd); + } + chmodSync(target, entry.mode); + utimesSync(target, entry.mtime, entry.mtime); + } + }); + } + // Last, and children first: a directory may not be writable by its owner. + for (const [directory, mode] of [...directoryModes].sort(([left], [right]) => right.length - left.length)) chmodSync(directory, mode); + return { digest: image.descriptor.digest, layers: image.layers.length, files }; +} + +const SLIM_LAYERS = Object.freeze([ + "provider pack dependency files of 1 MiB or more", + "everything else", +]); +/** + * The layers of each variant, in order. Content that changes rarely is apart + * from content that changes with every Runner source change, so a new image + * shares its large layers with the one before it. The full image is the slim + * image plus two layers that hold the transport archives and their receipt + * and nothing else, so a reader can fetch all of them without the pack. + * + * "Dependency files" are the pack's files outside `dist`, which the Runner's + * own build writes. A large file under `dist` changes with the Runner's + * source, so it and its transport archive are with the content that does. + */ +export const RUNNER_IMAGE_LAYERS = Object.freeze({ + slim: SLIM_LAYERS, + full: Object.freeze([ + ...SLIM_LAYERS, + "transport archives of the files in layer 1", + "the other transport archives, the overlay archive and the receipt", + ]), +}); + +/** + * Verifies the layer rule on OCI archives of one platform's image. With both + * variants it also verifies that the full image starts with the slim image's + * layers. + * @param {{ slim?: string | null, full?: string | null }} archives + * @returns {Promise<{ problems: string[], layers: Record }>} + */ +export async function checkRunnerImageLayers({ slim = null, full = null }) { + if (slim === null && full === null) throw new Error("check-layers needs --slim, --full or both"); + /** @type {string[]} */ + const problems = []; + /** @type {Record} */ + const layers = {}; + const pack = RUNNER_IMAGE_PATHS.providerPack; + const archivesDirectory = RUNNER_IMAGE_PATHS.providerPackArchives; + const receiptPath = RUNNER_IMAGE_PATHS.providerPackArchiveReceipt; + const transport = (name) => name === receiptPath || name === archivesDirectory || name.startsWith(`${archivesDirectory}/`); + // Transport content anywhere else: under another directory, or under another name. + const transportLike = (name) => name.split("/").some((part) => part === receiptPath || part === archivesDirectory || part.endsWith(".tar.zst")); + /** @type {Record} */ + const images = {}; + for (const [variant, archive] of Object.entries({ slim, full })) { + if (archive === null) continue; + const image = await listOciLayers(archive); + const roles = RUNNER_IMAGE_LAYERS[variant]; + layers[variant] = image.layers.map((layer, position) => ({ + role: roles[position] ?? "unexpected layer", digest: layer.digest, bytes: layer.bytes, entries: layer.entries.length, + })); + if (image.layers.length !== roles.length) { + problems.push(`the ${variant} image has ${image.layers.length} layers, not ${roles.length}`); + continue; + } + const listed = image.layers.map((layer) => ({ + digest: layer.digest, + names: [...new Set(layer.entries.map((entry) => entry.name))], + files: [...new Set(layer.entries.filter((entry) => entry.kind !== "directory").map((entry) => entry.name))], + })); + images[variant] = listed; + const [large, rest] = listed; + // Each layer only adds. A directory may be in several layers; a file or a link is in one. + /** @type {Map} */ + const written = new Map(); + for (const layer of listed) { + const files = new Set(layer.files); + for (const name of layer.names) { + const entry = written.get(name) ?? { layers: 0, file: false }; + written.set(name, { layers: entry.layers + 1, file: entry.file || files.has(name) }); + } + } + summarize(problems, `${variant}: more than one layer writes`, [...written].filter(([, entry]) => entry.file && entry.layers > 1).map(([name]) => name)); + summarize(problems, `${variant}: the first layer holds entries outside the pack's dependency files`, + large.names.filter((name) => name !== pack && (!name.startsWith(`${pack}/`) || name === `${pack}/dist` || name.startsWith(`${pack}/dist/`)))); + summarize(problems, `${variant}: transport content is in the layers both variants share`, + [...large.names, ...rest.names].filter((name) => transport(name) || transportLike(name))); + if (!rest.files.includes(RUNNER_IMAGE_PATHS.manifest)) problems.push(`${variant}: the second layer does not hold ${RUNNER_IMAGE_PATHS.manifest}`); + } + if (images.full) { + const [large, , stable, changing] = images.full; + const numbered = (layer) => layer.files.filter((name) => name !== `${archivesDirectory}/overlay.tar.zst` && name !== receiptPath); + summarize(problems, "full: the third layer holds something other than transport archives", + stable.names.filter((name) => name !== archivesDirectory && !numbered(stable).includes(name))); + summarize(problems, "full: the last layer holds something other than transport content", changing.names.filter((name) => !transport(name))); + summarize(problems, "full: the last layer lacks", [receiptPath, `${archivesDirectory}/overlay.tar.zst`].filter((name) => !changing.files.includes(name))); + /** @type {{ path: string }[] | null} */ + let files = null; + if (changing.files.includes(receiptPath)) { + try { + files = JSON.parse(await readLayerFile(/** @type {string} */ (full), 3, receiptPath)).files; + } catch { + problems.push("full: the receipt in the last layer is not readable"); + } + } + if (Array.isArray(files)) { + // Archive N holds file N of the receipt. The receipt decides the layer. + const own = (file) => file.path === "dist" || file.path.startsWith("dist/"); + const archiveOf = (file) => `${archivesDirectory}/${files.indexOf(file)}.tar.zst`; + const expect = (label, actual, expected) => { + summarize(problems, `full: ${label} lacks`, expected.filter((name) => !actual.includes(name))); + summarize(problems, `full: ${label} holds, against the receipt,`, actual.filter((name) => !expected.includes(name))); + }; + expect("the first layer", large.files, files.filter((file) => !own(file)).map((file) => `${pack}/${file.path}`)); + expect("the third layer", numbered(stable), files.filter((file) => !own(file)).map(archiveOf)); + expect("the last layer", numbered(changing), files.filter(own).map(archiveOf)); + } + } + if (images.slim && images.full) { + // The same blobs in the same order, not only the same files. + const shared = images.slim.map((layer) => layer.digest); + if (canonicalJson(images.full.slice(0, shared.length).map((layer) => layer.digest)) !== canonicalJson(shared)) { + problems.push("the full image does not start with the layers of the slim image"); + } + } + return { problems, layers }; +} + +/** The text of one file in one layer of an archive. */ +async function readLayerFile(archive, position, name) { + const image = readOciArchive(archive); + /** @type {Buffer[] | null} */ + let chunks = null; + await image.readLayer(image.layers[position], async (raw, content) => { + if (layerEntry(raw)?.name !== name) return; + chunks = []; + for await (const chunk of content()) chunks.push(chunk); + }); + if (chunks === null) throw new Error(`${name} is not in layer ${position + 1}`); + return Buffer.concat(chunks).toString("utf8"); +} + +const COMMAND_FLAGS = Object.freeze({ + unpack: { out: "value" }, + "check-layers": { slim: "value", full: "value" }, +}); + +const invoked = process.argv[1] ? pathToFileURL(path.resolve(process.argv[1])).href : null; +if (invoked === import.meta.url) { + try { + const [command, ...rest] = process.argv.slice(2); + if (command !== "unpack" && command !== "check-layers") { + throw new Error("Usage: runner-image-oci.mjs unpack IMAGE.oci.tar --out DIR | check-layers [--slim TAR] [--full TAR]"); + } + const { positional, flags } = parseLayoutArguments(command, rest, COMMAND_FLAGS[command]); + if (command === "unpack") { + if (positional.length !== 1 || typeof flags.out !== "string") throw new Error("unpack takes exactly one OCI archive and --out DIR"); + const result = await unpackOciImage(path.resolve(positional[0]), path.resolve(flags.out)); + process.stdout.write(`Unpacked ${result.files} files from the ${result.layers} layers of ${result.digest} into ${path.resolve(flags.out)}.\n`); + } else { + if (positional.length > 0) throw new Error(`check-layers takes no positional argument: ${positional[0]}`); + const archive = (name) => (flags[name] === undefined ? null : path.resolve(String(flags[name]))); + const { problems, layers } = await checkRunnerImageLayers({ slim: archive("slim"), full: archive("full") }); + for (const [variant, rows] of Object.entries(layers)) { + for (const [position, layer] of rows.entries()) { + process.stdout.write(`${variant} layer ${position + 1}: ${layer.digest} ${String(layer.bytes).padStart(11)} bytes ${String(layer.entries).padStart(6)} entries ${layer.role}\n`); + } + } + if (problems.length > 0) { + process.stderr.write(`Runner image layer check failed:\n${problems.map((problem) => `- ${problem}`).join("\n")}\n`); + process.exit(1); + } + process.stdout.write("Runner image layer check passed.\n"); + } + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exit(1); + } +} diff --git a/server/src/__tests__/runner-image-identity.test.ts b/server/src/__tests__/runner-image-identity.test.ts index 2081c3963ef..af441f6b70a 100644 --- a/server/src/__tests__/runner-image-identity.test.ts +++ b/server/src/__tests__/runner-image-identity.test.ts @@ -189,17 +189,50 @@ describe("Runner image identity", () => { * the whole of what the build reads only while these three rules hold. */ describe("Runner image stages", () => { - const stages = [...dockerfile.matchAll(/^FROM (\S+)(?: AS (\S+))?$/gm)].map((match, index, all) => ({ - from: match[1], - name: match[2] ?? "", - lines: dockerfile.slice(match.index, all[index + 1]?.index ?? dockerfile.length).split("\n").filter((line) => !/^\s*#/.test(line)), + // BuildKit accepts an instruction in any case and with any indentation, and + // FROM with flags. A stage starts wherever BuildKit starts one; the first + // rule below then allows only the spelling the other rules can read. + const stagesOf = (text: string) => [...text.matchAll(/^[ \t]*from[ \t].*$/gim)].map((match, index, all) => ({ + from: /^FROM (\S+)(?: AS \S+)?$/.exec(match[0])?.[1] ?? "", + // The name BuildKit gives the stage, however the line is spelt. + name: (/\sas\s+(\S+)\s*$/i.exec(match[0])?.[1] ?? "").toLowerCase(), + lines: text.slice(match.index, all[index + 1]?.index ?? text.length).split("\n").filter((line) => !/^\s*#/.test(line)), })); + type Stage = ReturnType[number]; + /** The lines of a stage that read something and are not in the one spelling these rules read. */ + const misspelt = (stage: Stage) => stage.lines.filter((line, index) => (index === 0 + // No lowercase keyword, no indentation, no flag on FROM. + ? !/^FROM \S+ AS runner-artifacts[a-z-]*$/.test(line) + // No ADD or ONBUILD, no other flag, and no source on a continuation line. + : /^\s*(?:from|copy|add|onbuild)(?:\s|$)/i.test(line) + && !/^COPY (?:--link )?(?:--from=runner-artifacts[a-z-]* )?[^-\s]\S*(?: [^\s\\]+)+$/.test(line))); + const stages = stagesOf(dockerfile); const names = stages.map((stage) => stage.name); const runner = stages.filter((stage) => stage.name.startsWith("runner-artifacts")); - const copies = (stage: (typeof stages)[number]) => stage.lines.filter((line) => /^COPY\s/.test(line)); + const copies = (stage: Stage) => stage.lines.filter((line) => /^COPY\s/.test(line)); + + it("spell every FROM and COPY the one way these rules read, and use no ADD or ONBUILD", () => { + expect(runner.length).toBeGreaterThan(3); + for (const stage of runner) expect(misspelt(stage), stage.name).toEqual([]); + }); + + it.each([ + ["a lowercase stage keyword", "FROM scratch as runner-artifacts-extra\nCOPY secrets /\n"], + ["a lowercase from", "from scratch AS runner-artifacts-extra\n"], + ["a flag on FROM", "FROM --platform=linux/amd64 node:24 AS runner-artifacts-extra\n"], + ["a lowercase copy", "FROM scratch AS runner-artifacts-extra\ncopy secrets /\n"], + ["an indented COPY", "FROM scratch AS runner-artifacts-extra\n COPY secrets /\n"], + ["an ADD in any case", "FROM scratch AS runner-artifacts-extra\n add secrets /\n"], + ["a COPY continued on the next line", "FROM scratch AS runner-artifacts-extra\nCOPY package.json \\\n secrets /\n"], + ["a COPY with another flag", "FROM scratch AS runner-artifacts-extra\nCOPY --parents secrets /\n"], + ["an ONBUILD trigger", "FROM scratch AS runner-artifacts-extra\nONBUILD COPY secrets /\n"], + ])("would catch %s in a stage BuildKit reads as one of the Runner's", (_label, text) => { + const added = stagesOf(text).filter((stage) => stage.name.startsWith("runner-artifacts")); + expect(added).toHaveLength(1); + expect(misspelt(added[0])).not.toEqual([]); + }); it("start only from each other, from scratch, or from an image pinned by digest", () => { - expect(runner.length).toBeGreaterThan(3); for (const stage of runner) { if (stage.from === "scratch" || names.includes(stage.from)) { expect(stage.from === "scratch" || stage.from.startsWith("runner-artifacts"), stage.name).toBe(true); @@ -233,10 +266,23 @@ describe("Runner image stages", () => { expect([...sources].sort()).toEqual([...RUNNER_IMAGE_CONTEXT_INPUTS].sort()); }); - it("end in a file tree, and leave the default target and the application stages alone", () => { - const target = runner.find((stage) => stage.name === "runner-artifacts")!; - expect(target.from).toBe("scratch"); - expect(copies(target)).toEqual(["COPY --from=runner-artifacts-build /out/ /"]); + it("end in two file trees, the full one being the slim one plus the transport layers", () => { + // Each COPY --link is one layer that does not depend on the layers under + // it. The slim variant is the first two; the full variant starts from it, + // so it has the same two layers and then the transport archives. + const slim = runner.find((stage) => stage.name === "runner-artifacts-slim")!; + const full = runner.find((stage) => stage.name === "runner-artifacts")!; + const instructions = (stage: typeof slim) => stage.lines.filter((line) => /^[A-Z]+\s/.test(line)).slice(1); + expect(slim.from).toBe("scratch"); + expect(instructions(slim)).toEqual([ + "COPY --link --from=runner-artifacts-build /out-large/ /", + "COPY --link --from=runner-artifacts-build /out/ /", + ]); + expect(full.from).toBe("runner-artifacts-slim"); + expect(instructions(full)).toEqual([ + "COPY --link --from=runner-artifacts-build /out-archives/ /", + "COPY --link --from=runner-artifacts-build /out-transport/ /", + ]); // `docker build .` builds the last stage. expect(names.at(-1)).toBe("cloud"); for (const stage of stages) { diff --git a/server/src/__tests__/runner-image-layout.test.ts b/server/src/__tests__/runner-image-layout.test.ts index 3c2118400bf..1ef1b9babee 100644 --- a/server/src/__tests__/runner-image-layout.test.ts +++ b/server/src/__tests__/runner-image-layout.test.ts @@ -33,11 +33,15 @@ import { RUNNER_IMAGE_SCHEMA, canonicalJson, checkProviderPackArchives, - checkRunnerImageLayout, + checkRunnerImageLayout as checkLayout, parseLayoutArguments, writeRunnerImageManifest, } from "../../../scripts/runner-image-layout.mjs"; +/** The fixture is a full image, so most tests check it as one. */ +const checkRunnerImageLayout = (root: string, options: Omit[1], "variant"> = {}) => + checkLayout(root, { variant: "full", ...options }); + /** * The layout contract of the Runner image. Consumers copy the documented * paths out of the image and trust runner-image.json, so a tree that @@ -182,6 +186,7 @@ describe("Runner image layout contract", () => { platform: { os: "linux", architecture: "amd64" }, libc: { family: "glibc", baseline: "debian-bookworm", buildVersion: "2.36", runnerdRequires: "2.34" }, paths: RUNNER_IMAGE_PATHS, + fullImageOnly: ["providerPackArchiveReceipt", "providerPackArchives"], }); // One daemon, at the root for sandbox images and inside the build output // for a build that vendors the Runner. @@ -209,6 +214,7 @@ describe("Runner image layout contract", () => { ["a short inputs digest", (m) => { m.inputsDigest.digest = "abc"; }, /inputsDigest must be/], ["another build-inputs schema", (m) => { m.buildInputsDigest.schema = "paperclip.runner-build-inputs.v1"; }, /buildInputsDigest must be/], ["an abbreviated source revision", (m) => { m.sourceRevision = "abc1234"; }, /sourceRevision/], + ["transport paths claimed for both variants", (m) => { m.fullImageOnly = []; }, /fullImageOnly does not name the paths only the full image has/], ["no word on whether it is a publication build", (m) => { delete m.publication; }, /publication must say/], ["a publication build without a source revision", (m) => { m.sourceRevision = null; }, /publication build must have a source revision/], ["an unsupported platform", (m) => { m.platform = { os: "linux", architecture: "riscv64" }; }, /platform must be/], @@ -378,6 +384,59 @@ describe("Runner image layout contract", () => { }); }); + describe("variants", () => { + /** A slim tree: the full fixture without its transport archives and receipt. */ + function slim(tree: string) { + rmSync(path.join(tree, "provider-pack.archives"), { recursive: true }); + rmSync(path.join(tree, "provider-pack.archive.json")); + return tree; + } + + it("accepts a slim tree only as slim, and a full tree only as full", () => { + expect(checkLayout(out, { variant: "full" })).toEqual([]); + expect(checkLayout(out, { variant: "slim" })).toEqual([ + "provider-pack.archive.json (providerPackArchiveReceipt) is in a slim image; only the full image has the transport archives", + "provider-pack.archives (providerPackArchives) is in a slim image; only the full image has the transport archives", + ]); + slim(out); + expect(checkLayout(out, { variant: "slim", expectInputsDigest: INPUTS, expectSourceRevision: REVISION, expectArchitecture: "amd64" })).toEqual([]); + expect(checkLayout(out, { variant: "full" })).toEqual(expect.arrayContaining([ + "documented path provider-pack.archive.json (providerPackArchiveReceipt) is missing", + "documented path provider-pack.archives (providerPackArchives) is missing", + ])); + }); + + it("refuses a slim tree that holds only one of the two transport paths", () => { + rmSync(path.join(out, "provider-pack.archive.json")); + expect(checkLayout(out, { variant: "slim" }).join("\n")).toMatch(/provider-pack\.archives \(providerPackArchives\) is in a slim image/); + }); + + it("still checks everything else in a slim tree", () => { + appendFileSync(path.join(slim(out), "provider-pack/dist/cli/opencode-app-server-proxy.js"), "// later edit\n"); + expect(checkLayout(out, { variant: "slim" }).join("\n")).toMatch(/the provider pack dist tree does not match its tree digest/); + }); + + it("has no default: the variant is always given", () => { + // @ts-expect-error the variant is required + expect(() => checkLayout(out, {})).toThrow(/must be told the variant: slim or full/); + expect(() => checkLayout(out, { variant: "both" })).toThrow(/must be told the variant/); + const check = (...args: string[]) => spawnSync(process.execPath, [layoutScript, "check", out, ...args], { encoding: "utf8" }); + expect(check().stderr).toMatch(/check needs --variant slim or --variant full/); + expect(check("--variant", "both").stderr).toMatch(/check needs --variant slim or --variant full/); + expect(check("--variant", "slim").stderr).toMatch(/is in a slim image/); + const passed = spawnSync(process.execPath, [layoutScript, "check", slim(out), "--variant", "slim"], { encoding: "utf8" }); + expect(passed.status).toBe(0); + expect(passed.stdout).toMatch(/A slim image has no transport archives/); + }); + + it("writes one runner-image.json for both, which names the paths only the full image has", () => { + const manifest = manifestOf(out); + expect(manifest.fullImageOnly.map((name: keyof typeof RUNNER_IMAGE_PATHS) => RUNNER_IMAGE_PATHS[name])) + .toEqual(["provider-pack.archive.json", "provider-pack.archives"]); + expect(manifest.paths).toEqual(RUNNER_IMAGE_PATHS); + }); + }); + describe("transport archive round trip", () => { // The archives' bytes can match the receipt while the archives do not // hold the pack. Only extracting them, the way a sandbox does, shows it. @@ -555,7 +614,7 @@ describe("Runner image layout contract", () => { }); it("is part of the check command, which says which tar extracted the archives", async () => { - const check = (tree: string) => spawnSync(process.execPath, [layoutScript, "check", tree], { encoding: "utf8" }); + const check = (tree: string) => spawnSync(process.execPath, [layoutScript, "check", tree, "--variant", "full"], { encoding: "utf8" }); const passed = check(out); expect(passed.status).toBe(0); expect(passed.stdout).toMatch(/The transport archives were extracted with (GNU tar, as a sandbox extracts them,|bsdtar) and match the provider pack/); @@ -567,7 +626,7 @@ describe("Runner image layout contract", () => { expect(failed.stderr).toMatch(/transport archive overlay\.tar\.zst is not a zstd archive/); expect(failed.stdout).not.toMatch(/passed/); // A tree without a pack has nothing to extract, and the command says so. - const unstamped = spawnSync(process.execPath, [layoutScript, "check", await image({ stamped: false }), "--allow-unstamped"], { encoding: "utf8" }); + const unstamped = spawnSync(process.execPath, [layoutScript, "check", await image({ stamped: false }), "--variant", "full", "--allow-unstamped"], { encoding: "utf8" }); expect(unstamped.status).toBe(0); expect(unstamped.stdout).toMatch(/no provider pack, so there are no transport archives to extract/); }); @@ -634,7 +693,7 @@ describe("Runner image layout contract", () => { }); describe("command line", () => { - const check = (...args: string[]) => spawnSync(process.execPath, [layoutScript, "check", ...args], { encoding: "utf8" }); + const check = (...args: string[]) => spawnSync(process.execPath, [layoutScript, "check", ...args, "--variant", "full"], { encoding: "utf8" }); it("passes for a valid tree and fails for a wrong expectation", () => { expect(check(out, "--expect-inputs-digest", INPUTS, "--expect-source-revision", REVISION, "--expect-architecture", "amd64").status).toBe(0); @@ -675,6 +734,9 @@ describe("Runner image layout contract", () => { expect(() => parseLayoutArguments("write", ["--out", "o", "--source-revision", ""])).toThrow(/--source-revision needs a value/); expect(parseLayoutArguments("write", ["--out", "o", "--publication"]).flags).toEqual({ out: "o", publication: true }); expect(() => parseLayoutArguments("check", ["tree", "--publication"])).toThrow(/check does not take --publication/); + // Another script can give the same parser its own flags. + expect(parseLayoutArguments("unpack", ["a.tar", "--out", "d"], { out: "value" })).toEqual({ positional: ["a.tar"], flags: { out: "d" } }); + expect(() => parseLayoutArguments("unpack", ["--variant", "slim"], { out: "value" })).toThrow(/unpack does not take --variant/); expect(() => parseLayoutArguments("write", ["--expect-architecture", "arm64"])).toThrow(/write does not take/); }); }); diff --git a/server/src/__tests__/runner-image-oci.test.ts b/server/src/__tests__/runner-image-oci.test.ts new file mode 100644 index 00000000000..393fc2a3896 --- /dev/null +++ b/server/src/__tests__/runner-image-oci.test.ts @@ -0,0 +1,360 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readFileSync, + readlinkSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { gzipSync } from "node:zlib"; +import { afterAll, describe, expect, it } from "vitest"; +import { + RUNNER_IMAGE_LAYERS, + checkRunnerImageLayers, + listOciLayers, + readOciArchive, + unpackOciImage, +} from "../../../scripts/runner-image-oci.mjs"; + +/** + * The OCI archives of the Runner image. CI builds each variant as an archive + * only, unpacks it, and checks the tree that comes out, so the reader must + * give exactly the files of the image and must not write anything an additive + * layer of a `FROM scratch` image cannot hold. + */ + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); +const ociScript = path.join(repoRoot, "scripts", "runner-image-oci.mjs"); +const sha256 = (bytes: Buffer | string) => `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +const scratch = mkdtempSync(path.join(tmpdir(), "runner-image-oci-")); +afterAll(() => { + spawnSync("chmod", ["-R", "u+rwX", scratch]); + rmSync(scratch, { recursive: true, force: true }); +}); +let sequence = 0; +const fresh = () => path.join(scratch, String(sequence += 1)); + +type Entry = { name: string; type?: string; content?: string | Buffer; linkname?: string; mode?: number }; + +/** A tar stream of exactly these entries, in the POSIX format. Nothing is added or cleaned. */ +function tarOf(entries: Entry[]) { + const blocks: Buffer[] = []; + for (const entry of entries) { + const type = entry.type ?? (entry.name.endsWith("/") ? "5" : "0"); + const content = Buffer.from(entry.content ?? ""); + const header = Buffer.alloc(512); + const field = (value: string, start: number, length: number) => { header.write(value, start, length, "utf8"); }; + const octal = (value: number, start: number, length: number) => field(`${value.toString(8).padStart(length - 1, "0")}\0`, start, length); + expect(Buffer.byteLength(entry.name)).toBeLessThanOrEqual(100); + field(entry.name, 0, 100); + octal(entry.mode ?? (type === "5" ? 0o755 : 0o644), 100, 8); + octal(0, 108, 8); + octal(0, 116, 8); + octal(type === "0" ? content.length : 0, 124, 12); + octal(946684800, 136, 12); + field(" ", 148, 8); + field(type, 156, 1); + field(entry.linkname ?? "", 157, 100); + field("ustar\0", 257, 6); + field("00", 263, 2); + octal(header.reduce((sum, byte) => sum + byte, 0), 148, 7); + blocks.push(header); + if (type === "0") blocks.push(content, Buffer.alloc((512 - (content.length % 512)) % 512)); + } + return Buffer.concat([...blocks, Buffer.alloc(1024)]); +} + +/** Entries for a layer that holds these files, with every parent directory first. */ +function layerOf(files: Record): Entry[] { + const entries = new Map(); + for (const [name, content] of Object.entries(files)) { + const parts = name.split("/"); + for (let depth = 1; depth < parts.length; depth += 1) { + const directory = `${parts.slice(0, depth).join("/")}/`; + if (!entries.has(directory)) entries.set(directory, { name: directory }); + } + entries.set(name, { name, content }); + } + return [...entries.values()]; +} + +/** + * What `docker buildx build --output type=oci` writes for one platform, with + * these layers: tar streams, or the files of a layer. `members` names the + * members of the archive the way buildx does; a test can name them otherwise. + */ +function ociArchive(layers: (Buffer | Record)[], { members = ["blobs", "index.json", "oci-layout"], tamper = (_layout: string) => {} } = {}) { + const root = fresh(); + const layout = path.join(root, "layout"); + mkdirSync(path.join(layout, "blobs", "sha256"), { recursive: true }); + const blob = (bytes: Buffer) => { + writeFileSync(path.join(layout, "blobs", "sha256", sha256(bytes).slice(7)), bytes); + return { digest: sha256(bytes), size: bytes.length }; + }; + const descriptors = layers.map((layer) => ({ + mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", + ...blob(gzipSync(Buffer.isBuffer(layer) ? layer : tarOf(layerOf(layer)))), + })); + const manifest = blob(Buffer.from(JSON.stringify({ schemaVersion: 2, config: blob(Buffer.from("{}")), layers: descriptors }))); + writeFileSync(path.join(layout, "index.json"), JSON.stringify({ schemaVersion: 2, manifests: [{ mediaType: "application/vnd.oci.image.manifest.v1+json", ...manifest }] })); + writeFileSync(path.join(layout, "oci-layout"), '{"imageLayoutVersion":"1.0.0"}'); + tamper(layout); + const archive = path.join(root, "image.oci.tar"); + expect(spawnSync("tar", ["-cf", archive, "-C", layout, ...members]).status).toBe(0); + return { archive, digest: manifest.digest, layers: descriptors }; +} + +describe("OCI archive reader", () => { + it("reads the image, its layers and their entries", async () => { + const image = ociArchive([{ "a/one": "1" }, { "a/two": "22", top: "t" }]); + const read = readOciArchive(image.archive); + expect(read.descriptor.digest).toBe(image.digest); + expect(read.layers.map((layer) => layer.digest)).toEqual(image.layers.map((layer) => layer.digest)); + const listed = await listOciLayers(image.archive); + expect(listed.digest).toBe(image.digest); + expect(listed.layers.map((layer) => layer.entries.map((entry) => `${entry.kind} ${entry.name}`))).toEqual([ + ["directory a", "file a/one"], + ["directory a", "file a/two", "file top"], + ]); + }); + + it("finds a member only under the name buildx gives it", () => { + // `tar -cf x -C layout .` names the members ./index.json and ./blobs/... + // A reader that accepted those would read archives no registry tool reads. + expect(() => readOciArchive(ociArchive([{ a: "1" }], { members: ["."] }).archive)).toThrow(/has no member index\.json/); + }); + + it("refuses a manifest or a layer that does not have its digest", async () => { + const swapped = ociArchive([{ a: "1" }], { + tamper: (layout) => { + const index = JSON.parse(readFileSync(path.join(layout, "index.json"), "utf8")); + writeFileSync(path.join(layout, "blobs", "sha256", index.manifests[0].digest.slice(7)), '{"schemaVersion":2,"layers":[]}'); + }, + }); + expect(() => readOciArchive(swapped.archive)).toThrow(/the manifest does not have the digest the index names/); + const other = gzipSync(tarOf(layerOf({ a: "another file of the same length?" }))); + const changed = ociArchive([{ a: "1" }], { + tamper: (layout) => { + const index = JSON.parse(readFileSync(path.join(layout, "index.json"), "utf8")); + const manifest = JSON.parse(readFileSync(path.join(layout, "blobs", "sha256", index.manifests[0].digest.slice(7)), "utf8")); + const layer = path.join(layout, "blobs", "sha256", manifest.layers[0].digest.slice(7)); + // Other bytes of the same length, so only the digest can tell. + writeFileSync(layer, Buffer.concat([other, Buffer.alloc(4096)]).subarray(0, readFileSync(layer).length)); + }, + }); + await expect(listOciLayers(changed.archive)).rejects.toThrow(); + const index = ociArchive([{ a: "1" }], { + tamper: (layout) => writeFileSync(path.join(layout, "index.json"), JSON.stringify({ schemaVersion: 2, manifests: [] })), + }); + expect(() => readOciArchive(index.archive)).toThrow(/must hold exactly one image manifest/); + }); + + it("reads layers the system's tar wrote, with long names and links", async () => { + // GNU tar and bsdtar write long names in two different ways. BuildKit uses a third. + const directory = fresh(); + const deep = `node_modules/.pnpm/${"a".repeat(60)}/node_modules/${"b".repeat(60)}/${"c".repeat(40)}`; + mkdirSync(path.join(directory, deep), { recursive: true }); + writeFileSync(path.join(directory, deep, "file.bin"), "long"); + symlinkSync(`${deep}/file.bin`, path.join(directory, "link")); + const tarball = spawnSync("tar", ["-cf", "-", "-C", directory, "."], { maxBuffer: 16 * 1024 * 1024 }); + expect(tarball.status).toBe(0); + const { archive } = ociArchive([tarball.stdout]); + const entries = (await listOciLayers(archive)).layers[0].entries; + expect(entries.find((entry) => entry.name === `${deep}/file.bin`)).toMatchObject({ kind: "file", size: 4 }); + expect(entries.find((entry) => entry.name === "link")).toMatchObject({ kind: "link", target: `${deep}/file.bin` }); + const out = fresh(); + await unpackOciImage(archive, out); + expect(readFileSync(path.join(out, deep, "file.bin"), "utf8")).toBe("long"); + expect(readFileSync(path.join(out, "link"), "utf8")).toBe("long"); + }); +}); + +describe("unpacking an image", () => { + it("writes every layer in order: files with their modes, directories, links and hard links", async () => { + const image = ociArchive([ + tarOf([ + { name: "bin/" }, + { name: "bin/tool", content: "#!/bin/sh\n", mode: 0o755 }, + { name: "bin/again", type: "1", linkname: "bin/tool" }, + { name: "current", type: "2", linkname: "bin/tool" }, + { name: "sealed/", mode: 0o555 }, + { name: "sealed/read-only", content: "ro", mode: 0o444 }, + ]), + tarOf([{ name: "./" }, { name: "./bin/" }, { name: "./bin/other", content: "x" }, { name: "new/deep/file", content: "made without its directories" }]), + ]); + const out = fresh(); + const result = await unpackOciImage(image.archive, out); + expect(result).toEqual({ digest: image.digest, layers: 2, files: 6 }); + expect(readFileSync(path.join(out, "bin/tool"), "utf8")).toBe("#!/bin/sh\n"); + expect(lstatSync(path.join(out, "bin/tool")).mode & 0o7777).toBe(0o755); + expect(lstatSync(path.join(out, "bin/again")).ino).toBe(lstatSync(path.join(out, "bin/tool")).ino); + expect(readlinkSync(path.join(out, "current"))).toBe("bin/tool"); + expect(lstatSync(path.join(out, "sealed")).mode & 0o7777).toBe(0o555); + expect(lstatSync(path.join(out, "sealed/read-only")).mode & 0o7777).toBe(0o444); + expect(readFileSync(path.join(out, "bin/other"), "utf8")).toBe("x"); + expect(readFileSync(path.join(out, "new/deep/file"), "utf8")).toBe("made without its directories"); + expect(Math.round(lstatSync(path.join(out, "bin/tool")).mtimeMs / 1000)).toBe(946684800); + }); + + it("writes large files whole", async () => { + const large = Buffer.alloc(3 * 1024 * 1024 + 17, "n"); + const { archive } = ociArchive([tarOf([{ name: "large.bin", content: large }, { name: "after", content: "still aligned" }])]); + const out = fresh(); + await unpackOciImage(archive, out); + expect(sha256(readFileSync(path.join(out, "large.bin")))).toBe(sha256(large)); + expect(readFileSync(path.join(out, "after"), "utf8")).toBe("still aligned"); + }); + + it.each<[string, Entry[][], RegExp]>([ + ["a whiteout", [[{ name: "a/" }, { name: "a/.wh.gone" }]], /is a whiteout: the image only adds files/], + ["an opaque directory marker", [[{ name: "a/" }, { name: "a/.wh..wh..opq" }]], /is a whiteout/], + ["an absolute path", [[{ name: "/etc/passwd", content: "x" }]], /is absolute/], + ["a path that leaves the tree", [[{ name: "../outside", content: "x" }]], /is not a plain relative path/], + ["a path that goes up inside the tree", [[{ name: "a/" }, { name: "a/../b", content: "x" }]], /is not a plain relative path/], + ["a file behind a link", [[{ name: "link", type: "2", linkname: "/tmp" }, { name: "link/file", content: "x" }]], /is behind link, which is not a directory/], + ["a file behind a link of an earlier layer", [[{ name: "link", type: "2", linkname: "." }], [{ name: "link/file", content: "x" }]], /is behind link/], + ["a file behind a file", [[{ name: "a", content: "x" }, { name: "a/b", content: "y" }]], /is behind a, which is not a directory/], + ["a file a later layer writes again", [[{ name: "a", content: "1" }], [{ name: "a", content: "2" }]], /a layer writes a again: the image only adds files/], + ["a link over a file", [[{ name: "a", content: "1" }], [{ name: "a", type: "2", linkname: "b" }]], /a layer writes a again/], + ["a directory over a file", [[{ name: "a", content: "1" }], [{ name: "a/" }]], /a layer writes a again, as a directory/], + ["a file over a directory", [[{ name: "a/" }], [{ name: "a", content: "1" }]], /a layer writes a again/], + ["a hard link to a path outside the tree", [[{ name: "a", type: "1", linkname: "../../etc/passwd" }]], /hard link target ..\/..\/etc\/passwd is not a plain relative path/], + ["a hard link to nothing", [[{ name: "a", type: "1", linkname: "missing" }]], /is a hard link to missing, which is not a file of the image/], + ["a hard link to a link", [[{ name: "l", type: "2", linkname: "/etc/passwd" }, { name: "a", type: "1", linkname: "l" }]], /which is not a file of the image/], + ["a device", [[{ name: "dev", type: "3" }]], /not a file, a directory or a link/], + ["a fifo", [[{ name: "pipe", type: "6" }]], /not a file, a directory or a link/], + ["a file in place of the root", [[{ name: "./", type: "0", content: "x" }]], /replaces the image root/], + ])("refuses %s", async (_label, layers, expected) => { + const out = fresh(); + await expect(unpackOciImage(ociArchive(layers.map(tarOf)).archive, out)).rejects.toThrow(expected); + // Nothing was written outside the destination. + expect(existsSync(path.join(path.dirname(out), "outside"))).toBe(false); + }); + + it("writes only into an empty directory", async () => { + const out = fresh(); + mkdirSync(out, { recursive: true }); + writeFileSync(path.join(out, "left-over"), "x"); + await expect(unpackOciImage(ociArchive([{ a: "1" }]).archive, out)).rejects.toThrow(/is not empty/); + }); + + it("is a command that says which image it unpacked", () => { + const image = ociArchive([{ "a/one": "1" }]); + const out = fresh(); + const run = (...args: string[]) => spawnSync(process.execPath, [ociScript, "unpack", ...args], { encoding: "utf8" }); + const passed = run(image.archive, "--out", out); + expect(passed.status).toBe(0); + expect(passed.stdout).toContain(`Unpacked 1 files from the 1 layers of ${image.digest}`); + expect(readFileSync(path.join(out, "a/one"), "utf8")).toBe("1"); + expect(run(image.archive).stderr).toMatch(/unpack takes exactly one OCI archive and --out DIR/); + expect(run(image.archive, "--out", out).stderr).toMatch(/is not empty/); + expect(run(image.archive, "--into", out).stderr).toMatch(/unpack does not take --into/); + expect(spawnSync(process.execPath, [ociScript, "extract", image.archive], { encoding: "utf8" }).stderr).toMatch(/Usage/); + }); +}); + +describe("image layers", () => { + // Archive N is file N of the receipt. The last file is the Runner's own output. + const receipt = JSON.stringify({ files: [{ path: "node_modules/big/big.bin" }, { path: "node_modules/node/bin/node" }, { path: "dist/cli/sidecar.cjs" }] }); + const large = { "provider-pack/node_modules/node/bin/node": "node", "provider-pack/node_modules/big/big.bin": "big" }; + const rest = { + "runner-image.json": "{}", "paperclip-runnerd": "elf", "pnpm-lock.yaml": "lock", + "provider-pack/provider-pack.json": "{}", "provider-pack/dist/cli/sidecar.cjs": "sidecar", + }; + const stable = { "provider-pack.archives/0.tar.zst": "0", "provider-pack.archives/1.tar.zst": "1" }; + const changing = { "provider-pack.archives/2.tar.zst": "2", "provider-pack.archives/overlay.tar.zst": "overlay", "provider-pack.archive.json": receipt }; + type Layers = Record[]; + const problemsOf = async (archives: { slim?: Layers; full?: Layers }) => (await checkRunnerImageLayers({ + slim: archives.slim ? ociArchive(archives.slim).archive : null, full: archives.full ? ociArchive(archives.full).archive : null, + })).problems.join("\n"); + + it("accepts a full image that is the slim image plus the two transport layers", async () => { + const result = await checkRunnerImageLayers({ slim: ociArchive([large, rest]).archive, full: ociArchive([large, rest, stable, changing]).archive }); + expect(result.problems).toEqual([]); + expect(result.layers.slim.map((layer) => layer.role)).toEqual(RUNNER_IMAGE_LAYERS.slim); + expect(result.layers.full.map((layer) => layer.role)).toEqual(RUNNER_IMAGE_LAYERS.full); + expect(RUNNER_IMAGE_LAYERS.full.slice(0, 2)).toEqual(RUNNER_IMAGE_LAYERS.slim); + // The same blobs, not only the same files. + expect(result.layers.full.slice(0, 2).map((layer) => layer.digest)).toEqual(result.layers.slim.map((layer) => layer.digest)); + }); + + it("checks one variant alone, and needs at least one", async () => { + expect(await problemsOf({ slim: [large, rest] })).toBe(""); + expect(await problemsOf({ full: [large, rest, stable, changing] })).toBe(""); + await expect(checkRunnerImageLayers({})).rejects.toThrow(/needs --slim, --full or both/); + }); + + it.each<[string, { slim?: Layers; full?: Layers }, RegExp]>([ + ["a full image whose shared layers are not the slim image's blobs", + { slim: [large, rest], full: [large, { ...rest, "paperclip-runnerd": "another build" }, stable, changing] }, + /the full image does not start with the layers of the slim image/], + ["a full image with the slim layers in another order", + { slim: [large, rest], full: [rest, large, stable, changing] }, /the full image does not start with the layers of the slim image/], + ["a slim image with the transport layers", { slim: [large, rest, stable, changing] }, /the slim image has 4 layers, not 2/], + ["a slim image that holds a transport archive", { slim: [large, { ...rest, "provider-pack.archives/1.tar.zst": "1" }] }, + /slim: transport content is in the layers both variants share \(2\): provider-pack\.archives, provider-pack\.archives\/1\.tar\.zst/], + ["a slim image that holds a transport archive under another directory", { slim: [large, { ...rest, "provider-pack/spare/0.tar.zst": "0" }] }, + /slim: transport content is in the layers both variants share \(1\): provider-pack\/spare\/0\.tar\.zst/], + ["a slim image that holds the archives directory somewhere else", { slim: [large, { ...rest, "backup/provider-pack.archives/overlay.bin": "x" }] }, + /slim: transport content is in the layers both variants share \(2\): backup\/provider-pack\.archives, backup\/provider-pack\.archives\/overlay\.bin/], + ["a slim image that holds a receipt inside the pack", { slim: [{ ...large, "provider-pack/node_modules/provider-pack.archive.json": "{}" }, rest] }, + /slim: transport content is in the layers both variants share \(1\): provider-pack\/node_modules\/provider-pack\.archive\.json/], + ["a file that two layers of the slim image write", { slim: [large, { ...rest, "provider-pack/node_modules/node/bin/node": "another node" }] }, + /slim: more than one layer writes \(1\): provider-pack\/node_modules\/node\/bin\/node/], + ["a file that a transport layer writes over a shared layer", { full: [large, rest, stable, { ...changing, "pnpm-lock.yaml": "another lock" }] }, + /full: more than one layer writes \(1\): pnpm-lock\.yaml/], + ["an archive that both transport layers write", { full: [large, rest, stable, { ...changing, "provider-pack.archives/1.tar.zst": "another 1" }] }, + /full: more than one layer writes \(1\): provider-pack\.archives\/1\.tar\.zst/], + ["a full image in a single layer", { full: [{ ...large, ...rest, ...stable, ...changing }] }, /the full image has 1 layers, not 4/], + ["the receipt in a shared layer", { full: [large, { ...rest, "provider-pack.archive.json": receipt }, stable, changing] }, + /full: transport content is in the layers both variants share/], + ["a file outside the pack in the first layer", { full: [{ ...large, "paperclip-runnerd": "elf" }, rest, stable, changing] }, + /full: the first layer holds entries outside the pack's dependency files/], + ["the Runner's own large output in the first layer", { full: [{ ...large, "provider-pack/dist/cli/sidecar.cjs": "sidecar" }, rest, stable, changing] }, + /full: the first layer holds entries outside the pack's dependency files/], + ["a large dependency file left in the layer that changes with every build", + { full: [{ "provider-pack/node_modules/node/bin/node": "node" }, { ...rest, "provider-pack/node_modules/big/big.bin": "big" }, stable, changing] }, + /full: the first layer lacks \(1\): provider-pack\/node_modules\/big\/big\.bin/], + ["the overlay archive in the stable transport layer", + { full: [large, rest, { ...stable, "provider-pack.archives/overlay.tar.zst": "overlay" }, changing] }, + /full: the third layer holds something other than transport archives/], + ["a pack file in the last layer", { full: [large, rest, stable, { ...changing, "provider-pack/extra.json": "{}" }] }, + /full: the last layer holds something other than transport content/], + ["a stable archive in the layer that changes with every build", + { full: [large, rest, { "provider-pack.archives/0.tar.zst": "0" }, { ...changing, "provider-pack.archives/1.tar.zst": "1" }] }, + /full: the third layer lacks \(1\): provider-pack\.archives\/1\.tar\.zst/], + ["the archive of the Runner's own output in the stable layer", + { full: [large, rest, { ...stable, "provider-pack.archives/2.tar.zst": "2" }, { "provider-pack.archives/overlay.tar.zst": "overlay", "provider-pack.archive.json": receipt }] }, + /full: the third layer holds, against the receipt, \(1\): provider-pack\.archives\/2\.tar\.zst/], + ["an archive the receipt does not list", { full: [large, rest, { ...stable, "provider-pack.archives/3.tar.zst": "3" }, changing] }, + /full: the third layer holds, against the receipt, \(1\): provider-pack\.archives\/3\.tar\.zst/], + ["no receipt in the last layer", + { full: [large, rest, stable, { "provider-pack.archives/2.tar.zst": "2", "provider-pack.archives/overlay.tar.zst": "overlay" }] }, + /full: the last layer lacks \(1\): provider-pack\.archive\.json/], + ])("refuses %s", async (_label, archives, expected) => { + expect(await problemsOf(archives)).toMatch(expected); + }); + + it("is a command that prints the layer table of each variant", () => { + const run = (...args: string[]) => spawnSync(process.execPath, [ociScript, "check-layers", ...args], { encoding: "utf8" }); + const slim = ociArchive([large, rest]).archive; + const passed = run("--slim", slim, "--full", ociArchive([large, rest, stable, changing]).archive); + expect(passed.status).toBe(0); + expect(passed.stdout).toMatch(/slim layer 2: sha256:[0-9a-f]{64} +\d+ bytes +\d+ entries +everything else/); + expect(passed.stdout).toMatch(/full layer 4: sha256:[0-9a-f]{64} +\d+ bytes +\d+ entries +the other transport archives, the overlay archive and the receipt/); + const failed = run("--slim", slim, "--full", ociArchive([large, rest, stable]).archive); + expect(failed.status).toBe(1); + expect(failed.stderr).toMatch(/the full image has 3 layers, not 4/); + expect(run().stderr).toMatch(/needs --slim, --full or both/); + expect(run(slim).stderr).toMatch(/takes no positional argument/); + expect(run("--variant", "slim").stderr).toMatch(/check-layers does not take --variant/); + }); +}); diff --git a/server/src/__tests__/runner-image-workflow.test.ts b/server/src/__tests__/runner-image-workflow.test.ts index 36cc97d438a..a1f9e565e68 100644 --- a/server/src/__tests__/runner-image-workflow.test.ts +++ b/server/src/__tests__/runner-image-workflow.test.ts @@ -47,13 +47,28 @@ describe("Runner image workflow", () => { for (const script of scripts) expect(script).not.toContain("${{"); }); + it("gives every image build one export and a time limit", () => { + // A command with two exports stalled on the hosted runners until the job + // was cancelled. One export for each command, and a limit on each step, + // so a stall fails in minutes. + const building = workflow.split(/\n(?= {6}- name: )/).filter((step) => step.includes("docker buildx build")); + expect(building.length).toBeGreaterThanOrEqual(3); + for (const step of building) { + expect(step).toMatch(/^ {8}timeout-minutes: \d+$/m); + for (const command of step.split("docker buildx build").slice(1)) expect(command.match(/--output /g) ?? []).toHaveLength(1); + } + // The trees the checks read come from the archives, not from a second export. + expect(workflow).toContain("node scripts/runner-image-oci.mjs unpack"); + expect(workflow.match(/type=local/g)).toHaveLength(2); + }); + it("runs for a pull request that changes any input of the image identity", () => { const paths = [...workflow.matchAll(/^ {6}- (\S+)$/gm)].map((match) => match[1]); const covered = (file: string) => paths.some((pattern) => pattern === file || (pattern.endsWith("/**") && (file === pattern.slice(0, -3) || file.startsWith(pattern.slice(0, -2))))); // .dockerignore is not in the identity, but it decides which files the build receives. for (const input of ["Dockerfile", ".dockerignore", ".github/workflows/runner-image.yml", "docker/runner-image/Dockerfile.smoke", - ...RUNNER_IMAGE_CONTEXT_INPUTS]) { + "scripts/runner-image-oci.mjs", ...RUNNER_IMAGE_CONTEXT_INPUTS]) { expect(covered(input), input).toBe(true); } }); diff --git a/server/src/services/native-runtime/provider-pack-archive.test.ts b/server/src/services/native-runtime/provider-pack-archive.test.ts index ec1b1ec1916..ee891ed4925 100644 --- a/server/src/services/native-runtime/provider-pack-archive.test.ts +++ b/server/src/services/native-runtime/provider-pack-archive.test.ts @@ -94,6 +94,56 @@ async function layeredFixture() { return { root, pack, receipt, archive }; } +it("writes the same archive for the same file every time, with nothing after the frame", async () => { + // Ending the compressor while its last chunk was still queued made Node + // finish the zstd stream twice, and about half the archives got an empty + // frame at their end. An image layer of these archives then never repeated. + const emptyFrame = Buffer.from("28b52ffd2000010000", "hex"); + const seen = new Set(); + for (let trial = 0; trial < 12; trial += 1) { + const { pack } = await fixture(); + const large = path.join(pack, "bin", "large"); + await fs.writeFile(large, Buffer.alloc(1024 * 1024, 42), { mode: 0o755 }); + await fs.utimes(large, 0, 0); + const receipt = await createProviderPackLayers(pack); + seen.add(receipt.files[0].archive.sha256); + for (const name of ["0.tar.zst", "overlay.tar.zst"]) { + const bytes = await fs.readFile(path.join(`${pack}.archives`, name)); + expect(bytes.subarray(-emptyFrame.length).equals(emptyFrame), name).toBe(false); + } + } + expect([...seen]).toHaveLength(1); +}); + +it("names the archives of dependency files without regard to the Runner's own large files", async () => { + // Archive N holds file N of the receipt. `dist` sorts before `node_modules`, + // so in path order one more large file in the Runner's build output renamed + // the archive of every dependency file. + const large = Buffer.alloc(1024 * 1024, 42); + const receiptOf = async (own: string[]) => { + const { pack } = await fixture(); + for (const file of ["node_modules/dependency/tool", "bin/large", ...own]) { + await fs.mkdir(path.dirname(path.join(pack, file)), { recursive: true }); + await fs.writeFile(path.join(pack, file), large, { mode: 0o755 }); + await fs.utimes(path.join(pack, file), 0, 0); + } + const receipt = await createProviderPackLayers(pack); + const archive = (await readProviderPackArchive(pack, digest))!; + if (!("kind" in archive)) throw Error("Expected layered archive"); + // What a reader pairs: file N of the receipt with N.tar.zst. + const paired = await Promise.all(archive.files.map(async (file: { path: string; archive: { sha256: string } }, index: number) => + [file.path, `${index}.tar.zst`, createHash("sha256").update(await fs.readFile(path.join(`${pack}.archives`, `${index}.tar.zst`))).digest("hex") === file.archive.sha256])); + return { paths: receipt.files.map((file: { path: string }) => file.path), paired }; + }; + const before = await receiptOf(["dist/cli/one.js"]); + const after = await receiptOf(["dist/cli/one.js", "dist/a-new-large-file.js", "dist/cli/two.js"]); + expect(before.paths).toEqual(["bin/large", "node_modules/dependency/tool", "dist/cli/one.js"]); + expect(after.paths).toEqual(["bin/large", "node_modules/dependency/tool", "dist/a-new-large-file.js", "dist/cli/one.js", "dist/cli/two.js"]); + // The dependency files keep their archive names, and every pairing a reader makes still holds. + expect(after.paired.slice(0, 2)).toEqual(before.paired.slice(0, 2)); + expect(after.paired.every(([, , matches]) => matches)).toBe(true); +}); + it.skipIf(!tar || !hasZstd)("reuses only byte-matched image files, keeps modes, and never shares writable hardlinks", async () => { const { root, pack, archive } = await layeredFixture(); const remote = await runner(root);