diff --git a/.github/workflows/docker-runner-check.yml b/.github/workflows/docker-runner-check.yml index 6e72256daf3..eb5135eebea 100644 --- a/.github/workflows/docker-runner-check.yml +++ b/.github/workflows/docker-runner-check.yml @@ -35,48 +35,15 @@ on: description: "Build the canonical eval report viewer on hosted Linux" type: boolean default: false - pull_request: - paths: - - .github/workflows/docker-runner-check.yml - - Dockerfile - - .dockerignore - - scripts/check-docker-runner-cache.sh - - packages/paperclip-runner/rust-toolchain.toml - - packages/paperclip-runner/runner/** - - packages/paperclip-runner/protocol/** permissions: {} concurrency: # Publishing a newer image must not discard an earlier verification's evidence. - group: docker-runner-check-${{ github.event.pull_request.number || github.ref }}-${{ inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} + group: docker-runner-check-${{ github.ref }}-${{ inputs.verify_source && 'verification' || inputs.verify_public_install && 'public-install' || inputs.build_eval_viewer && 'viewer' || 'image' }} cancel-in-progress: true jobs: - runner: - if: github.event_name == 'pull_request' - name: Compile isolated native Runner - runs-on: ubuntu-latest - timeout-minutes: 20 - permissions: - contents: read - steps: - - name: Checkout repository - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 - with: - persist-credentials: false - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 - # Compile the real target, then change source in a disposable context. - # A fresh builder must import dependencies and produce changed binary metadata. - # The baseline build anonymously seeds from the public BuildKit cache at - # ghcr.io/paperclipai/paperclip:buildcache-{amd64,arm64}; the verification - # build imports only this run's locally exported cache on a fresh builder. - # No registry credentials or image publication. - - name: Verify native build and dependency cache reuse - run: bash scripts/check-docker-runner-cache.sh - - authorize_manual: if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest diff --git a/.github/workflows/pr-merge-review.yml b/.github/workflows/pr-merge-review.yml index 193119c4581..888060e889e 100644 --- a/.github/workflows/pr-merge-review.yml +++ b/.github/workflows/pr-merge-review.yml @@ -6,7 +6,6 @@ on: - PR - Runner Windows - Sentry SDK contract - - Docker Runner check - Runner image - commitperclip PR Review - Storybook Visual diff --git a/doc/DOCKER.md b/doc/DOCKER.md index 86b7d32d261..7585abe97e2 100644 --- a/doc/DOCKER.md +++ b/doc/DOCKER.md @@ -424,18 +424,3 @@ input file times are normalized in both stages so fresh checkouts do not force Cargo to rebuild unchanged source. Changes made by build scripts still reach Cargo's normal validation. The final application copy excludes Cargo's target directory as before. Cache misses only cost compilation time. - -Pull requests that change the Dockerfile, Docker ignore rules, or Runner native -inputs also build the isolated `runner-build` target in `Docker Runner check`. -The check runs `bash scripts/check-docker-runner-cache.sh` against a disposable -copy of tracked source and the actual Docker ignore rules. It compiles a baseline -and exports a local cache, removes that builder, changes a Rust metadata constant, -and rebuilds on a fresh builder using only the exported cache. It requires a -cached dependency build, an unchanged dependency recipe, and changed metadata -from the real binary. It also verifies that a dependency declaration change -alters the recipe. The probe exports small metadata results instead of importing -a large test image into the Docker daemon. Temporary builders and cache files -are removed afterward. It catches missing embedded inputs before the post-merge -build. It uses a GitHub-hosted runner with read-only repository access and never -publishes images or registry caches. Allow up to 20 minutes for its cold build and -source rebuild. diff --git a/scripts/check-docker-runner-cache.sh b/scripts/check-docker-runner-cache.sh deleted file mode 100644 index a4b2e0c17a0..00000000000 --- a/scripts/check-docker-runner-cache.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Build the real Docker target on two fresh builders using an exported cache. -# Export only metadata, avoiding a multi-gigabyte test image in the daemon. -# External access: the baseline build anonymously reads the public BuildKit -# cache at ghcr.io/paperclipai/paperclip:buildcache-{amd64,arm64} (see -# RUNNER_CHECK_SEED_CACHE below). No credentials are used or required, and -# nothing is pushed. -set -euo pipefail -repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -probe_dir="$(mktemp -d "${TMPDIR:-/tmp}/paperclip-runner-cache.XXXXXX")" -baseline_builder="${probe_dir##*/}-baseline" -rebuild_builder="${probe_dir##*/}-rebuild" -cleanup() { - for builder in "$baseline_builder" "$rebuild_builder"; do - docker buildx rm "$builder" >/dev/null 2>&1 || true - done - rm -rf "$probe_dir" -} -trap cleanup EXIT -mkdir "$probe_dir/context" -cd "$repo_root" -git ls-files -z | tar -cf - --null -T - | tar -xf - -C "$probe_dir/context" -cd "$probe_dir/context" -export PROBE_DIR="$probe_dir" -cp Dockerfile "$probe_dir/cache-probe.Dockerfile" -cat >> "$probe_dir/cache-probe.Dockerfile" <<'DOCKER' -FROM runner-build AS cache-proof -RUN ./runner/target/release/paperclip-runnerd --build-metadata > /metadata.json -FROM scratch AS cache-proof-export -COPY --from=cache-proof /metadata.json /metadata.json -COPY --from=runner-plan /tmp/runner-recipe.json /recipe.json -FROM scratch AS recipe-proof-export -COPY --from=runner-plan /tmp/runner-recipe.json /recipe.json -DOCKER -build_proof() { - local result="$1" builder="$2" - shift 2 - docker buildx build --builder "$builder" --file "$probe_dir/cache-probe.Dockerfile" --target cache-proof-export --output "type=local,dest=$probe_dir/$result" --progress plain "$@" . 2>&1 | tee "$probe_dir/$result.log" -} -docker buildx create --name "$baseline_builder" --driver docker-container -# Seed only the baseline with the public BuildKit cache that docker.yml -# refreshes on every master push. The rust stages consume none of that -# build's args, so their layer keys match, and mode=max re-exports the -# imported layers into $probe_dir/cache — the verification build below -# still proves what it always proved from this run's exported cache -# alone, on a fresh builder. Anonymous pull only, nothing is pushed; a -# missing or unreachable ref is a BuildKit warning and the baseline -# degrades to the previous cold compile. Set RUNNER_CHECK_SEED_CACHE to -# another ref, or to the empty string to force the cold path. -if [[ -z "${RUNNER_CHECK_SEED_CACHE+x}" ]]; then - case "$(uname -m)" in - x86_64) RUNNER_CHECK_SEED_CACHE="ghcr.io/paperclipai/paperclip:buildcache-amd64" ;; - aarch64 | arm64) RUNNER_CHECK_SEED_CACHE="ghcr.io/paperclipai/paperclip:buildcache-arm64" ;; - *) RUNNER_CHECK_SEED_CACHE="" ;; - esac -fi -seed_args=() -if [[ -n "$RUNNER_CHECK_SEED_CACHE" ]]; then - seed_args=(--cache-from "type=registry,ref=${RUNNER_CHECK_SEED_CACHE}") -fi -build_proof baseline "$baseline_builder" ${seed_args[@]+"${seed_args[@]}"} --cache-to "type=local,dest=$probe_dir/cache,mode=max" -# Removing the first builder proves the second build cannot use daemon-local -# state, and releases its disk space before importing the exported cache. -docker buildx rm "$baseline_builder" -docker buildx create --name "$rebuild_builder" --driver docker-container -python3 - <<'CHECK' -from pathlib import Path -p=Path('packages/paperclip-runner/runner/crates/runner-core/src/bin/paperclip-runnerd.rs') -s=p.read_text(); needle='paperclip-runner/runnerd-build-metadata/v1' -assert s.count(needle)==1 -p.write_text(s.replace(needle,needle+'-cache-probe')) -CHECK -build_proof source-change "$rebuild_builder" --cache-from "type=local,src=$probe_dir/cache" -python3 - <<'CHECK' -import os,json,re -from pathlib import Path -root=Path(os.environ['PROBE_DIR']) -before=json.loads((root/'baseline/metadata.json').read_text()) -after=json.loads((root/'source-change/metadata.json').read_text()) -assert before['schema']=='paperclip-runner/runnerd-build-metadata/v1' -assert after['schema']==before['schema']+'-cache-probe' -assert (root/'baseline/recipe.json').read_bytes()==(root/'source-change/recipe.json').read_bytes() -log=(root/'source-change.log').read_text() -step=re.search(r'#(\d+) \[runner-deps[^\n]+ RUN cargo chef cook',log)[1] -assert f'#{step} CACHED' in log -assert 'Compiling paperclip-runner-core' in log -print('PASS: fresh builder imported compiled dependencies; real binary changed.') -p=Path('packages/paperclip-runner/runner/Cargo.toml') -s=p.read_text(); assert 'serde_json = "1.0"' in s -p.write_text(s.replace('serde_json = "1.0"','serde_json = ">=1.0.0, <2.0.0"')) -CHECK -docker buildx build --builder "$rebuild_builder" --file "$probe_dir/cache-probe.Dockerfile" --target recipe-proof-export --output "type=local,dest=$probe_dir/manifest-change" --progress plain . -python3 - <<'CHECK' -from pathlib import Path -import os -root=Path(os.environ['PROBE_DIR']) -assert (root/'source-change/recipe.json').read_bytes()!=(root/'manifest-change/recipe.json').read_bytes() -print('PASS: dependency declaration change invalidates the recipe.') -CHECK