PaperClipAI/.github/workflows/docker.yml

name: Docker

on:
  push:
    branches:
      - "master"
    tags:
      - "v*"
      - "nightly/v*"
      - "beta/v*"
  # Release workflows push lane tags with GITHUB_TOKEN, and GitHub suppresses
  # push-triggered runs for those, so release.yml dispatches this workflow at
  # the new tag ref instead. The tag mapping below keys off github.ref either
  # way.
  workflow_dispatch:

# Least privilege: nothing at the workflow level; each job declares exactly
# the token scopes it uses (checkout needs contents:read, GHCR pushes need
# packages:write).
permissions: {}

# Serialise builds per ref without killing an in-flight one: a newer push
# supersedes only the pending slot, so the image build that is already
# running always finishes and publishes. Canary TAG refs each get their
# own group on purpose: their builds run in parallel so every published
# canary gets its sha images regardless of merge cadence. The mutable
# `:canary` channel tags are NOT written by the build matrix (which
# would race across parallel runs) — each canary-tag run retags the
# channel afterwards, only if it still matches the npm `canary`
# dist-tag, so the channel moves monotonically and always mirrors npm.
concurrency:
  group: docker-${{ github.ref }}
  cancel-in-progress: false

jobs:
  # Multi-arch by native runner, not QEMU.
  #
  # This was one job building linux/amd64,linux/arm64 together on an x86
  # runner. The arm64 half is emulated there, and it did not merely run slow:
  # it wedged, every time, in `RUN pnpm --filter @paperclipai/server build`,
  # emitting nothing for 38-45 minutes until `timeout-minutes: 60` killed the
  # job. Verified across three consecutive runs on 2026-09-04; the amd64 half
  # reached `production 5/5` minutes earlier in every one.
  #
  # A timed-out job is reported as *cancelled*, not failed, so the run read
  # "cancelled" and the production image simply stopped publishing without
  # anything going red in an obvious way.
  #
  # It also starved the queue. A run that burns the full hour holds the
  # top-level concurrency slot for that hour, and `cancel-in-progress: false`
  # keeps exactly one pending slot — so with merges arriving faster than one
  # an hour, most runs were superseded before they ever started a job. Five of
  # ten master commits sampled that day never produced an image at all.
  #
  # Each platform now builds on a runner of its own architecture and pushes by
  # digest; `merge` assembles the manifest list. arm64 is kept rather than
  # dropped (the cloud variant below dropped it and is amd64-only) because
  # this is the self-hosted image, and ARM hosts consume it.
  build-and-push:
    strategy:
      # Independent legs: one architecture failing should still publish
      # nothing, but it must not also hide the other's logs behind a
      # cancellation.
      fail-fast: false
      matrix:
        include:
          - platform: linux/amd64
            runner: ubuntu-latest
            arch: amd64
          - platform: linux/arm64
            runner: ubuntu-24.04-arm
            arch: arm64
    runs-on: ${{ matrix.runner }}
    # Native builds land well inside this; it is a backstop, not a budget.
    # (The interim fix while this PR landed raised the single QEMU job's cap
    # to 120 minutes; native per-arch legs make that headroom unnecessary.)
    timeout-minutes: 60
    permissions:
      contents: read
      packages: write
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          # Full history and tags so `git describe` below can compute the
          # release version to stamp into the image.
          fetch-depth: 0

      # `.git` is dockerignored, so a running image cannot derive its own
      # version and otherwise reports the source package.json placeholder in
      # analytics and the debug panel. Compute it here from the pristine
      # checkout (real CalVer drift from the nearest release tag) and pass it
      # into both builds. Empty when no release tag is reachable — the server
      # then keeps its existing fallbacks.
      - name: Compute build version
        id: build-version
        run: |
          set -euo pipefail
          case "${GITHUB_REF}" in
            refs/tags/nightly/v*)
              # Lane tags carry the exact published version; stamp it verbatim
              # instead of describing drift from the nearest stable tag.
              version="${GITHUB_REF#refs/tags/nightly/v}"
              ;;
            refs/tags/beta/v*)
              version="${GITHUB_REF#refs/tags/beta/v}"
              ;;
            *)
              version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
              ;;
          esac
          echo "version=${version}" >> "$GITHUB_OUTPUT"
          echo "Stamping build version: ${version:-<none>}"

      # ISO week stamp for the Dockerfile's tool layer: the layer caches
      # across commits and re-pulls the @latest CLI tools when the week rolls
      # over, instead of on every build.
      - name: Compute tool cache epoch
        id: tools-epoch
        run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"

      - name: Setup pnpm
        uses: pnpm/action-setup@v6
        with:
          version: 9.15.4
          run_install: false

      # No dependency cache here: this workflow publishes release images, and
      # restoring a shared Actions cache into the build inputs would let a
      # poisoned cache entry reach the published artifact.
      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version: 24

      - name: Refresh lockfile for Docker build context
        run: |
          set -euo pipefail
          pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile

          changed="$(git status --porcelain)"
          if [ -z "$changed" ]; then
            echo "Lockfile already matches package metadata."
            exit 0
          fi

          if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
            echo "Unexpected files changed during lockfile refresh:"
            echo "$changed"
            exit 1
          fi

          echo "Using refreshed pnpm-lock.yaml in the Docker build context."

      - name: Free runner disk
        run: |
          set -euo pipefail
          echo "Disk before cleanup:"
          df -h

          pnpm store prune || true
          sudo apt-get clean || true
          sudo rm -rf \
            /usr/share/dotnet \
            /usr/share/swift \
            /usr/local/lib/android \
            /usr/local/share/boost \
            /usr/local/share/powershell \
            /opt/ghc \
            /opt/hostedtoolcache/CodeQL \
            /opt/hostedtoolcache/PyPy \
            /opt/hostedtoolcache/Ruby || true
          docker system prune -af || true

          echo "Disk after cleanup:"
          df -h

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      # Deployment tooling reads these labels from the registry to verify an
      # image's schema expectations against a migrator before deploying it,
      # without pulling the image. The server refuses to start when the
      # database is missing bundled migrations, so orchestrators need a cheap
      # way to check image/migrator compatibility up front.
      - name: Compute schema migration labels
        id: schema
        run: |
          set -euo pipefail
          last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
          count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
          echo "last=${last}" >> "$GITHUB_OUTPUT"
          echo "count=${count}" >> "$GITHUB_OUTPUT"

      # Lane tag mapping: nightly/v* tags publish `:nightly`, and only
      # stable v* tags move `:latest` and the versioned tags.
      # `:sha-<short>` is published on every build. `:canary` is
      # deliberately absent here — the channel tag is moved by the
      # dist-tag-checked retag step below, never by the build matrix,
      # so parallel canary builds cannot race it backwards.
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: ghcr.io/${{ github.repository }}
          tags: |
            type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
            type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=sha
            type=raw,value=sha-${{ github.sha }},enable=${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' }}
          labels: |
            io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
            io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
      - name: Build and push by digest
        id: build
        uses: docker/build-push-action@v7
        with:
          context: .
          # Pin the self-hosted image to the production stage explicitly:
          # the Dockerfile now declares a later `cloud` stage, and without a
          # target the default would silently become that stage.
          target: production
          build-args: |
            PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
            PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
            CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
          platforms: ${{ matrix.platform }}
          # By digest, not by tag: two runners cannot each push the same tag
          # and end up with a manifest list. Each leg publishes an untagged
          # image and `merge` names them together.
          outputs: type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=true
          # Registry-backed BuildKit cache instead of type=gha: the Actions
          # cache is capped at 10GB per repo, and two multi-arch mode=max jobs
          # evict each other, so most builds ran effectively cold. The cache
          # ref lives in ghcr next to the image and is written only by this
          # workflow (docker.yml runs on master/tag pushes, never on PRs).
          #
          # Per-arch refs now the legs are separate runners: a shared ref would
          # have each leg overwrite the other's cache on every build.
          cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }}
          cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=max
          labels: ${{ steps.meta.outputs.labels }}

      # The digest is the only thing `merge` needs from this job. Carried as an
      # empty file named for it, which is the upstream pattern — the name is
      # the payload, so several legs can upload without colliding on content.
      - name: Export digest
        run: |
          set -euo pipefail
          mkdir -p /tmp/digests
          digest="${{ steps.build.outputs.digest }}"
          test -n "$digest"
          touch "/tmp/digests/${digest#sha256:}"

      - name: Upload digest
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: digests-production-${{ matrix.arch }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

  # Names the per-architecture digests as one manifest list under the real
  # tags. Nothing is publicly tagged until this runs, so a half-published
  # multi-arch image is not a state anything can pull.
  merge-and-push:
    needs: build-and-push
    runs-on: ubuntu-latest
    timeout-minutes: 20
    permissions:
      contents: read
      packages: write
      id-token: write
      attestations: write
    steps:
      # Checked out for `packages/db` (schema labels) and the orphan-reaping
      # script the verification step pipes in.
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          fetch-depth: 0

      - name: Download digests
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
        with:
          path: /tmp/digests
          pattern: digests-production-*
          merge-multiple: true

      - name: Login to GitHub Container Registry
        uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4

      # Repeated from the build job rather than passed between them: job
      # outputs would have to survive a matrix, and this is two `ls` calls.
      - name: Compute schema migration labels
        id: schema
        run: |
          set -euo pipefail
          last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
          count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
          echo "last=${last}" >> "$GITHUB_OUTPUT"
          echo "count=${count}" >> "$GITHUB_OUTPUT"

      # Same lane mapping as the build job; this is where it is actually
      # applied, since the legs push untagged. `:canary` is deliberately
      # absent, exactly as in the build job's mapping: the channel tag is
      # moved only by the dist-tag-checked promote_canary_channel job below,
      # so parallel canary-tag builds can never race the channel backwards.
      - name: Docker meta
        id: meta
        uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
        with:
          images: ghcr.io/${{ github.repository }}
          tags: |
            type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
            type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
            type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
            type=sha
            type=raw,value=sha-${{ github.sha }},enable=${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' }}
          labels: |
            io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
            io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}

      - name: Create manifest list and push
        working-directory: /tmp/digests
        run: |
          set -euo pipefail
          docker buildx imagetools create \
            $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
            $(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)

      # PID 1 must be an init that reaps adopted orphans. With node there, the
      # orphans agent runs leave behind are never wait()ed and pin as zombies
      # until the cgroup pid limit is exhausted and every fork() in the
      # container fails. Run against the pushed manifest rather than a local
      # build: the legs push by digest, so nothing is loaded into this
      # runner's daemon. The independent cloud workflow checks its own image.
      - name: Verify PID 1 reaps orphaned processes
        env:
          # Through the environment, not interpolated into the script body, so
          # the tag text is data rather than shell.
          IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
        run: |
          set -euo pipefail
          image="$(printf '%s\n' "$IMAGE_TAGS" | head -n 1)"
          test -n "$image"
          echo "Verifying orphan reaping in $image"
          docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh

      # This generic contract is useful to any downstream image composer. Only
      # canonical master pushes own the full-SHA tag and its signed provenance.
      # Capture exact registry bytes; both the smoke and attestation use their
      # digest, so later tag movement cannot change what was verified.
      - name: Resolve standard image digest
        id: standard
        if: github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.event_name == 'push' && github.ref == 'refs/heads/master'
        run: node scripts/standard-image-contract.mjs --resolve "$GITHUB_SHA"
      - name: Verify the immutable standard image
        if: steps.standard.outputs.digest != ''
        env:
          IMAGE_DIGEST: ${{ steps.standard.outputs.digest }}
        run: docker run --rm -i --pull always "ghcr.io/paperclipai/paperclip@$IMAGE_DIGEST" sh -s < scripts/assert-orphan-reaping.sh
      - name: Attest the standard production image
        if: steps.standard.outputs.digest != ''
        uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4
        with:
          subject-name: ghcr.io/paperclipai/paperclip
          subject-digest: ${{ steps.standard.outputs.digest }}
          push-to-registry: true

  # Master cloud builds start independently in docker-cloud.yml. Tag builds
  # and manual Docker dispatches call the same implementation, preserving the
  # release tags and the canary promotion dependency below.
  build-and-push-cloud:
    if: github.event_name != 'push' || github.ref != 'refs/heads/master'
    uses: ./.github/workflows/docker-cloud.yml
    permissions:
      contents: read
      packages: write

  # Moves the mutable `:canary` / `:canary-cloud` channel tags. Kept OUT
  # of the build jobs and serialized in its own lane, and — the load-
  # bearing property — CONVERGENT rather than self-interested: a
  # promotion does not promote "its own" canary, it retags the channel
  # to whatever the npm `canary` dist-tag names at execution time,
  # provided that version's sha images are published. GitHub's shared
  # concurrency lane keeps one running and one pending promotion and
  # REPLACES the pending slot with the latest enqueued — an older build
  # finishing late can therefore evict the newest canary's pending
  # promotion. With convergent promotion that eviction is harmless:
  # whichever promotion survives resolves the current dist-tag fresh
  # and lands the channel there (the current canary's images always
  # exist by the time any later promotion runs, because per-tag build
  # groups mean canary builds are never superseded and each run's
  # promotion is gated on its own completed pushes). Every interleaving
  # converges the Docker channel onto the npm channel.
  promote_canary_channel:
    if: startsWith(github.ref, 'refs/tags/canary/v')
    # merge-and-push, not build-and-push: the per-arch legs push untagged
    # digests, and the production `sha-*` tags this promotion retags only
    # exist once the manifest merge has named them.
    needs: [merge-and-push, build-and-push-cloud]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    permissions:
      contents: read
      packages: write
    concurrency:
      group: docker-canary-channel-promotion
      cancel-in-progress: false
    steps:
      - name: Login to GitHub Container Registry
        uses: docker/login-action@v4
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Converge the channel tags onto the current npm canary
        env:
          IMAGE: ghcr.io/${{ github.repository }}
          GH_TOKEN: ${{ github.token }}
        run: |
          current="$(curl -fsS "https://registry.npmjs.org/-/package/@paperclipai%2Fdb/dist-tags" | jq -er .canary)"
          sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/$(printf 'canary/v%s' "$current" | jq -sRr @uri)" --jq .sha 2>/dev/null || true)"
          if [ -z "$sha" ]; then
            echo "canary/v${current} does not resolve yet; a later promotion converges the channel"
            exit 0
          fi
          short="$(printf '%s' "$sha" | cut -c1-7)"
          if ! docker buildx imagetools inspect "$IMAGE:sha-${short}-cloud" >/dev/null 2>&1; then
            echo "images for ${current} (sha-${short}) not published yet; its own promotion converges the channel"
            exit 0
          fi
          docker buildx imagetools create -t "$IMAGE:canary" "$IMAGE:sha-${short}"
          docker buildx imagetools create -t "$IMAGE:canary-cloud" "$IMAGE:sha-${short}-cloud"
          echo "channel tags moved to canary ${current} (sha-${short})"