Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-10-28 12:04:41 +01:00
Code Issues Packages Projects Releases Wiki Activity

Reconcile work-folder foundation with current master

Browse Source 
Keep current task continuation, chat UI, provider security, and multi-architecture build behavior. Regenerate the work-folder schema after the published migration sequence and make it replay-safe for preview databases. Verify preserved file/trash/checkpoint references and company constraints on PostgreSQL.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-11 14:12:19 -05:00
2 parents 41f7b05157 847d00bdc3
commit f03b8e7097
1325 files changed
+1488090 -52241
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/cloud-artifacts.yml
+34
View File
@@ -0,0 +1,34 @@
name: Cloud artifacts
on:
push:
branches: [master]
workflow_dispatch:
permissions: {}
jobs:
dispatch_migrator:
name: Start exact-source cloud migrator publication
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
actions: write
steps:
# This separate workflow starts at merge, outside the full npm release's
# concurrency group. Publication stays in release.yml so npm recognizes
# the established trusted-publisher identity and npm-canary environment.
# No source checkout or package code runs with the dispatch credential.
- name: Dispatch the migrator-only release
env:
GH_TOKEN: ${{ github.token }}
SOURCE_SHA: ${{ github.sha }}
run: |
set -euo pipefail
request_id="$(cat /proc/sys/kernel/random/uuid)"
gh workflow run release.yml --repo "$GITHUB_REPOSITORY" --ref master \
--field channel=cloud-migrator \
--field source_ref="$SOURCE_SHA" \
--field request_id="$request_id"
echo "Started Cloud migrator $SOURCE_SHA in release.yml (request $request_id)." >> "$GITHUB_STEP_SUMMARY"
.github/workflows/cloud-readiness.yml
+93
View File
@@ -0,0 +1,93 @@
name: Cloud readiness
run-name: Cloud readiness ${{ github.sha }}
on:
push:
branches: [master]
workflow_dispatch:
permissions: {}
# Source verification must start outside the full npm release's queue.
concurrency:
group: cloud-readiness-${{ github.sha }}
cancel-in-progress: false
jobs:
image:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
permissions:
contents: read
packages: write
uses: ./.github/workflows/docker-cloud.yml
verify:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
permissions:
contents: read
uses: ./.github/workflows/release-verify.yml
with:
ref: ${{ github.sha }}
artifacts:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
name: Wait for exact-source cloud artifacts
runs-on: ubuntu-latest
timeout-minutes: 35
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Wait for verified image and exact-source migrator
env:
SOURCE_SHA: ${{ github.sha }}
run: node scripts/cloud-readiness.mjs "$SOURCE_SHA"
source_verified:
# npm canary publication reuses this exact-source verification proof.
# Keep it independent of image/migrator availability, and fail closed when
# any source check fails, is cancelled, or is skipped.
name: Cloud source verified v1
needs: [verify]
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Check the source verification consumer
run: node --test scripts/cloud-source-verification.test.mjs
- name: Record source verification
env:
SOURCE_SHA: ${{ github.sha }}
run: |
echo "Cloud source verified v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
ready:
# Versioned consumer contract. Never add always() or continue-on-error:
# failed, cancelled, or skipped prerequisites must not report readiness.
name: Cloud deployable v1
needs: [verify, image, artifacts]
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Record cloud readiness
env:
SOURCE_SHA: ${{ github.sha }}
run: |
echo "Cloud deployable v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
echo "Source verification passed; the full-SHA image and exact-source migrator are available." >> "$GITHUB_STEP_SUMMARY"
echo "Deployment tooling must still resolve and pin the image and migrator and validate migration compatibility." >> "$GITHUB_STEP_SUMMARY"
.github/workflows/docker-cloud.yml
+300
View File
@@ -0,0 +1,300 @@
name: Docker cloud
on:
workflow_dispatch:
workflow_call:
permissions: {}
# Independent SHAs can build immediately on separate runners.
# Repeated requests for the same source serialize without cancelling a build.
# No mutable canary channel is promoted here; docker.yml owns that operation.
concurrency:
group: docker-cloud-${{ github.sha }}
cancel-in-progress: false
jobs:
build-and-push-cloud:
# Only canonical master builds can consume the release Fleet. The runner
# group must also allow this workflow only at refs/heads/master.
# Keep an operator switch for a full-run retry on GitHub-hosted runners.
runs-on: ${{ vars.AWS_CLOUD_BUILDS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-cloud-build-x64/env=public-ci' || 'ubuntu-latest' }}
# Fleet instances expire after 45 minutes, including bootstrap and cleanup.
timeout-minutes: ${{ vars.AWS_CLOUD_BUILDS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 40 || 60 }}
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history and tags so `git describe` below can compute the
# release version to stamp into the image.
fetch-depth: 0
# `.git` is dockerignored, so a running image cannot derive its own
# version and otherwise reports the source package.json placeholder in
# analytics and the debug panel. Compute it here from the pristine
# checkout (real CalVer drift from the nearest release tag) and pass it
# into the build. Empty when no release tag is reachable — the server
# then keeps its existing fallbacks.
- name: Compute build version
id: build-version
run: |
set -euo pipefail
case "${GITHUB_REF}" in
refs/tags/nightly/v*)
# Lane tags carry the exact published version; stamp it verbatim
# instead of describing drift from the nearest stable tag.
version="${GITHUB_REF#refs/tags/nightly/v}"
;;
refs/tags/beta/v*)
version="${GITHUB_REF#refs/tags/beta/v}"
;;
*)
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
;;
esac
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "Stamping build version: ${version:-<none>}"
# ISO week stamp for the Dockerfile's tool layer: the layer caches
# across commits and re-pulls the @latest CLI tools when the week rolls
# over, instead of on every build.
- name: Compute tool cache epoch
id: tools-epoch
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
# Each SHA exports its own cache. Import recent first-parent caches so
# a late older build cannot overwrite a newer build's cache manifest.
# The legacy ref keeps the first builds warm during the transition.
- name: Select cloud cache ancestry
id: cloud-cache
env:
CACHE_IMAGE: ghcr.io/${{ github.repository }}
run: |
set -euo pipefail
{
echo 'sources<<CACHE_SOURCES'
for commit in $(git rev-list --first-parent --max-count=10 HEAD); do
echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud-$commit"
done
echo "type=registry,ref=$CACHE_IMAGE:buildcache-cloud"
echo 'CACHE_SOURCES'
} >> "$GITHUB_OUTPUT"
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
# No dependency cache here: this workflow publishes release images, and
# restoring a shared Actions cache into the build inputs would let a
# poisoned cache entry reach the published artifact.
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Refresh lockfile for Docker build context
run: |
set -euo pipefail
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
changed="$(git status --porcelain)"
if [ -z "$changed" ]; then
echo "Lockfile already matches package metadata."
exit 0
fi
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
echo "Unexpected files changed during lockfile refresh:"
echo "$changed"
exit 1
fi
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
- name: Free runner disk
run: |
set -euo pipefail
echo "Disk before cleanup:"
df -h
# A measured hosted cloud build started with 86 GB available.
# Keep ample headroom for BuildKit and image verification, but
# avoid minutes deleting SDKs when neither filesystem needs space.
minimum_free_kib=$((64 * 1024 * 1024))
if docker_root="$(docker info --format '{{.DockerRootDir}}')" \
&& available_kib="$(df -Pk "$docker_root" "$GITHUB_WORKSPACE" | awk 'NR > 1 { rows++; if ($4 !~ /^[0-9]+$/) invalid = 1; if (min == "" || $4 < min) min = $4 } END { if (invalid || rows != 2) exit 1; print min }')" \
&& [[ "$available_kib" =~ ^[0-9]+$ ]] \
&& (( available_kib >= minimum_free_kib )); then
echo "At least 64 GiB is available for Docker and the workspace; skipping cleanup."
exit 0
fi
pnpm store prune || true
sudo apt-get clean || true
sudo rm -rf \
/usr/share/dotnet \
/usr/share/swift \
/usr/local/lib/android \
/usr/local/share/boost \
/usr/local/share/powershell \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/opt/hostedtoolcache/PyPy \
/opt/hostedtoolcache/Ruby || true
docker system prune -af || true
echo "Disk after cleanup:"
df -h
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Deployment tooling reads these labels from the registry to verify an
# image's schema expectations against a migrator before deploying it,
# without pulling the image. The server refuses to start when the
# database is missing bundled migrations, so orchestrators need a cheap
# way to check image/migrator compatibility up front.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Published under the same lane tag set as the self-hosted image, with a
# `-cloud` suffix (nightly-cloud, latest-cloud, <version>-cloud,
# sha-<short>-cloud). `:canary-cloud` follows the same retag-step
# ownership rule as `:canary` above.
- name: Docker meta (cloud)
id: meta-cloud
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
with:
images: ghcr.io/${{ github.repository }}
flavor: |
suffix=-cloud,onlatest=true
tags: |
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push (cloud)
id: build-cloud
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
target: cloud
# Space-separated sandbox-provider directory names to build into
# the variant; add here when managed deployments need another.
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
# variant installs from server/package.json's declared version;
# add another name there when a managed tenant needs it.
build-args: |
USER_UID=1001
USER_GID=1001
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
# amd64 only, unlike the self-hosted image above: the cloud variant
# is consumed exclusively by managed-deployment hosts, which run
# amd64. The QEMU-emulated arm64 half dominated this job's wall
# clock, and dropping it roughly halves time-to-deployable-image.
platforms: linux/amd64
push: true
# Same-SHA builds serialize above; different SHAs never share a
# writable cache ref. Registry layers are content-addressed and
# shared even when cache manifests have separate tags.
cache-from: ${{ steps.cloud-cache.outputs.sources }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
tags: ${{ steps.meta-cloud.outputs.tags }}
labels: ${{ steps.meta-cloud.outputs.labels }}
# The cloud target installs @sentry/node at the version
# server/package.json declares, into a directory the server's own
# module resolution walks. Verify the image this job just pushed, not
# a local build, so a build-cache or layer-ordering regression is
# caught before any tenant runs the image.
- name: Verify the pushed image resolves the declared Sentry version
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: |
set -euo pipefail
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
test -n "$expected"
installed="$(docker run --rm --pull always \
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
--entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)"
echo "Declared optional peer version: $expected"
echo "Installed in the pushed image: $installed"
if [ "$installed" != "$expected" ]; then
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
exit 1
fi
echo "The pushed image resolves the declared @sentry/node version."
# Managed hosts run node as 1001:1001. Bake that identity into the image
# so usermod does not walk the mounted home on every container start.
# Check before the entrypoint can repair a wrongly built identity.
- name: Verify cloud runtime user
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: |
set -euo pipefail
docker run --rm --entrypoint sh "$IMAGE" -ec '
test "$(id -u node)" = 1001
test "$(id -g node)" = 1001
test "$USER_UID" = 1001
test "$USER_GID" = 1001
'
docker run --rm -e USER_UID=1001 -e USER_GID=1001 "$IMAGE" sh -ec '
test "$(id -u)" = 1001
test "$(id -g)" = 1001
test -w "$PAPERCLIP_HOME"
'
# Verify the independently published cloud image without waiting for
# the self-hosted manifest job. The Sentry check already pulled it.
- name: Verify cloud PID 1 reaps orphaned processes
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: docker run --rm -i "$IMAGE" sh -s < scripts/assert-orphan-reaping.sh
# Cloud's commit resolver and preview-artifact planner use the full SHA.
# Publish that address only after checking this build's exact digest.
# Retagging reuses the registry manifest and does not rebuild the image.
- name: Publish verified full-SHA cloud tag
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud
run: |
set -euo pipefail
revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')"
test "$revision" = "$GITHUB_SHA"
test "$platform" = linux/amd64
docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE"
.github/workflows/docker-runner-check.yml
+37
View File
@@ -0,0 +1,37 @@
name: Docker Runner check
on:
pull_request:
paths:
- .github/workflows/docker-runner-check.yml
- Dockerfile
- .dockerignore
- packages/paperclip-runner/rust-toolchain.toml
- packages/paperclip-runner/runner/**
- packages/paperclip-runner/protocol/**
permissions: {}
concurrency:
group: docker-runner-check-${{ github.event.pull_request.number }}
cancel-in-progress: true
jobs:
runner:
name: Compile isolated native Runner
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Compile the real target with the real .dockerignore. This catches new
# Cargo or embedded protocol inputs that the isolated COPY set omits.
# No registry credentials, cache imports/exports, or image publication.
- name: Compile the Runner from its isolated Docker context
run: docker buildx build --target runner-build --progress plain .
.github/workflows/docker.yml
+223 -217
View File
@@ -19,13 +19,20 @@ on:
type: string
default: ""
permissions:
contents: read
packages: write
# Least privilege: nothing at the workflow level; each job declares exactly
# the token scopes it uses (checkout needs contents:read, GHCR pushes need
# packages:write).
permissions: {}
# Serialise builds per ref without killing an in-flight one: a newer push
# supersedes only the pending slot, so the image build that is already
# running always finishes and publishes.
# running always finishes and publishes. Canary TAG refs each get their
# own group on purpose: their builds run in parallel so every published
# canary gets its sha images regardless of merge cadence. The mutable
# `:canary` channel tags are NOT written by the build matrix (which
# would race across parallel runs) — each canary-tag run retags the
# channel afterwards, only if it still matches the npm `canary`
# dist-tag, so the channel moves monotonically and always mirrors npm.
concurrency:
group: docker-${{ github.ref }}
cancel-in-progress: false
@@ -64,9 +71,51 @@ jobs:
if-no-files-found: error
retention-days: 30
# Multi-arch by native runner, not QEMU.
#
# This was one job building linux/amd64,linux/arm64 together on an x86
# runner. The arm64 half is emulated there, and it did not merely run slow:
# it wedged, every time, in `RUN pnpm --filter @paperclipai/server build`,
# emitting nothing for 38-45 minutes until `timeout-minutes: 60` killed the
# job. Verified across three consecutive runs on 2026-09-04; the amd64 half
# reached `production 5/5` minutes earlier in every one.
#
# A timed-out job is reported as *cancelled*, not failed, so the run read
# "cancelled" and the production image simply stopped publishing without
# anything going red in an obvious way.
#
# It also starved the queue. A run that burns the full hour holds the
# top-level concurrency slot for that hour, and `cancel-in-progress: false`
# keeps exactly one pending slot — so with merges arriving faster than one
# an hour, most runs were superseded before they ever started a job. Five of
# ten master commits sampled that day never produced an image at all.
#
# Each platform now builds on a runner of its own architecture and pushes by
# digest; `merge` assembles the manifest list. arm64 is kept rather than
# dropped (the cloud variant below dropped it and is amd64-only) because
# this is the self-hosted image, and ARM hosts consume it.
build-and-push:
runs-on: ubuntu-latest
strategy:
# Independent legs: one architecture failing should still publish
# nothing, but it must not also hide the other's logs behind a
# cancellation.
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-latest
arch: amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
arch: arm64
runs-on: ${{ matrix.runner }}
# Native builds land well inside this; it is a backstop, not a budget.
# (The interim fix while this PR landed raised the single QEMU job's cap
# to 120 minutes; native per-arch legs make that headroom unnecessary.)
timeout-minutes: 60
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@v7
@@ -188,16 +237,18 @@ jobs:
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Lane tag mapping: master pushes publish `:canary`, nightly/v* tags
# publish `:nightly`, and only stable v* tags move `:latest` and the
# versioned tags. `:sha-<short>` is published on every build.
# Lane tag mapping: nightly/v* tags publish `:nightly`, and only
# stable v* tags move `:latest` and the versioned tags.
# `:sha-<short>` is published on every build. `:canary` is
# deliberately absent here — the channel tag is moved by the
# dist-tag-checked retag step below, never by the build matrix,
# so parallel canary builds cannot race it backwards.
- name: Docker meta
id: meta
uses: docker/metadata-action@v6
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=raw,value=canary,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
@@ -207,8 +258,8 @@ jobs:
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push
- name: Build and push by digest
id: build
uses: docker/build-push-action@v7
with:
context: .
@@ -220,25 +271,123 @@ jobs:
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
platforms: linux/amd64,linux/arm64
push: true
platforms: ${{ matrix.platform }}
# By digest, not by tag: two runners cannot each push the same tag
# and end up with a manifest list. Each leg publishes an untagged
# image and `merge` names them together.
outputs: type=image,name=ghcr.io/${{ github.repository }},push-by-digest=true,name-canonical=true,push=true
# Registry-backed BuildKit cache instead of type=gha: the Actions
# cache is capped at 10GB per repo, and two multi-arch mode=max jobs
# evict each other, so most builds ran effectively cold. The cache
# ref lives in ghcr next to the image and is written only by this
# workflow (docker.yml runs on master/tag pushes, never on PRs).
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache,mode=max
tags: ${{ steps.meta.outputs.tags }}
#
# Per-arch refs now the legs are separate runners: a shared ref would
# have each leg overwrite the other's cache on every build.
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-${{ matrix.arch }},mode=max
labels: ${{ steps.meta.outputs.labels }}
# The digest is the only thing `merge` needs from this job. Carried as an
# empty file named for it, which is the upstream pattern — the name is
# the payload, so several legs can upload without colliding on content.
- name: Export digest
run: |
set -euo pipefail
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
test -n "$digest"
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: digests-production-${{ matrix.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
# Names the per-architecture digests as one manifest list under the real
# tags. Nothing is publicly tagged until this runs, so a half-published
# multi-arch image is not a state anything can pull.
merge-and-push:
needs: build-and-push
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
packages: write
steps:
# Checked out for `packages/db` (schema labels) and the orphan-reaping
# script the verification step pipes in.
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0
- name: Download digests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
path: /tmp/digests
pattern: digests-production-*
merge-multiple: true
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Repeated from the build job rather than passed between them: job
# outputs would have to survive a matrix, and this is two `ls` calls.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Same lane mapping as the build job; this is where it is actually
# applied, since the legs push untagged. `:canary` is deliberately
# absent, exactly as in the build job's mapping: the channel tag is
# moved only by the dist-tag-checked promote_canary_channel job below,
# so parallel canary-tag builds can never race the channel backwards.
- name: Docker meta
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
with:
images: ghcr.io/${{ github.repository }}
tags: |
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Create manifest list and push
working-directory: /tmp/digests
run: |
set -euo pipefail
docker buildx imagetools create \
$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf 'ghcr.io/${{ github.repository }}@sha256:%s ' *)
# PID 1 must be an init that reaps adopted orphans. With node there, the
# orphans agent runs leave behind are never wait()ed and pin as zombies
# until the cgroup pid limit is exhausted and every fork() in the
# container fails. Run against the pushed image rather than a local
# build: the step above is multi-arch with `push: true`, so nothing is
# loaded into the runner's daemon. The cloud variant is FROM production
# and inherits the same ENTRYPOINT, so checking this image covers both.
# container fails. Run against the pushed manifest rather than a local
# build: the legs push by digest, so nothing is loaded into this
# runner's daemon. The independent cloud workflow checks its own image.
- name: Verify PID 1 reaps orphaned processes
env:
# Through the environment, not interpolated into the script body, so
@@ -251,113 +400,47 @@ jobs:
echo "Verifying orphan reaping in $image"
docker run --rm -i --pull always "$image" sh -s < scripts/assert-orphan-reaping.sh
# The cloud variant carries built bundled plugins for managed deployments
# (see the `cloud` stage in the Dockerfile). It runs as its own job with no
# `needs:` on the stock publish above, so the two builds run in parallel and
# a failure or slow build in one never gates, delays, or skips the other.
# Both jobs share only the single top-level concurrency slot. Each job is a
# separate runner, so this one carries its own copy of the prep steps
# (checkout through schema labels) — the accepted cost of that isolation.
# Master cloud builds start independently in docker-cloud.yml. Tag builds
# and manual Docker dispatches call the same implementation, preserving the
# release tags and the canary promotion dependency below.
build-and-push-cloud:
if: github.event_name != 'push' || github.ref != 'refs/heads/master'
uses: ./.github/workflows/docker-cloud.yml
permissions:
contents: read
packages: write
# Moves the mutable `:canary` / `:canary-cloud` channel tags. Kept OUT
# of the build jobs and serialized in its own lane, and — the load-
# bearing property — CONVERGENT rather than self-interested: a
# promotion does not promote "its own" canary, it retags the channel
# to whatever the npm `canary` dist-tag names at execution time,
# provided that version's sha images are published. GitHub's shared
# concurrency lane keeps one running and one pending promotion and
# REPLACES the pending slot with the latest enqueued — an older build
# finishing late can therefore evict the newest canary's pending
# promotion. With convergent promotion that eviction is harmless:
# whichever promotion survives resolves the current dist-tag fresh
# and lands the channel there (the current canary's images always
# exist by the time any later promotion runs, because per-tag build
# groups mean canary builds are never superseded and each run's
# promotion is gated on its own completed pushes). Every interleaving
# converges the Docker channel onto the npm channel.
promote_canary_channel:
if: startsWith(github.ref, 'refs/tags/canary/v')
# merge-and-push, not build-and-push: the per-arch legs push untagged
# digests, and the production `sha-*` tags this promotion retags only
# exist once the manifest merge has named them.
needs: [merge-and-push, build-and-push-cloud]
runs-on: ubuntu-latest
timeout-minutes: 60
timeout-minutes: 10
permissions:
contents: read
packages: write
concurrency:
group: docker-canary-channel-promotion
cancel-in-progress: false
steps:
- name: Checkout
uses: actions/checkout@v7
with:
# Full history and tags so `git describe` below can compute the
# release version to stamp into the image.
fetch-depth: 0
# `.git` is dockerignored, so a running image cannot derive its own
# version and otherwise reports the source package.json placeholder in
# analytics and the debug panel. Compute it here from the pristine
# checkout (real CalVer drift from the nearest release tag) and pass it
# into the build. Empty when no release tag is reachable — the server
# then keeps its existing fallbacks.
- name: Compute build version
id: build-version
run: |
set -euo pipefail
case "${GITHUB_REF}" in
refs/tags/nightly/v*)
# Lane tags carry the exact published version; stamp it verbatim
# instead of describing drift from the nearest stable tag.
version="${GITHUB_REF#refs/tags/nightly/v}"
;;
refs/tags/beta/v*)
version="${GITHUB_REF#refs/tags/beta/v}"
;;
*)
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
;;
esac
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "Stamping build version: ${version:-<none>}"
# ISO week stamp for the Dockerfile's tool layer: the layer caches
# across commits and re-pulls the @latest CLI tools when the week rolls
# over, instead of on every build.
- name: Compute tool cache epoch
id: tools-epoch
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
- name: Setup pnpm
uses: pnpm/action-setup@v6
with:
version: 9.15.4
run_install: false
# No dependency cache here: this workflow publishes release images, and
# restoring a shared Actions cache into the build inputs would let a
# poisoned cache entry reach the published artifact.
- name: Setup Node.js
uses: actions/setup-node@v7
with:
node-version: 24
- name: Refresh lockfile for Docker build context
run: |
set -euo pipefail
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
changed="$(git status --porcelain)"
if [ -z "$changed" ]; then
echo "Lockfile already matches package metadata."
exit 0
fi
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
echo "Unexpected files changed during lockfile refresh:"
echo "$changed"
exit 1
fi
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
- name: Free runner disk
run: |
set -euo pipefail
echo "Disk before cleanup:"
df -h
pnpm store prune || true
sudo apt-get clean || true
sudo rm -rf \
/usr/share/dotnet \
/usr/share/swift \
/usr/local/lib/android \
/usr/local/share/boost \
/usr/local/share/powershell \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/opt/hostedtoolcache/PyPy \
/opt/hostedtoolcache/Ruby || true
docker system prune -af || true
echo "Disk after cleanup:"
df -h
- name: Login to GitHub Container Registry
uses: docker/login-action@v4
with:
@@ -365,99 +448,22 @@ jobs:
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v4
# Deployment tooling reads these labels from the registry to verify an
# image's schema expectations against a migrator before deploying it,
# without pulling the image. The server refuses to start when the
# database is missing bundled migrations, so orchestrators need a cheap
# way to check image/migrator compatibility up front.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Published under the same lane tag set as the self-hosted image, with a
# `-cloud` suffix (canary-cloud, nightly-cloud, latest-cloud,
# <version>-cloud, sha-<short>-cloud).
- name: Docker meta (cloud)
id: meta-cloud
uses: docker/metadata-action@v6
with:
images: ghcr.io/${{ github.repository }}
flavor: |
suffix=-cloud,onlatest=true
tags: |
type=raw,value=canary,enable=${{ github.ref == 'refs/heads/master' }}
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push (cloud)
uses: docker/build-push-action@v7
with:
context: .
target: cloud
# Space-separated sandbox-provider directory names to build into
# the variant; add here when managed deployments need another.
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
# variant installs from server/package.json's declared version;
# add another name there when a managed tenant needs it.
build-args: |
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
# amd64 only, unlike the self-hosted image above: the cloud variant
# is consumed exclusively by managed-deployment hosts, which run
# amd64. The QEMU-emulated arm64 half dominated this job's wall
# clock, and dropping it roughly halves time-to-deployable-image.
platforms: linux/amd64
push: true
# Registry-backed BuildKit cache, separate ref from the self-hosted
# job so the two parallel builds never clobber each other's cache
# manifest (see the rationale on the job above).
cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud,mode=max
tags: ${{ steps.meta-cloud.outputs.tags }}
labels: ${{ steps.meta-cloud.outputs.labels }}
# The cloud target installs @sentry/node at the version
# server/package.json declares, into a directory the server's own
# module resolution walks. Verify the image this job just pushed, not
# a local build, so a build-cache or layer-ordering regression is
# caught before any tenant runs the image.
- name: Verify the pushed image resolves the declared Sentry version
- name: Converge the channel tags onto the current npm canary
env:
IMAGE_TAGS: ${{ steps.meta-cloud.outputs.tags }}
IMAGE: ghcr.io/${{ github.repository }}
GH_TOKEN: ${{ github.token }}
run: |
set -euo pipefail
image="$(printf '%s\n' "$IMAGE_TAGS" | head -n 1)"
test -n "$image"
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
test -n "$expected"
installed="$(docker run --rm --pull always \
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
--entrypoint node "$image" /app/server/.ci-sentry-probe.mjs)"
echo "Declared optional peer version: $expected"
echo "Installed in the pushed image: $installed"
if [ "$installed" != "$expected" ]; then
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
exit 1
current="$(curl -fsS "https://registry.npmjs.org/-/package/@paperclipai%2Fdb/dist-tags" | jq -er .canary)"
sha="$(gh api "repos/${GITHUB_REPOSITORY}/commits/$(printf 'canary/v%s' "$current" | jq -sRr @uri)" --jq .sha 2>/dev/null || true)"
if [ -z "$sha" ]; then
echo "canary/v${current} does not resolve yet; a later promotion converges the channel"
exit 0
fi
echo "The pushed image resolves the declared @sentry/node version."
short="$(printf '%s' "$sha" | cut -c1-7)"
if ! docker buildx imagetools inspect "$IMAGE:sha-${short}-cloud" >/dev/null 2>&1; then
echo "images for ${current} (sha-${short}) not published yet; its own promotion converges the channel"
exit 0
fi
docker buildx imagetools create -t "$IMAGE:canary" "$IMAGE:sha-${short}"
docker buildx imagetools create -t "$IMAGE:canary-cloud" "$IMAGE:sha-${short}-cloud"
echo "channel tags moved to canary ${current} (sha-${short})"
.github/workflows/pr-trusted.yml
+50 -4
View File
@@ -335,7 +335,7 @@ jobs:
run: node --test ./scripts/__tests__/e2e-shard.test.mjs
- name: Test release verify workflow wiring
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs
run: node --test ./scripts/__tests__/release-verify-workflow.test.mjs ./scripts/cloud-source-verification.test.mjs
- name: Test standalone package build concurrency
run: node --test ./scripts/__tests__/build-standalone-concurrency.test.mjs
@@ -553,7 +553,7 @@ jobs:
# Preserve the legacy required-check name while the underlying work runs in parallel.
name: verify
if: ${{ always() }}
needs: [gate, policy, typecheck_release_registry, general_tests, build, docker_context_integrity]
needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity]
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 5
@@ -564,6 +564,7 @@ jobs:
POLICY_RESULT: ${{ needs.policy.result }}
TYPECHECK_RELEASE_REGISTRY_RESULT: ${{ needs.typecheck_release_registry.result }}
GENERAL_TESTS_RESULT: ${{ needs.general_tests.result }}
RUNNER_VERIFICATION_RESULT: ${{ needs.verify_paperclip_runner.result }}
BUILD_RESULT: ${{ needs.build.result }}
DOCKER_CONTEXT_INTEGRITY_RESULT: ${{ needs.docker_context_integrity.result }}
run: |
@@ -572,12 +573,14 @@ jobs:
true)
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "success"
test "$GENERAL_TESTS_RESULT" = "success"
test "$RUNNER_VERIFICATION_RESULT" = "success"
test "$BUILD_RESULT" = "success"
test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "success"
;;
false)
test "$TYPECHECK_RELEASE_REGISTRY_RESULT" = "skipped"
test "$GENERAL_TESTS_RESULT" = "skipped"
test "$RUNNER_VERIFICATION_RESULT" = "skipped"
test "$BUILD_RESULT" = "skipped"
test "$DOCKER_CONTEXT_INTEGRITY_RESULT" = "skipped"
;;
@@ -587,8 +590,8 @@ jobs:
;;
esac
build:
name: Build
verify_paperclip_runner:
name: Verify Paperclip Runner
needs: [gate, policy]
if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
@@ -633,6 +636,49 @@ jobs:
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
build:
name: Build
needs: [gate, policy]
if: ${{ needs.gate.outputs.full_ci == 'true' }}
runs-on: ${{ needs.gate.outputs.runner }}
timeout-minutes: 20
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Setup Node.js for pnpm bootstrap
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
env:
NPM_CONFIG_AUDIT: "false"
NPM_CONFIG_FUND: "false"
NPM_CONFIG_UPDATE_NOTIFIER: "false"
with:
version: 9.15.4
- name: Restore regenerated PR lockfile (if policy uploaded one)
if: needs.policy.outputs.lockfile_regenerated == '1'
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: pr-lockfile
path: .
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
cache: pnpm
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Build Runner Evalbook viewer
run: pnpm --filter @paperclipai/paperclip-runner build:issue-thread
.github/workflows/release-verify.yml
+102 -8
View File
@@ -58,16 +58,59 @@ jobs:
fail-fast: false
matrix:
include:
- group: general-server
group_label: server (1/3)
# Split the long chat file by collected test locations, and balance
# the remaining server files across ten runners. Normal PR/local
# invocations retain their complete general-server group.
- group: general-server-without-chat
group_label: server (1/10)
shard_index: 0
shard_count: 10
- group: general-server-without-chat
group_label: server (2/10)
shard_index: 1
shard_count: 10
- group: general-server-without-chat
group_label: server (3/10)
shard_index: 2
shard_count: 10
- group: general-server-without-chat
group_label: server (4/10)
shard_index: 3
shard_count: 10
- group: general-server-without-chat
group_label: server (5/10)
shard_index: 4
shard_count: 10
- group: general-server-without-chat
group_label: server (6/10)
shard_index: 5
shard_count: 10
- group: general-server-without-chat
group_label: server (7/10)
shard_index: 6
shard_count: 10
- group: general-server-without-chat
group_label: server (8/10)
shard_index: 7
shard_count: 10
- group: general-server-without-chat
group_label: server (9/10)
shard_index: 8
shard_count: 10
- group: general-server-without-chat
group_label: server (10/10)
shard_index: 9
shard_count: 10
- group: general-chat
group_label: chat (1/3)
shard_index: 0
shard_count: 3
- group: general-server
group_label: server (2/3)
- group: general-chat
group_label: chat (2/3)
shard_index: 1
shard_count: 3
- group: general-server
group_label: server (3/3)
- group: general-chat
group_label: chat (3/3)
shard_index: 2
shard_count: 3
# Keep parity with pr.yml: workspaces-a is split with Vitest's
@@ -191,8 +234,8 @@ jobs:
- name: Run deterministic Runner workflow scorer tests
run: pnpm test:runner-workflow-evals
build:
name: Build
verify_paperclip_runner:
name: Verify Paperclip Runner
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
@@ -215,11 +258,62 @@ jobs:
node-version: 24
cache: pnpm
- name: Select the pinned Runner Rust toolchain
working-directory: packages/paperclip-runner
run: |
set -euo pipefail
rustup show
toolchain="$(rustup show active-toolchain | awk '{print $1}')"
echo "RUSTUP_TOOLCHAIN=$toolchain" >> "$GITHUB_ENV"
- name: Cache Runner Rust dependencies
# Restore and save only within trusted master-push verification. GitHub
# isolates branch/PR caches from master; other callers compile afresh.
if: ${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }}
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: packages/paperclip-runner/runner -> target
shared-key: release-runner-v1
# Rebuild workspace code and rerun every check. Cache only compiled
# dependencies; never restore installed executables from cargo/bin.
cache-workspace-crates: false
cache-bin: false
# The step guard also restricts restores. Save only after a successful
# master-push verification of that push's exact commit.
save-if: ${{ github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }}
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
- name: Verify Paperclip Runner
run: pnpm --filter @paperclipai/paperclip-runner check:all
build:
name: Build
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.ref }}
persist-credentials: false
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Install dependencies
run: pnpm install --no-frozen-lockfile
- name: Build
run: pnpm build
.github/workflows/release.yml
+278 -6
View File
@@ -1,4 +1,5 @@
name: Release
run-name: ${{ inputs.channel == 'preview' && format('Stack deploy {0} build', inputs.request_id) || inputs.channel == 'cloud-migrator' && format('Cloud migrator {0}', inputs.source_ref) || 'Release' }}
on:
push:
@@ -17,12 +18,22 @@ on:
- stable
- beta
- nightly
- preview
- cloud-migrator
default: stable
source_ref:
description: (stable) Commit SHA, branch, or tag to publish as stable
description: Stable source ref, or full immutable SHA for a preview or cloud migrator build
required: true
type: string
default: master
request_id:
description: (preview/cloud-migrator) Correlation UUID
type: string
default: ""
preview_migrator:
description: (preview) Publish isolated shared and database packages if missing
type: boolean
default: false
stable_date:
description: Enter a UTC date in YYYY-MM-DD format, for example 2026-03-18. Do not enter a version string. The workflow will resolve that date to a stable version such as 2026.318.0, then 2026.318.1 for the next same-day stable.
required: false
@@ -46,7 +57,7 @@ on:
default: false
concurrency:
group: release-${{ github.event_name }}-${{ github.ref }}
group: ${{ (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && format('{0}-{1}', inputs.channel, inputs.source_ref) || format('release-{0}-{1}', github.event_name, github.ref) }}
cancel-in-progress: false
env:
@@ -64,11 +75,247 @@ env:
NPM_PUBLISH_VERIFY_DELAY_SECONDS: "10"
jobs:
plan_preview:
name: Check preview artifacts
if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
image: ${{ steps.plan.outputs.image }}
packages: ${{ steps.plan.outputs.packages }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Validate immutable source and inspect existing artifacts
id: plan
env:
SOURCE_SHA: ${{ inputs.source_ref }}
REQUEST_ID: ${{ inputs.request_id }}
PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}
PLAN_COMMAND: ${{ inputs.channel == 'cloud-migrator' && 'plan-migrator' || 'plan' }}
run: node scripts/preview-artifacts.mjs "$PLAN_COMMAND" "$SOURCE_SHA" "$REQUEST_ID" "$PREVIEW_MIGRATOR"
package_preview:
name: Build preview migrator
needs: plan_preview
if: needs.plan_preview.outputs.packages == 'true'
runs-on: ubuntu-latest
timeout-minutes: 30
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
path: trusted
persist-credentials: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.source_ref }}
path: source
persist-credentials: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Install build dependencies without lifecycle scripts
working-directory: source
run: pnpm install --ignore-scripts --no-frozen-lockfile
- name: Build and pack exact-source preview packages
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: node trusted/scripts/preview-artifacts.mjs pack source packages "$SOURCE_SHA"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: preview-packages
overwrite: true
path: packages/*.tgz
if-no-files-found: error
retention-days: 7
publish_preview:
name: Publish preview migrator
needs: [plan_preview, package_preview]
if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.packages == 'true' && needs.package_preview.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 30
# A manual preview and a merge-triggered migrator may compile in parallel.
# Serialize only publication so they cannot race an immutable npm version,
# without making the migrator wait for a preview's separate image build.
concurrency:
group: preview-package-publish-${{ inputs.source_ref }}
cancel-in-progress: false
# Reuse release.yml's established npm trusted-publisher identity. This job
# publishes only isolated preview versions; it cannot advance lane tags.
environment: npm-canary
permissions:
contents: read
id-token: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Install npm with trusted publishing support
run: npm install --global npm@11.18.0 --ignore-scripts
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: preview-packages
path: preview-packages
- name: Publish immutable preview packages without running package code
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: node scripts/preview-artifacts.mjs publish preview-packages "$SOURCE_SHA"
image_preview:
name: Build preview cloud image
needs: plan_preview
if: needs.plan_preview.outputs.image == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ inputs.source_ref }}
persist-credentials: false
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Prepare locked image context
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: |
set -euo pipefail
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
pnpm install --resolution-only --ignore-scripts --ignore-pnpmfile --no-frozen-lockfile
echo "TOOLS_EPOCH=$(date -u +%G-W%V)" >> "$GITHUB_ENV"
- uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
- name: Build the immutable cloud image without registry credentials
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
target: cloud
platforms: linux/amd64
push: false
provenance: false # Docker archives cannot carry registry attestations.
outputs: type=docker,dest=${{ runner.temp }}/preview-image.tar
tags: ghcr.io/paperclipai/paperclip:sha-${{ inputs.source_ref }}-cloud
build-args: |
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_COMMIT=${{ inputs.source_ref }}
PAPERCLIP_BUILD_VERSION=0.0.0-preview.g${{ inputs.source_ref }}
CLI_TOOLS_CACHE_EPOCH=${{ env.TOOLS_EPOCH }}
labels: |
org.opencontainers.image.revision=${{ inputs.source_ref }}
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: preview-image
overwrite: true
path: ${{ runner.temp }}/preview-image.tar
compression-level: 0
if-no-files-found: error
retention-days: 1
publish_image_preview:
name: Publish preview cloud image
needs: [plan_preview, image_preview]
if: github.ref == 'refs/heads/master' && needs.plan_preview.outputs.image == 'true' && needs.image_preview.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 30
# This existing environment has an external master-only branch policy.
environment: npm-canary
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
with:
name: preview-image
path: preview-image
- uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Verify image identity and publish without executing image code
env:
SOURCE_SHA: ${{ inputs.source_ref }}
run: node scripts/preview-artifacts.mjs publish-image preview-image/preview-image.tar "$SOURCE_SHA"
result_preview:
name: Verify preview artifacts
needs: [plan_preview, image_preview, publish_image_preview, package_preview, publish_preview]
if: >-
always() && inputs.channel == 'preview' && needs.plan_preview.result == 'success' &&
(needs.publish_image_preview.result == 'success' || needs.plan_preview.outputs.image == 'false') &&
(needs.publish_preview.result == 'success' || needs.plan_preview.outputs.packages == 'false')
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Confirm exact artifacts are visible
env:
SOURCE_SHA: ${{ inputs.source_ref }}
REQUEST_ID: ${{ inputs.request_id }}
PREVIEW_MIGRATOR: ${{ inputs.preview_migrator }}
run: node scripts/preview-artifacts.mjs result "$SOURCE_SHA" "$REQUEST_ID"
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
with:
name: stack-deploy-result
overwrite: true
path: stack-deploy-result/result.json
if-no-files-found: error
retention-days: 30
verify_canary:
if: github.event_name == 'push'
uses: ./.github/workflows/release-verify.yml
with:
ref: ${{ github.sha }}
name: Reuse exact-source verification
if: github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
timeout-minutes: 50
permissions:
contents: read
actions: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Require successful source checks for this exact master push
env:
GITHUB_TOKEN: ${{ github.token }}
SOURCE_SHA: ${{ github.sha }}
run: node scripts/cloud-source-verification.mjs "$SOURCE_SHA"
publish_canary:
if: github.event_name == 'push'
@@ -81,6 +328,8 @@ jobs:
permissions:
contents: write
id-token: write
# For the explicit docker.yml dispatch below.
actions: write
steps:
- name: Checkout repository
@@ -139,6 +388,29 @@ jobs:
git push origin "refs/tags/${tag}"
echo "version=${tag#canary/v}" >> "$GITHUB_OUTPUT"
# Canary images previously relied on the master-push docker.yml run,
# whose single pending concurrency slot gets superseded by every
# newer push — on a busy day no canary image publishes at all (five
# consecutive canaries shipped npm packages with no cloud image on
# 2026-09-06, starving downstream managed deploys for ~18 hours).
# Tag pushes made with GITHUB_TOKEN do not fire docker.yml's
# triggers, so dispatch the image build at the canary tag
# explicitly, exactly like the nightly and beta lanes: the run keys
# its concurrency off the tag ref, so no master push can supersede
# it, and docker.yml's `type=sha` mapping publishes the
# sha-<short> and sha-<short>-cloud images either way.
- name: Build Docker images for the canary tag
env:
GH_TOKEN: ${{ github.token }}
run: |
{
echo "## Canary published"
echo ""
echo "- Published canary: \`${{ steps.canary_tag.outputs.version }}\`"
echo "- Docker build dispatched at \`canary/v${{ steps.canary_tag.outputs.version }}\`"
} >> "$GITHUB_STEP_SUMMARY"
gh workflow run docker.yml --ref "refs/tags/canary/v${{ steps.canary_tag.outputs.version }}" --repo "$GITHUB_REPOSITORY"
# The package is already public when this gate runs. A red result leaves the
# immutable canary in npm, but makes the release workflow visibly fail before
# anyone mistakes an installable package for an onboardable one.
.github/workflows/runner-chaos-evals.yml
+3 -1
View File
@@ -12,7 +12,9 @@ on:
type: string
concurrency:
group: runner-chaos-evals-${{ inputs.ref || github.ref }}
# Reusable calls inherit the caller's workflow name. Cloud readiness and
# Release verify the same SHA independently and must not cancel each other.
group: runner-chaos-evals-${{ github.workflow }}-${{ inputs.ref || github.ref }}
cancel-in-progress: true
jobs:
.github/workflows/storybook-deploy.yml
+169
View File
@@ -0,0 +1,169 @@
name: Storybook Deploy
on:
workflow_dispatch:
inputs:
branch:
description: "Repository branch to publish (empty uses the selected workflow branch)"
type: string
default: ""
# Also exposed by Storybook Visual, which is already available on master.
workflow_call:
inputs:
branch:
type: string
default: ""
permissions:
contents: read
jobs:
authorize:
name: Authorize Storybook publisher
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
sha: ${{ steps.source.outputs.sha }}
branch: ${{ steps.source.outputs.branch }}
branch_key: ${{ steps.source.outputs.branch_key }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Require CODEOWNER initiator and rerunner
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
const authorize = require('./.github/scripts/authorize-storybook-deploy.cjs');
await authorize({ github, context });
- name: Pin requested repository branch
id: source
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
SOURCE_BRANCH: ${{ inputs.branch }}
STORYBOOK_S3_BUCKET: ${{ vars.STORYBOOK_S3_BUCKET }}
STORYBOOK_PUBLIC_BASE_URL: ${{ vars.STORYBOOK_PUBLIC_BASE_URL }}
with:
script: |
const branch = process.env.SOURCE_BRANCH || context.ref.slice('refs/heads/'.length);
const { data } = await github.rest.git.getRef({ ...context.repo, ref: `heads/${branch}` });
if (data.ref !== `refs/heads/${branch}` || data.object.type !== 'commit') {
throw new Error('Select an existing branch in this repository.');
}
// With no source override, preserve the exact dispatched commit.
const sha = process.env.SOURCE_BRANCH ? data.object.sha : context.sha;
const { storybookDestination } = require('./.github/scripts/storybook-destination.cjs');
const destination = storybookDestination({ branch, sha,
runId: context.runId, runAttempt: process.env.GITHUB_RUN_ATTEMPT,
bucket: process.env.STORYBOOK_S3_BUCKET, baseUrl: process.env.STORYBOOK_PUBLIC_BASE_URL });
core.setOutput('sha', sha);
core.setOutput('branch_key', destination.branchKey);
core.setOutput('branch', branch);
build:
name: Build selected branch Storybook
permissions: {}
needs: authorize
runs-on: ubuntu-latest
timeout-minutes: 25
outputs:
artifact_name: ${{ steps.artifact.outputs.name }}
env:
STORYBOOK_DISABLE_TELEMETRY: "1"
steps:
- name: Download public source without repository credentials
env:
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
run: |
[[ "$SOURCE_SHA" =~ ^[a-f0-9]{40}$ ]]
curl --fail --silent --show-error --location --retry 3 \
"https://codeload.github.com/paperclipai/paperclip/tar.gz/$SOURCE_SHA" \
--output "$RUNNER_TEMP/source.tar.gz"
tar -xzf "$RUNNER_TEMP/source.tar.gz" --strip-components=1
- uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
package-manager-cache: false
- run: pnpm install --frozen-lockfile --ignore-scripts
- run: pnpm build-storybook
- name: Record source and validate output
id: artifact
env:
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
SOURCE_BRANCH: ${{ needs.authorize.outputs.branch }}
run: |
test -s ui/storybook-static/index.html
test -s ui/storybook-static/iframe.html
test -s ui/storybook-static/index.json
jq -n --arg sha "$SOURCE_SHA" --arg branch "$SOURCE_BRANCH" \
'{sha: $sha, branch: $branch}' > ui/storybook-static/deployment.json
echo "name=storybook-deploy-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" >> "$GITHUB_OUTPUT"
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: ${{ steps.artifact.outputs.name }}
path: ui/storybook-static
if-no-files-found: error
retention-days: 7
deploy:
name: Publish branch Storybook to S3
needs: [authorize, build]
runs-on: ubuntu-latest
timeout-minutes: 15
concurrency:
group: storybook-deploy-${{ needs.authorize.outputs.branch_key }}
cancel-in-progress: false
permissions:
contents: read
id-token: write
environment:
name: storybook-deploy
url: ${{ steps.deployment.outputs.url }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
sparse-checkout: .github/scripts
- name: Recheck CODEOWNER access before publishing
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
with:
script: |
const authorize = require('./.github/scripts/authorize-storybook-deploy.cjs');
await authorize({ github, context });
- name: Download the successful build artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: ${{ needs.build.outputs.artifact_name }}
path: storybook-static
- name: Assume the Storybook-only uploader role
uses: aws-actions/configure-aws-credentials@e6de054238d6b7531b4efff3b6587d9aade6a06c # v6
with:
role-to-assume: ${{ vars.STORYBOOK_AWS_ROLE_ARN }}
aws-region: ${{ vars.STORYBOOK_AWS_REGION }}
role-duration-seconds: 900
- name: Publish this branch preview
id: deployment
env:
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
SOURCE_BRANCH: ${{ needs.authorize.outputs.branch }}
STORYBOOK_S3_BUCKET: ${{ vars.STORYBOOK_S3_BUCKET }}
STORYBOOK_PUBLIC_BASE_URL: ${{ vars.STORYBOOK_PUBLIC_BASE_URL }}
run: node .github/scripts/publish-storybook.cjs
- name: Verify public build and stable branch URL
env:
BUILD_URL: ${{ steps.deployment.outputs.build_url }}
BRANCH_URL: ${{ steps.deployment.outputs.url }}
SOURCE_SHA: ${{ needs.authorize.outputs.sha }}
run: node .github/scripts/verify-storybook.cjs
- name: Upload deployment links
if: ${{ !cancelled() && steps.deployment.outcome == 'success' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: storybook-deployment-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ steps.deployment.outputs.report_path }}
if-no-files-found: error
retention-days: 30
.github/workflows/storybook-visual.yml
+22 -2
View File
@@ -3,6 +3,16 @@ name: Storybook Visual
on:
workflow_dispatch:
inputs:
branch:
description: "Repository branch to publish (deployment only; empty uses the selected workflow branch)"
required: false
type: string
default: ""
deploy_preview:
description: "Publish this branch to S3/CloudFront instead of running visual tests (CODEOWNERS only)"
required: false
type: boolean
default: false
update_snapshots:
description: "Generate updated snapshots and a baseline review bundle"
required: false
@@ -18,7 +28,7 @@ on:
- labeled
concurrency:
group: storybook-visual-${{ github.event.pull_request.number || github.ref }}
group: storybook-visual-${{ inputs.deploy_preview && github.run_id || 'visual' }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
permissions:
@@ -28,7 +38,7 @@ jobs:
visual:
name: Storybook visual regression
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event_name == 'workflow_dispatch' && !inputs.deploy_preview) ||
(github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'storybook-visual'))
runs-on: ubuntu-latest
timeout-minutes: 35
@@ -100,3 +110,13 @@ jobs:
path: tests/storybook-visual/baseline-review/snapshots.tgz
retention-days: 30
if-no-files-found: error
preview:
name: Deploy selected branch Storybook
if: github.event_name == 'workflow_dispatch' && inputs.deploy_preview
permissions:
contents: read
id-token: write
uses: ./.github/workflows/storybook-deploy.yml
with:
branch: ${{ inputs.branch }}