Merge origin/master into codex/grok-native-runner

Preserve Grok credential staging alongside the new remote Codex assigned-MCP credential boundary. Seven focused native-runtime checks pass.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-09-24 13:58:13 -07:00
commit 5a176ea301
178 files changed
+8994 -1999

No files matched your search

-50
View File
@@ -14,13 +14,6 @@ concurrency:
cancel-in-progress: false
jobs:
image:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
permissions:
contents: read
packages: write
uses: ./.github/workflows/docker-cloud.yml
verify:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
permissions:
@@ -29,29 +22,6 @@ jobs:
with:
ref: ${{ github.sha }}
artifacts:
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
name: Wait for exact-source cloud artifacts
# Bookkeeping must not wait for the AWS builders it observes.
runs-on: ubuntu-latest
timeout-minutes: 35
permissions:
contents: read
actions: read
attestations: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Wait for verified image and exact-source migrator
env:
GH_TOKEN: ${{ github.token }}
SOURCE_SHA: ${{ github.sha }}
run: node scripts/cloud-readiness.mjs "$SOURCE_SHA"
source_verified:
# npm canary publication reuses this exact-source verification proof.
# Keep it independent of image/migrator availability, and fail closed when
@@ -79,23 +49,3 @@ jobs:
SOURCE_SHA: ${{ github.sha }}
run: |
echo "Cloud source verified v1: $SOURCE_SHA" >> "$GITHUB_STEP_SUMMARY"
ready:
# Versioned consumer contract. Never add always() or continue-on-error:
# failed, cancelled, or skipped prerequisites must not report readiness.
name: Cloud deployable v1
needs: [verify, image, artifacts]
if: github.repository == 'paperclipai/paperclip' && github.ref == 'refs/heads/master'
# Bookkeeping must not wait for the AWS builders it observes.
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- name: Record cloud readiness
env:
SOURCE_SHA: ${{ github.sha }}
run: |
{
echo "Cloud deployable v1: $SOURCE_SHA"
echo "Source verification passed; the full-SHA image and exact-source migrator are available."
echo "Deployment tooling must still resolve and pin the image and migrator and validate migration compatibility."
} >> "$GITHUB_STEP_SUMMARY"
-290
View File
@@ -1,290 +0,0 @@
name: Docker cloud
on:
workflow_dispatch:
workflow_call:
permissions: {}
# Independent SHAs can build immediately on separate runners.
# Repeated requests for the same source serialize without cancelling a build.
# No mutable canary channel is promoted here; docker.yml owns that operation.
concurrency:
group: docker-cloud-${{ github.sha }}
cancel-in-progress: false
jobs:
build-and-push-cloud:
# Only canonical master builds can consume the release Fleet. The runner
# group must also allow this workflow only at refs/heads/master.
# Keep an operator switch for a full-run retry on GitHub-hosted runners.
runs-on: ${{ vars.AWS_CLOUD_BUILDS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 'runs-on/fleet=paperclip-cloud-build-x64/env=public-ci' || 'ubuntu-latest' }}
# Fleet instances expire after 45 minutes, including bootstrap and cleanup.
timeout-minutes: ${{ vars.AWS_CLOUD_BUILDS_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && 40 || 60 }}
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Full history and tags so `git describe` below can compute the
# release version to stamp into the image.
fetch-depth: 0
# `.git` is dockerignored, so a running image cannot derive its own
# version and otherwise reports the source package.json placeholder in
# analytics and the debug panel. Compute it here from the pristine
# checkout (real CalVer drift from the nearest release tag) and pass it
# into the build. Empty when no release tag is reachable — the server
# then keeps its existing fallbacks.
- name: Compute build version
id: build-version
run: |
set -euo pipefail
case "${GITHUB_REF}" in
refs/tags/nightly/v*)
# Lane tags carry the exact published version; stamp it verbatim
# instead of describing drift from the nearest stable tag.
version="${GITHUB_REF#refs/tags/nightly/v}"
;;
refs/tags/beta/v*)
version="${GITHUB_REF#refs/tags/beta/v}"
;;
*)
version="$(git describe --tags --match 'v*' --long --dirty 2>/dev/null || true)"
;;
esac
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "Stamping build version: ${version:-<none>}"
# ISO week stamp for the Dockerfile's tool layer: the layer caches
# across commits and re-pulls the @latest CLI tools when the week rolls
# over, instead of on every build.
- name: Compute tool cache epoch
id: tools-epoch
run: echo "epoch=$(date -u +%G-W%V)" >> "$GITHUB_OUTPUT"
- name: Setup pnpm
uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6
with:
version: 9.15.4
run_install: false
# No dependency cache here: this workflow publishes release images, and
# restoring a shared Actions cache into the build inputs would let a
# poisoned cache entry reach the published artifact.
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- name: Refresh lockfile for Docker build context
run: |
set -euo pipefail
pnpm install --resolution-only --ignore-scripts --no-frozen-lockfile
changed="$(git status --porcelain)"
if [ -z "$changed" ]; then
echo "Lockfile already matches package metadata."
exit 0
fi
if printf '%s\n' "$changed" | grep -Fvq ' pnpm-lock.yaml'; then
echo "Unexpected files changed during lockfile refresh:"
echo "$changed"
exit 1
fi
echo "Using refreshed pnpm-lock.yaml in the Docker build context."
- name: Free runner disk
run: |
set -euo pipefail
echo "Disk before cleanup:"
df -h
# A measured hosted cloud build started with 86 GB available.
# Keep ample headroom for BuildKit and image verification, but
# avoid minutes deleting SDKs when neither filesystem needs space.
minimum_free_kib=$((64 * 1024 * 1024))
if docker_root="$(docker info --format '{{.DockerRootDir}}')" \
&& available_kib="$(df -Pk "$docker_root" "$GITHUB_WORKSPACE" | awk 'NR > 1 { rows++; if ($4 !~ /^[0-9]+$/) invalid = 1; if (min == "" || $4 < min) min = $4 } END { if (invalid || rows != 2) exit 1; print min }')" \
&& [[ "$available_kib" =~ ^[0-9]+$ ]] \
&& (( available_kib >= minimum_free_kib )); then
echo "At least 64 GiB is available for Docker and the workspace; skipping cleanup."
exit 0
fi
pnpm store prune || true
sudo apt-get clean || true
sudo rm -rf \
/usr/share/dotnet \
/usr/share/swift \
/usr/local/lib/android \
/usr/local/share/boost \
/usr/local/share/powershell \
/opt/ghc \
/opt/hostedtoolcache/CodeQL \
/opt/hostedtoolcache/PyPy \
/opt/hostedtoolcache/Ruby || true
docker system prune -af || true
echo "Disk after cleanup:"
df -h
- name: Login to GitHub Container Registry
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4
# Mixing several historical manifests missed otherwise reusable native
# layers on fresh builders. Import the nearest available complete cache.
- name: Select cloud cache ancestry
id: cloud-cache
env:
CACHE_IMAGE: ghcr.io/${{ github.repository }}
run: node scripts/select-cloud-cache.mjs
# Deployment tooling reads these labels from the registry to verify an
# image's schema expectations against a migrator before deploying it,
# without pulling the image. The server refuses to start when the
# database is missing bundled migrations, so orchestrators need a cheap
# way to check image/migrator compatibility up front.
- name: Compute schema migration labels
id: schema
run: |
set -euo pipefail
last=$(ls packages/db/src/migrations/*.sql | sed 's|.*/||' | LC_ALL=C sort | tail -1)
count=$(ls packages/db/src/migrations/*.sql | wc -l | tr -d ' ')
echo "last=${last}" >> "$GITHUB_OUTPUT"
echo "count=${count}" >> "$GITHUB_OUTPUT"
# Published under the same lane tag set as the self-hosted image, with a
# `-cloud` suffix (nightly-cloud, latest-cloud, <version>-cloud,
# sha-<short>-cloud). `:canary-cloud` follows the same retag-step
# ownership rule as `:canary` above.
- name: Docker meta (cloud)
id: meta-cloud
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6
with:
images: ghcr.io/${{ github.repository }}
flavor: |
suffix=-cloud,onlatest=true
tags: |
type=raw,value=nightly,enable=${{ startsWith(github.ref, 'refs/tags/nightly/v') }}
type=raw,value=beta,enable=${{ startsWith(github.ref, 'refs/tags/beta/v') }}
type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{version}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=semver,pattern={{major}}.{{minor}},enable=${{ startsWith(github.ref, 'refs/tags/v') }}
type=sha
labels: |
io.github.paperclipai.schema.last-migration=${{ steps.schema.outputs.last }}
io.github.paperclipai.schema.migration-count=${{ steps.schema.outputs.count }}
- name: Build and push (cloud)
id: build-cloud
uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
with:
context: .
target: cloud
# Space-separated sandbox-provider directory names to build into
# the variant; add here when managed deployments need another.
# CLOUD_BUNDLED_SERVER_DEPS names the optional peer packages the
# variant installs from server/package.json's declared version;
# add another name there when a managed tenant needs it.
build-args: |
USER_UID=1001
USER_GID=1001
CLOUD_BUNDLED_PLUGINS=daytona
CLOUD_BUNDLED_SERVER_DEPS=@sentry/node
PAPERCLIP_BUILD_VERSION=${{ steps.build-version.outputs.version }}
PAPERCLIP_BUILD_COMMIT=${{ github.sha }}
CLI_TOOLS_CACHE_EPOCH=${{ steps.tools-epoch.outputs.epoch }}
# amd64 only, unlike the self-hosted image above: the cloud variant
# is consumed exclusively by managed-deployment hosts, which run
# amd64. The QEMU-emulated arm64 half dominated this job's wall
# clock, and dropping it roughly halves time-to-deployable-image.
platforms: linux/amd64
push: true
# Same-SHA builds serialize above; different SHAs never share a
# writable cache ref. Registry layers are content-addressed and
# shared even when cache manifests have separate tags.
cache-from: ${{ steps.cloud-cache.outputs.source }}
cache-to: type=registry,ref=ghcr.io/${{ github.repository }}:buildcache-cloud-${{ github.sha }},mode=max
tags: ${{ steps.meta-cloud.outputs.tags }}
labels: ${{ steps.meta-cloud.outputs.labels }}
# The cloud target installs @sentry/node at the version
# server/package.json declares, into a directory the server's own
# module resolution walks. Verify the image this job just pushed, not
# a local build, so a build-cache or layer-ordering regression is
# caught before any tenant runs the image.
- name: Verify the pushed image resolves the declared Sentry version
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: |
set -euo pipefail
expected="$(node -e "process.stdout.write(require('./server/package.json').peerDependencies['@sentry/node'])")"
test -n "$expected"
installed="$(docker run --rm --pull always \
-v "$PWD/scripts/assert-cloud-image-sentry.mjs:/app/server/.ci-sentry-probe.mjs:ro" \
--entrypoint node "$IMAGE" /app/server/.ci-sentry-probe.mjs)"
echo "Declared optional peer version: $expected"
echo "Installed in the pushed image: $installed"
if [ "$installed" != "$expected" ]; then
echo "ERROR: the pushed image resolves @sentry/node@$installed, expected @sentry/node@$expected" >&2
exit 1
fi
echo "The pushed image resolves the declared @sentry/node version."
# Managed hosts run node as 1001:1001. Bake that identity into the image
# so usermod does not walk the mounted home on every container start.
# Check before the entrypoint can repair a wrongly built identity.
- name: Verify cloud runtime user
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: |
set -euo pipefail
docker run --rm --entrypoint sh "$IMAGE" -ec '
test "$(id -u node)" = 1001
test "$(id -g node)" = 1001
test "$USER_UID" = 1001
test "$USER_GID" = 1001
'
docker run --rm -e USER_UID=1001 -e USER_GID=1001 "$IMAGE" sh -ec '
test "$(id -u)" = 1001
test "$(id -g)" = 1001
test -w "$PAPERCLIP_HOME"
'
# Verify the independently published cloud image without waiting for
# the self-hosted manifest job. The Sentry check already pulled it.
- name: Verify cloud PID 1 reaps orphaned processes
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
run: docker run --rm -i "$IMAGE" sh -s < scripts/assert-orphan-reaping.sh
# Cloud's commit resolver and preview-artifact planner use the full SHA.
# Publish that address only after checking this build's exact digest.
# Retagging reuses the registry manifest and does not rebuild the image.
- name: Publish verified full-SHA cloud tag
env:
IMAGE: ghcr.io/${{ github.repository }}@${{ steps.build-cloud.outputs.digest }}
FULL_SHA_TAG: ghcr.io/${{ github.repository }}:sha-${{ github.sha }}-cloud
run: |
set -euo pipefail
revision="$(docker image inspect "$IMAGE" --format '{{ index .Config.Labels "org.opencontainers.image.revision" }}')"
platform="$(docker image inspect "$IMAGE" --format '{{ .Os }}/{{ .Architecture }}')"
test "$revision" = "$GITHUB_SHA"
test "$platform" = linux/amd64
docker buildx imagetools create --prefer-index=false --tag "$FULL_SHA_TAG" "$IMAGE"
+6 -18
View File
@@ -53,9 +53,8 @@ jobs:
# ten master commits sampled that day never produced an image at all.
#
# Each platform now builds on a runner of its own architecture and pushes by
# digest; `merge` assembles the manifest list. arm64 is kept rather than
# dropped (the cloud variant below dropped it and is amd64-only) because
# this is the self-hosted image, and ARM hosts consume it.
# digest; `merge` assembles the manifest list. Both architectures remain
# available to self-hosted installations and downstream image composers.
build-and-push:
strategy:
# Independent legs: one architecture failing should still publish
@@ -353,7 +352,7 @@ jobs:
# until the cgroup pid limit is exhausted and every fork() in the
# container fails. Run against the pushed manifest rather than a local
# build: the legs push by digest, so nothing is loaded into this
# runner's daemon. The independent cloud workflow checks its own image.
# runner's daemon.
- name: Verify PID 1 reaps orphaned processes
env:
# Through the environment, not interpolated into the script body, so
@@ -387,17 +386,7 @@ jobs:
subject-digest: ${{ steps.standard.outputs.digest }}
push-to-registry: true
# Master cloud builds start independently in docker-cloud.yml. Tag builds
# and manual Docker dispatches call the same implementation, preserving the
# release tags and the canary promotion dependency below.
build-and-push-cloud:
if: github.event_name != 'push' || github.ref != 'refs/heads/master'
uses: ./.github/workflows/docker-cloud.yml
permissions:
contents: read
packages: write
# Moves the mutable `:canary` / `:canary-cloud` channel tags. Kept OUT
# Moves the mutable `:canary` channel tag. Kept OUT
# of the build jobs and serialized in its own lane, and — the load-
# bearing property — CONVERGENT rather than self-interested: a
# promotion does not promote "its own" canary, it retags the channel
@@ -418,7 +407,7 @@ jobs:
# merge-and-push, not build-and-push: the per-arch legs push untagged
# digests, and the production `sha-*` tags this promotion retags only
# exist once the manifest merge has named them.
needs: [merge-and-push, build-and-push-cloud]
needs: [merge-and-push]
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -447,10 +436,9 @@ jobs:
exit 0
fi
short="$(printf '%s' "$sha" | cut -c1-7)"
if ! docker buildx imagetools inspect "$IMAGE:sha-${short}-cloud" >/dev/null 2>&1; then
if ! docker buildx imagetools inspect "$IMAGE:sha-${short}" >/dev/null 2>&1; then
echo "images for ${current} (sha-${short}) not published yet; its own promotion converges the channel"
exit 0
fi
docker buildx imagetools create -t "$IMAGE:canary" "$IMAGE:sha-${short}"
docker buildx imagetools create -t "$IMAGE:canary-cloud" "$IMAGE:sha-${short}-cloud"
echo "channel tags moved to canary ${current} (sha-${short})"
+1 -1
View File
@@ -417,7 +417,7 @@ jobs:
# explicitly, exactly like the nightly and beta lanes: the run keys
# its concurrency off the tag ref, so no master push can supersede
# it, and docker.yml's `type=sha` mapping publishes the
# sha-<short> and sha-<short>-cloud images either way.
# standard sha-<short> images either way.
- name: Build Docker images for the canary tag
env:
GH_TOKEN: ${{ github.token }}
@@ -22,6 +22,14 @@ on:
description: "Optional lower concurrency (at least 2, no higher than the configured campaign limit)"
type: string
required: false
grok_authentication:
description: "Explicit Grok credential source; subscription uses protected GROK_AUTH_JSON"
type: choice
options:
- api_key
- subscription
default: api_key
required: false
max_infrastructure_retries:
description: "Automatic retries only for explicitly retryable infrastructure failures (0-3)"
type: number
@@ -256,6 +264,7 @@ jobs:
MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }}
REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}
ROSTERS: ${{ inputs.rosters || 'all' }}
GROK_AUTHENTICATION: ${{ inputs.grok_authentication || 'api_key' }}
run: |
set -euo pipefail
if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then
@@ -272,6 +281,7 @@ jobs:
node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog \
--evals-root .paperclip-evals \
--rosters "$ROSTERS" \
--grok-authentication "$GROK_AUTHENTICATION" \
--campaign-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \
--max-parallel "$MAX_PARALLEL" \
--output runner-protocol-eval-catalog.json
@@ -542,6 +552,7 @@ jobs:
ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }}
OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }}
XAI_API_KEY: ${{ matrix.credentialName == 'XAI_API_KEY' && secrets.XAI_API_KEY || '' }}
PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET: ${{ matrix.credentialName == 'PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET' && secrets.GROK_AUTH_JSON || '' }}
PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID }}
PAPERCLIP_CLAUDE_MANAGED_AGENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_ID }}
PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION }}
@@ -582,19 +593,36 @@ jobs:
--runner-package runner-protocol-build/extracted/paperclip-runner.tgz \
--runnerd runner-protocol-build/extracted/paperclip-runnerd \
--runs-root cell-output/runs \
--summary-path cell-output/roster-summary.json \
--max-infrastructure-retries "$MAX_INFRASTRUCTURE_RETRIES" \
--run-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CELL_ID}"
status=$?
set -e
CELL_EXIT_CODE="$status" node --input-type=module <<'NODE'
import { writeFileSync } from "node:fs";
import { readFileSync, writeFileSync } from "node:fs";
let authenticationMode;
let authenticationEvidenceFailure;
if (["XAI_API_KEY", "PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET"].includes(process.env.CREDENTIAL_NAME)) {
const expected = process.env.CREDENTIAL_NAME === "XAI_API_KEY" ? "api_key" : "subscription";
try {
const observed = JSON.parse(readFileSync("cell-output/roster-summary.json", "utf8")).authenticationMode;
// Never copy arbitrary provider output into cell metadata or errors.
if (["api_key", "subscription"].includes(observed)) authenticationMode = observed;
if (authenticationMode !== expected) authenticationEvidenceFailure = "grok_authentication_evidence_mismatch";
} catch {
authenticationEvidenceFailure = "grok_authentication_evidence_unreadable";
}
}
writeFileSync("cell-output/cell.json", `${JSON.stringify({
schema: "paperclip.runner-protocol-eval.cell/v1",
cellId: process.env.CELL_ID,
rosterFile: process.env.ROSTER_FILE,
caseId: process.env.CASE_ID,
...(authenticationMode ? { authenticationMode } : {}),
...(authenticationEvidenceFailure ? { authenticationEvidenceFailure } : {}),
exitCode: Number(process.env.CELL_EXIT_CODE),
}, null, 2)}\n`, { mode: 0o600 });
if (authenticationEvidenceFailure) throw new Error(authenticationEvidenceFailure);
NODE
exit "$status"