name: Runner Live Protocol Evals on: schedule: - cron: "23 9 * * 0" workflow_dispatch: inputs: target_branch: description: "Branch in paperclipai/paperclip to evaluate; trusted orchestration still runs from master" type: string required: false evals_sha: description: "Exact 40-character paperclipai/paperclip-evals commit to execute" type: string required: false rosters: description: "Comma-separated live roster IDs/files, or all for the maintained enabled direct suite" type: string default: "all" required: false max_parallel: description: "Optional lower concurrency (at least 2, no higher than the configured campaign limit)" type: string required: false grok_authentication: description: "Explicit Grok credential source; subscription uses protected GROK_AUTH_JSON" type: choice options: - api_key - subscription default: api_key required: false accounting_mode: description: "Controlled Codex/Claude costs and traces, or explicit legacy provider qualification" type: choice options: - controlled - legacy default: controlled max_infrastructure_retries: description: "Automatic retries only for explicitly retryable infrastructure failures (0-3)" type: number default: 0 permissions: contents: read concurrency: group: runner-protocol-live-evals-${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch && format('development-{0}', inputs.target_branch) || format('protected-{0}', github.run_id) }} cancel-in-progress: ${{ github.event_name == 'workflow_dispatch' && inputs.target_branch != '' && inputs.target_branch != github.event.repository.default_branch }} jobs: authorize: name: Authorize paid direct eval campaign if: github.event_name != 'schedule' || vars.RUNNER_PROTOCOL_EVAL_NIGHTLY_ENABLED == 'true' runs-on: ubuntu-latest timeout-minutes: 5 permissions: contents: read outputs: accounting_mode: ${{ steps.mode.outputs.mode }} retry_count: ${{ steps.mode.outputs.retries }} controller_lock_sha256: ${{ steps.mode.outputs.controller_lock_sha256 }} test_runner: ${{ steps.runner.outputs.runner }} max_parallel_default: ${{ steps.runner.outputs.max_parallel_default }} max_parallel_limit: ${{ steps.runner.outputs.max_parallel_limit }} target_sha: ${{ steps.target.outputs.sha }} target_ref: ${{ steps.target.outputs.ref }} evals_sha: ${{ steps.evals.outputs.sha }} steps: - name: Require default branch and allowlisted numeric actor IDs env: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} REF: ${{ github.ref }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} ACTOR: ${{ github.actor }} ACTOR_ID: ${{ github.actor_id }} TRIGGERING_ACTOR: ${{ github.triggering_actor }} ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} run: | set -euo pipefail if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then echo "Paid direct Runner eval campaigns may run only from the default branch." >&2 exit 1 fi if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 exit 1 fi triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then echo "GitHub actor identity contexts disagree; refusing the paid run." >&2 exit 1 fi for candidate in "$triggering_actor_id" "$ACTOR_ID"; do if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then echo "The initiating GitHub account is not authorized to run paid Runner eval campaigns." >&2 exit 1 fi done - name: Resolve requested Paperclip branch to an immutable commit id: target env: GH_TOKEN: ${{ github.token }} REPOSITORY: ${{ github.repository }} TARGET_BRANCH: ${{ inputs.target_branch || github.event.repository.default_branch }} run: | set -euo pipefail if [ -z "$TARGET_BRANCH" ] || [[ "$TARGET_BRANCH" == refs/* ]]; then echo "target_branch must name a branch in this repository without a refs/ prefix." >&2 exit 1 fi encoded_branch="$(jq -rn --arg branch "$TARGET_BRANCH" '$branch | @uri')" target_sha="$(gh api -X GET "repos/$REPOSITORY/branches/$encoded_branch" --jq .commit.sha)" [[ "$target_sha" =~ ^[0-9a-f]{40}$ ]] echo "sha=$target_sha" >> "$GITHUB_OUTPUT" echo "ref=refs/heads/$TARGET_BRANCH" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.sha }} persist-credentials: false - name: Generate private eval-repository token id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" echo "::add-mask::$token" echo "value=$token" >> "$GITHUB_OUTPUT" - name: Verify the private eval program is pinned to an exact commit id: evals env: GH_TOKEN: ${{ steps.evals_token.outputs.value }} EVALS_SHA: ${{ inputs.evals_sha || vars.RUNNER_PROTOCOL_EVALS_SHA }} run: | set -euo pipefail if ! [[ "$EVALS_SHA" =~ ^[0-9a-f]{40}$ ]]; then echo "evals_sha (or RUNNER_PROTOCOL_EVALS_SHA for schedules) must be an exact 40-character commit." >&2 exit 1 fi resolved="$(gh api -X GET "repos/paperclipai/paperclip-evals/commits/$EVALS_SHA" --jq .sha)" test "$resolved" = "$EVALS_SHA" echo "sha=$resolved" >> "$GITHUB_OUTPUT" - name: Validate accounting mode and retry envelope id: mode env: EVENT_NAME: ${{ github.event_name }} ACCOUNTING_MODE: ${{ inputs.accounting_mode || 'controlled' }} RETRIES: ${{ inputs.max_infrastructure_retries || 0 }} CONTROLLER_POLICY_JSON: ${{ vars.RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON }} TARGET_SHA: ${{ steps.target.outputs.sha }} run: | set -euo pipefail if [ "$EVENT_NAME" = schedule ]; then ACCOUNTING_MODE=controlled; RETRIES=0; fi [[ "$RETRIES" =~ ^[0-3]$ ]] case "$ACCOUNTING_MODE" in controlled) test "$TARGET_SHA" = "$GITHUB_SHA" test "$GITHUB_RUN_ATTEMPT" = 1 test "$RETRIES" = 0 if ! jq -e '.schema == "paperclip.runner-eval.hosted-policy/v1"' <<< "${CONTROLLER_POLICY_JSON:-}" >/dev/null; then echo 'Controlled runs require a reviewed RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON; no provider work started.' >&2 exit 1 fi lock_sha="$(jq -er '.controllerLockSha256 | select(type == "string" and test("^[a-f0-9]{64}$"))' <<< "$CONTROLLER_POLICY_JSON")" echo "controller_lock_sha256=$lock_sha" >> "$GITHUB_OUTPUT" ;; legacy) test "$EVENT_NAME" = workflow_dispatch ;; *) exit 1 ;; esac echo "mode=$ACCOUNTING_MODE" >> "$GITHUB_OUTPUT" echo "retries=$RETRIES" >> "$GITHUB_OUTPUT" - name: Select paid test runner id: runner env: AWS_PAID_RUNNER_ENABLED: ${{ vars.RUNNER_E2E_AWS_ENABLED }} run: | set -euo pipefail if [ "$AWS_PAID_RUNNER_ENABLED" = true ]; then { echo 'runner=runs-on/fleet=paperclip-public-pr-x64/env=public-ci' echo 'max_parallel_default=100' echo 'max_parallel_limit=100' } >> "$GITHUB_OUTPUT" else { echo 'runner=ubuntu-latest' echo 'max_parallel_default=32' echo 'max_parallel_limit=57' } >> "$GITHUB_OUTPUT" fi target_lock: name: Resolve target pnpm lockfile needs: authorize runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: read outputs: artifact_id: ${{ steps.upload.outputs.artifact-id }} lock_sha256: ${{ steps.lock.outputs.sha256 }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 env: NPM_CONFIG_AUDIT: "false" NPM_CONFIG_FUND: "false" NPM_CONFIG_UPDATE_NOTIFIER: "false" with: version: 9.15.4 - name: Resolve target lockfile without lifecycle scripts id: lock run: | set -euo pipefail pnpm install --ignore-scripts --no-frozen-lockfile --lockfile-only test -s pnpm-lock.yaml unexpected="$(git status --short | awk '$2 != "pnpm-lock.yaml" { print }')" if [ -n "$unexpected" ]; then echo "Lockfile resolution changed files other than pnpm-lock.yaml:" >&2 echo "$unexpected" >&2 exit 1 fi echo "sha256=$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" >> "$GITHUB_OUTPUT" - name: Upload resolved target lockfile id: upload uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-target-pnpm-lock-${{ github.run_id }}-${{ github.run_attempt }} path: pnpm-lock.yaml retention-days: 30 if-no-files-found: error catalog: name: Pin and fan out the direct Evalbook roster needs: authorize runs-on: ubuntu-latest timeout-minutes: 10 permissions: contents: read outputs: matrix_0: ${{ steps.catalog.outputs.matrix_0 }} matrix_1: ${{ steps.catalog.outputs.matrix_1 }} matrix_1_present: ${{ steps.catalog.outputs.matrix_1_present }} max_parallel_per_shard: ${{ steps.catalog.outputs.max_parallel_per_shard }} selected: ${{ steps.catalog.outputs.selected }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.sha }} persist-credentials: false - name: Generate private eval-repository token id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" echo "::add-mask::$token" echo "value=$token" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: paperclipai/paperclip-evals ref: ${{ needs.authorize.outputs.evals_sha }} path: .paperclip-evals token: ${{ steps.evals_token.outputs.value }} persist-credentials: false - name: Build the two bounded roster-plus-case matrices id: catalog env: PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }} MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }} MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }} REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }} ROSTERS: ${{ inputs.rosters || 'all' }} GROK_AUTHENTICATION: ${{ inputs.grok_authentication || 'api_key' }} ACCOUNTING_MODE: ${{ needs.authorize.outputs.accounting_mode }} CONTROLLER_POLICY_JSON: ${{ vars.RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON }} run: | set -euo pipefail if ! [[ "$MAX_PARALLEL" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL" -lt 2 ] || [ "$MAX_PARALLEL" -gt "$MAX_PARALLEL_LIMIT" ]; then echo "RUNNER_E2E_MAX_PARALLEL must be an integer from 2 through $MAX_PARALLEL_LIMIT for the two-shard direct suite." >&2 exit 1 fi if [ -n "${REQUESTED_MAX_PARALLEL:-}" ]; then if ! [[ "$REQUESTED_MAX_PARALLEL" =~ ^[1-9][0-9]{0,2}$ ]] || [ "$REQUESTED_MAX_PARALLEL" -lt 2 ] || [ "$REQUESTED_MAX_PARALLEL" -gt "$MAX_PARALLEL" ]; then echo "max_parallel must be an integer from 2 through the configured campaign limit." >&2 exit 1 fi MAX_PARALLEL="$REQUESTED_MAX_PARALLEL" fi node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs catalog \ --evals-root .paperclip-evals \ --rosters "$ROSTERS" \ --grok-authentication "$GROK_AUTHENTICATION" \ --campaign-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ --max-parallel "$MAX_PARALLEL" \ --output runner-protocol-eval-catalog.json if [ "$ACCOUNTING_MODE" = controlled ]; then jq -e 'all(.cells[]; (.provider == "codex" and (.acpxAgent // "") == "" and .driver == "codex_app_server") or (.provider == "acpx" and .acpxAgent == "claude" and .driver == "acpx_runtime"))' runner-protocol-eval-catalog.json >/dev/null || { echo 'Controlled selection supports native Codex and ACPX Claude only; select their rosters explicitly.' >&2; exit 1; } maximum="$(jq -er '.maxCells | select(type == "number" and floor == . and . > 0 and . <= 512)' <<< "$CONTROLLER_POLICY_JSON")" jq -e --argjson maximum "$maximum" '.cells | length <= $maximum' runner-protocol-eval-catalog.json >/dev/null fi - name: Require the chat-report renderer before paid execution run: | set -euo pipefail python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report --help | grep -q -- --public-viewer - name: Upload immutable campaign catalog uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-eval-catalog.json retention-days: 30 if-no-files-found: error build_runner: name: Build portable direct-eval runner once needs: [authorize, target_lock, catalog] runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: 30 permissions: contents: read steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ needs.authorize.outputs.target_sha }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 env: NPM_CONFIG_AUDIT: "false" NPM_CONFIG_FUND: "false" NPM_CONFIG_UPDATE_NOTIFIER: "false" with: version: 9.15.4 - name: Download resolved target lockfile uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} path: ${{ runner.temp }}/runner-protocol-target-lock - name: Restore resolved target lockfile env: TARGET_SHA: ${{ needs.authorize.outputs.target_sha }} EXPECTED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$TARGET_SHA" lock="$RUNNER_TEMP/runner-protocol-target-lock/pnpm-lock.yaml" test -f "$lock" test "$(find "$(dirname "$lock")" -type f | wc -l | tr -d ' ')" = 1 test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" cp "$lock" pnpm-lock.yaml test "$(sha256sum pnpm-lock.yaml | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 cache: pnpm - run: pnpm install --frozen-lockfile --ignore-scripts - name: Materialize the pinned OpenCode executable before packaging run: node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - name: Materialize the pinned Grok executable when the target includes it run: | if [ -f packages/paperclip-runner/scripts/provision-grok.mjs ]; then sudo node packages/paperclip-runner/scripts/provision-grok.mjs /opt/paperclip/providers/grok/1.0.13/grok fi - name: Build runner CLI, daemon, and canonical attempt viewer run: | set -euo pipefail pnpm --filter @paperclipai/paperclip-runner build:typescript pnpm --filter @paperclipai/paperclip-runner build:runner-binaries pnpm --filter @paperclipai/paperclip-runner build:issue-thread # Older target refs must fail before paid cells, not publish an empty viewer. grep -q 'paperclip-eval-report' packages/paperclip-runner/dist-issue-thread/assets/*.js grep -q 'evalbook-site' packages/paperclip-runner/dist-issue-thread/assets/*.css - name: Package a portable provider runtime run: | set -euo pipefail mkdir -p "$RUNNER_TEMP/runner-protocol-build/package" "$RUNNER_TEMP/runner-protocol-build/portable" pnpm --dir packages/paperclip-runner pack \ --pack-destination "$RUNNER_TEMP/runner-protocol-build/package" package="$(find "$RUNNER_TEMP/runner-protocol-build/package" -maxdepth 1 -type f -name '*.tgz' -print -quit)" test -f "$package" pnpm --filter @paperclipai/paperclip-runner deploy --prod \ "$RUNNER_TEMP/runner-protocol-build/portable" cp "$package" "$RUNNER_TEMP/runner-protocol-build/paperclip-runner.tgz" cp packages/paperclip-runner/runner/target/debug/paperclip-runnerd "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" cp -R packages/paperclip-runner/dist-issue-thread "$RUNNER_TEMP/runner-protocol-build/dist-issue-thread" test -f "$RUNNER_TEMP/runner-protocol-build/portable/dist/cli/eval-session.js" test -d "$RUNNER_TEMP/runner-protocol-build/portable/node_modules/.pnpm" test -x "$RUNNER_TEMP/runner-protocol-build/paperclip-runnerd" tar --create --gzip --file runner-protocol-build.tar.gz -C "$RUNNER_TEMP/runner-protocol-build" . sha256sum runner-protocol-build.tar.gz > runner-protocol-build.tar.gz.sha256 - name: Upload immutable portable runner uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} path: | runner-protocol-build.tar.gz runner-protocol-build.tar.gz.sha256 retention-days: 1 compression-level: 0 if-no-files-found: error - name: Upload canonical viewer for publisher byte verification uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }} path: packages/paperclip-runner/dist-issue-thread/ retention-days: 30 if-no-files-found: error build_controller: name: Build trusted controller without private archive access needs: [authorize, target_lock] runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read outputs: artifact_id: ${{ steps.upload.outputs.artifact-id }} runtime_receipt_sha256: ${{ steps.build.outputs.runtime_receipt_sha256 }} steps: - name: Select accounting mode run: test '${{ needs.authorize.outputs.accounting_mode }}' = controlled || test '${{ needs.authorize.outputs.accounting_mode }}' = legacy - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 if: needs.authorize.outputs.accounting_mode == 'controlled' with: ref: ${{ github.sha }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 if: needs.authorize.outputs.accounting_mode == 'controlled' with: node-version: 24 - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 if: needs.authorize.outputs.accounting_mode == 'controlled' env: NPM_CONFIG_AUDIT: "false" NPM_CONFIG_FUND: "false" NPM_CONFIG_UPDATE_NOTIFIER: "false" with: version: 9.15.4 - name: Download resolved lockfile for review binding if: needs.authorize.outputs.accounting_mode == 'controlled' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.target_lock.outputs.artifact_id }} path: ${{ runner.temp }}/runner-controller-lock - name: Build portable controller in an unprivileged job if: needs.authorize.outputs.accounting_mode == 'controlled' id: build env: EXPECTED_LOCK_SHA256: ${{ needs.authorize.outputs.controller_lock_sha256 }} RESOLVED_LOCK_SHA256: ${{ needs.target_lock.outputs.lock_sha256 }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$GITHUB_SHA" test "$RESOLVED_LOCK_SHA256" = "$EXPECTED_LOCK_SHA256" lock="$RUNNER_TEMP/runner-controller-lock/pnpm-lock.yaml" test "$(sha256sum "$lock" | cut -d ' ' -f 1)" = "$EXPECTED_LOCK_SHA256" cp "$lock" pnpm-lock.yaml pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/runner-controller-pnpm-store" node packages/paperclip-runner/scripts/controlled-evals/build-runtime.mjs \ "$RUNNER_TEMP/runner-protocol-runtime" "$GITHUB_SHA" > "$RUNNER_TEMP/controller-runtime-receipt.json" echo "runtime_receipt_sha256=$(jq -r .receiptSha256 "$RUNNER_TEMP/controller-runtime-receipt.json")" >> "$GITHUB_OUTPUT" - name: Retain the trusted controller code for every worker if: needs.authorize.outputs.accounting_mode == 'controlled' id: upload uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-runtime-${{ github.run_id }}-${{ github.run_attempt }} path: ${{ runner.temp }}/runner-protocol-runtime/ retention-days: 30 if-no-files-found: error prepare_controller: name: Prepare one shared cost and trace manifest needs: [authorize, catalog, build_runner, build_controller] runs-on: ubuntu-latest timeout-minutes: 15 permissions: contents: read id-token: write environment: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-history' || '' }} outputs: artifact_id: ${{ needs.build_controller.outputs.artifact_id }} manifest_sha256: ${{ steps.prepare.outputs.manifest_sha256 }} runtime_receipt_sha256: ${{ needs.build_controller.outputs.runtime_receipt_sha256 }} steps: - name: Select accounting mode run: test '${{ needs.authorize.outputs.accounting_mode }}' = controlled || test '${{ needs.authorize.outputs.accounting_mode }}' = legacy - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 if: needs.authorize.outputs.accounting_mode == 'controlled' with: ref: ${{ github.sha }} persist-credentials: false - name: Generate private eval-repository token if: needs.authorize.outputs.accounting_mode == 'controlled' id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" echo "::add-mask::$token" echo "value=$token" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 if: needs.authorize.outputs.accounting_mode == 'controlled' with: repository: paperclipai/paperclip-evals ref: ${{ needs.authorize.outputs.evals_sha }} path: .paperclip-evals token: ${{ steps.evals_token.outputs.value }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 if: needs.authorize.outputs.accounting_mode == 'controlled' with: node-version: 24 - name: Download the complete campaign catalog if: needs.authorize.outputs.accounting_mode == 'controlled' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-catalog - name: Download the immutable provider runtime if: needs.authorize.outputs.accounting_mode == 'controlled' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-build - name: Download trusted controller built without archive permissions if: needs.authorize.outputs.accounting_mode == 'controlled' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.build_controller.outputs.artifact_id }} path: ${{ runner.temp }}/runner-protocol-runtime - name: Verify copied controller before private preparation if: needs.authorize.outputs.accounting_mode == 'controlled' env: EXPECTED_LOCK_SHA256: ${{ needs.authorize.outputs.controller_lock_sha256 }} RUNTIME_RECEIPT_SHA256: ${{ needs.build_controller.outputs.runtime_receipt_sha256 }} run: | set -euo pipefail node packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs "$RUNNER_TEMP/runner-protocol-runtime" "$GITHUB_SHA" "$RUNTIME_RECEIPT_SHA256" test "$(jq -r '.inputs["pnpm-lock.yaml"]' "$RUNNER_TEMP/runner-protocol-runtime/runtime.json")" = "$EXPECTED_LOCK_SHA256" - name: Compile requests and bind the reviewed operating policy if: needs.authorize.outputs.accounting_mode == 'controlled' id: prepare env: PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }} CONTROLLER_POLICY_JSON: ${{ vars.RUNNER_PROTOCOL_EVAL_CONTROLLER_POLICY_JSON }} run: | set -euo pipefail test "$(git rev-parse HEAD)" = "$GITHUB_SHA" test "$(git -C .paperclip-evals rev-parse HEAD)" = "$PAPERCLIP_PROTOCOL_EVALS_SHA" python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report --help | grep -q -- --controller-manifest (cd runner-protocol-build && sha256sum --check runner-protocol-build.tar.gz.sha256) mkdir runner-protocol-build/extracted tar -xzf runner-protocol-build/runner-protocol-build.tar.gz -C runner-protocol-build/extracted policy="$RUNNER_TEMP/runner-controller-policy.json" POLICY_PATH="$policy" python3 - <<'PYTHON' import os with os.fdopen(os.open(os.environ["POLICY_PATH"], os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as target: target.write(os.environ["CONTROLLER_POLICY_JSON"]) PYTHON worker="/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}" python3 .paperclip-evals/evals/paperclip-runner/tools/prepare_controlled_campaign.py \ --catalog runner-protocol-catalog/runner-protocol-eval-catalog.json \ --runnerd runner-protocol-build/extracted/paperclip-runnerd \ --worker-runnerd "$worker/runner/paperclip-runnerd" \ --github-run-id "$GITHUB_RUN_ID" --definitions-revision "$PAPERCLIP_PROTOCOL_EVALS_SHA" \ --output "$RUNNER_TEMP/prepared-requests.json" control="$RUNNER_TEMP/runner-protocol-control" node "$RUNNER_TEMP/runner-protocol-runtime/controlled-evals/hosted.mjs" prepare \ runner-protocol-catalog/runner-protocol-eval-catalog.json "$RUNNER_TEMP/prepared-requests.json" \ "$policy" runner-protocol-build/extracted "$control" > "$RUNNER_TEMP/control-preparation.json" cp "$policy" "$control/hosted-policy.json" echo "manifest_sha256=$(jq -r .manifestSha256 "$RUNNER_TEMP/control-preparation.json")" >> "$GITHUB_OUTPUT" - name: Obtain private archive credentials for transfer steps only if: needs.authorize.outputs.accounting_mode == 'controlled' id: private_history_credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} output-env-credentials: false output-credentials: true disable-retry: true action-timeout-s: 60 - name: Retain the identical manifest and policy in private storage if: needs.authorize.outputs.accounting_mode == 'controlled' env: AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} AWS_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} run: node "$RUNNER_TEMP/runner-protocol-runtime/private-history.mjs" upload protocol-control "$GITHUB_RUN_ID" "$RUNNER_TEMP/runner-protocol-control" eval_shard_0: name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }} needs: [authorize, catalog, build_runner, prepare_controller] runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: 18 permissions: contents: read id-token: write environment: name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }} strategy: fail-fast: false max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }} matrix: ${{ fromJSON(needs.catalog.outputs.matrix_0) }} steps: &direct_eval_steps - name: Reauthorize paid execution before provider access env: GH_TOKEN: ${{ github.token }} REF: ${{ github.ref }} DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} ACTOR: ${{ github.actor }} ACTOR_ID: ${{ github.actor_id }} TRIGGERING_ACTOR: ${{ github.triggering_actor }} ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} run: | set -euo pipefail test "$REF" = "refs/heads/$DEFAULT_BRANCH" if [ '${{ needs.authorize.outputs.accounting_mode }}' = controlled ]; then test '${{ needs.authorize.outputs.target_sha }}' = "$GITHUB_SHA" test "$GITHUB_RUN_ATTEMPT" = 1 fi triggering_actor_id="$(gh api "users/$TRIGGERING_ACTOR" --jq .id)" test "$triggering_actor_id" = "$ACTOR_ID" || test "$TRIGGERING_ACTOR" != "$ACTOR" for candidate in "$triggering_actor_id" "$ACTOR_ID"; do jq -e --argjson candidate "$candidate" 'type == "array" and index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null done - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.sha }} persist-credentials: false - name: Generate private eval-repository token id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" echo "::add-mask::$token" echo "value=$token" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: paperclipai/paperclip-evals ref: ${{ needs.authorize.outputs.evals_sha }} path: .paperclip-evals token: ${{ steps.evals_token.outputs.value }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - name: Download portable runner uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-build - name: Verify and extract portable runner run: | set -euo pipefail cd runner-protocol-build sha256sum --check runner-protocol-build.tar.gz.sha256 mkdir extracted tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted test -x extracted/paperclip-runnerd - name: Download the trusted portable controller if: needs.authorize.outputs.accounting_mode == 'controlled' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.prepare_controller.outputs.artifact_id }} path: /tmp/paperclip-runner-controlled-${{ github.run_id }}/runtime - name: Verify controller code before archive access if: needs.authorize.outputs.accounting_mode == 'controlled' id: verify_controller env: RUNTIME_RECEIPT_SHA256: ${{ needs.prepare_controller.outputs.runtime_receipt_sha256 }} run: | set -euo pipefail root="/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}" node packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs "$root/runtime" "$GITHUB_SHA" "$RUNTIME_RECEIPT_SHA256" cp -a runner-protocol-build/extracted "$root/runner" # Keep the host policy aligned with runner-full-stack-e2e.yml. This is # deliberately before any provider credential or web-identity step. - name: Provision Codex sandbox on the disposable trusted runner if: matrix.provider == 'codex' || matrix.rosterId == 'protocol-live-acpx-codex-control' env: CONTROLLED_RUNNER_ROOT: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && format('/tmp/paperclip-runner-controlled-{0}/runner', github.run_id) || '' }} run: | node --input-type=module <<'NODE' import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; import { readFileSync, realpathSync, writeFileSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; if (process.platform !== "linux") process.exit(0); let restricted = "0"; try { restricted = readFileSync("/proc/sys/kernel/apparmor_restrict_unprivileged_userns", "utf8").trim(); } catch {} if (restricted !== "1") process.exit(0); const root = realpathSync(process.env.CONTROLLED_RUNNER_ROOT ? path.join(process.env.CONTROLLED_RUNNER_ROOT, "portable") : path.join(process.env.GITHUB_WORKSPACE, "runner-protocol-build/extracted/portable")); const runnerRequire = createRequire(path.join(root, "package.json")); const acpRequire = createRequire(runnerRequire.resolve("@agentclientprotocol/codex-acp/package.json")); const codexRequire = createRequire(acpRequire.resolve("@openai/codex/package.json")); const arch = process.arch === "x64" ? "x64" : process.arch === "arm64" ? "arm64" : null; if (!arch) throw new Error("Unsupported Codex CI architecture"); const platformPackage = codexRequire.resolve(`@openai/codex-linux-${arch}/package.json`); const triple = arch === "x64" ? "x86_64-unknown-linux-musl" : "aarch64-unknown-linux-musl"; const suffix = `/vendor/${triple}/bin/codex`; const binary = realpathSync(path.join(path.dirname(platformPackage), suffix)); if (!binary.startsWith(root + "/node_modules/.pnpm/") || !binary.endsWith(suffix) || !/^[/A-Za-z0-9_.@+\-]+$/.test(binary)) { throw new Error("Codex executable is outside the resolved dependency tree"); } const name = `paperclip-e2e-codex-${createHash("sha256").update(binary).digest("hex").slice(0,16)}`; const profilePath = path.join(process.env.RUNNER_TEMP, "paperclip-codex-userns.apparmor"); writeFileSync(profilePath, `abi ,\ninclude \nprofile ${name} "${binary}" flags=(unconfined) {\n userns,\n}\n`, {mode:0o600, flag:"wx"}); execFileSync("sudo", ["-n", "apparmor_parser", "-r", profilePath], {timeout:15000, stdio:"pipe"}); NODE - name: Obtain private archive credentials for transfer steps only if: needs.authorize.outputs.accounting_mode == 'controlled' id: private_history_credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} output-env-credentials: false output-credentials: true disable-retry: true action-timeout-s: 60 - name: Download the identical private campaign manifest if: needs.authorize.outputs.accounting_mode == 'controlled' env: CELL_ID: ${{ matrix.cellId }} AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} AWS_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} run: node "/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}/runtime/private-history.mjs" download protocol-control "$GITHUB_RUN_ID" "/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}/control" "$CELL_ID" - name: Verify current worker admission before provider credentials if: needs.authorize.outputs.accounting_mode == 'controlled' env: MANIFEST_SHA256: ${{ needs.prepare_controller.outputs.manifest_sha256 }} PAPERCLIP_RUNNER_EVAL_EXECUTION_ID: ${{ matrix.cellId }} run: | set -euo pipefail root="/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}" node "$root/runtime/controlled-evals/hosted.mjs" verify "$root/control/controller-manifest.json" "$MANIFEST_SHA256" "$PAPERCLIP_RUNNER_EVAL_EXECUTION_ID" { echo "PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST=$root/control/controller-manifest.json" echo "PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI=$root/runtime/controlled-evals/archive.mjs" echo "PAPERCLIP_RUNNER_EVAL_EXECUTION_ID=$PAPERCLIP_RUNNER_EVAL_EXECUTION_ID" } >> "$GITHUB_ENV" - name: Prepare short-lived AgentCore web identity if: matrix.credentialName == 'AWS_AGENTCORE_OIDC' env: AGENTCORE_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_EXECUTION_ROLE_ARN }} run: | set -euo pipefail test -n "$AGENTCORE_ROLE_ARN" token="$(curl --fail --silent --show-error \ -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=sts.amazonaws.com" | jq -r .value)" test -n "$token" echo "::add-mask::$token" token_file="$RUNNER_TEMP/runner-protocol-agentcore-token" printf '%s' "$token" > "$token_file" chmod 600 "$token_file" { echo "AWS_WEB_IDENTITY_TOKEN_FILE=$token_file" echo "AWS_ROLE_ARN=$AGENTCORE_ROLE_ARN" echo "AWS_ROLE_SESSION_NAME=runner-protocol-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" } >> "$GITHUB_ENV" - name: Run one immutable direct protocol cell id: direct_eval env: CELL_ID: ${{ matrix.cellId }} ROSTER_FILE: ${{ matrix.rosterFile }} CASE_ID: ${{ matrix.caseId }} CREDENTIAL_NAME: ${{ matrix.credentialName }} PROVIDER: ${{ matrix.provider }} MAX_INFRASTRUCTURE_RETRIES: ${{ needs.authorize.outputs.retry_count }} ACCOUNTING_MODE: ${{ needs.authorize.outputs.accounting_mode }} BRAINTRUST_API_KEY: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && secrets.BRAINTRUST_API_KEY || '' }} OPENAI_API_KEY: ${{ matrix.credentialName == 'OPENAI_API_KEY' && secrets.OPENAI_API_KEY || '' }} ANTHROPIC_API_KEY: ${{ matrix.credentialName == 'ANTHROPIC_API_KEY' && secrets.ANTHROPIC_API_KEY || '' }} OPENROUTER_API_KEY: ${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }} XAI_API_KEY: ${{ matrix.credentialName == 'XAI_API_KEY' && secrets.XAI_API_KEY || '' }} PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET: ${{ matrix.credentialName == 'PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET' && secrets.GROK_AUTH_JSON || '' }} COPILOT_GITHUB_TOKEN: ${{ matrix.credentialName == 'COPILOT_GITHUB_TOKEN' && secrets.COPILOT_GITHUB_TOKEN || '' }} PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID }} PAPERCLIP_CLAUDE_MANAGED_AGENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_ID }} PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION }} PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID: ${{ vars.PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID }} AWS_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }} AWS_DEFAULT_REGION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_REGION }} PAPERCLIP_AWS_AGENTCORE_PROFILE_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_PROFILE_ID }} PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ACCOUNT_ID }} PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_ARN }} PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_HARNESS_VERSION }} PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_ARN }} PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER: ${{ vars.PAPERCLIP_AWS_AGENTCORE_ENDPOINT_QUALIFIER }} PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_RUNTIME_ARN }} PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ARN }} PAPERCLIP_AWS_AGENTCORE_MEMORY_ID: ${{ vars.PAPERCLIP_AWS_AGENTCORE_MEMORY_ID }} PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_INVOCATION_ROLE_ARN }} PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_BUCKET }} PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_PREFIX }} PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN: ${{ vars.PAPERCLIP_AWS_AGENTCORE_CONTEXT_KMS_KEY_ARN }} PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION: ${{ vars.PAPERCLIP_AWS_AGENTCORE_QUALIFICATION_REVISION }} run: | set -euo pipefail mkdir -p cell-output/runs if [ "$ACCOUNTING_MODE" != controlled ] && [ "$CREDENTIAL_NAME" != "AWS_AGENTCORE_OIDC" ]; then test -n "${!CREDENTIAL_NAME:-}" fi if [ "$PROVIDER" = "claude_managed" ]; then test -n "$PAPERCLIP_CLAUDE_MANAGED_PROFILE_ID" test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_ID" test -n "$PAPERCLIP_CLAUDE_MANAGED_AGENT_VERSION" test -n "$PAPERCLIP_CLAUDE_MANAGED_ENVIRONMENT_ID" fi runner_cli=runner-protocol-build/extracted/portable/dist/cli/eval-session.js runner_package=runner-protocol-build/extracted/paperclip-runner.tgz runnerd=runner-protocol-build/extracted/paperclip-runnerd if [ "$ACCOUNTING_MODE" = controlled ]; then test -n "$BRAINTRUST_API_KEY" root="/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}" runner_cli="$root/runtime/controlled-evals/run.mjs" runner_package="$root/runner/paperclip-runner.tgz" runnerd="$root/runner/paperclip-runnerd" fi set +e python3 .paperclip-evals/evals/paperclip-runner/tools/run_live_roster.py run \ --roster ".paperclip-evals/evals/paperclip-runner/rosters/$ROSTER_FILE" \ --case "$CASE_ID" \ --runner-cli "$runner_cli" \ --runner-package "$runner_package" \ --runnerd "$runnerd" \ --runs-root cell-output/runs \ --summary-path cell-output/roster-summary.json \ --max-infrastructure-retries "$MAX_INFRASTRUCTURE_RETRIES" \ --run-id "gha-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${CELL_ID}" status=$? set -e CELL_EXIT_CODE="$status" node --input-type=module <<'NODE' import { readFileSync, writeFileSync } from "node:fs"; let authenticationMode; let authenticationEvidenceFailure; if (["XAI_API_KEY", "PAPERCLIP_ACPX_GROK_AUTH_JSON_SECRET"].includes(process.env.CREDENTIAL_NAME)) { const expected = process.env.CREDENTIAL_NAME === "XAI_API_KEY" ? "api_key" : "subscription"; try { const observed = JSON.parse(readFileSync("cell-output/roster-summary.json", "utf8")).authenticationMode; // Never copy arbitrary provider output into cell metadata or errors. if (["api_key", "subscription"].includes(observed)) authenticationMode = observed; if (authenticationMode !== expected) authenticationEvidenceFailure = "grok_authentication_evidence_mismatch"; } catch { authenticationEvidenceFailure = "grok_authentication_evidence_unreadable"; } } writeFileSync("cell-output/cell.json", `${JSON.stringify({ schema: "paperclip.runner-protocol-eval.cell/v1", cellId: process.env.CELL_ID, rosterFile: process.env.ROSTER_FILE, caseId: process.env.CASE_ID, ...(authenticationMode ? { authenticationMode } : {}), ...(authenticationEvidenceFailure ? { authenticationEvidenceFailure } : {}), accountingMode: process.env.ACCOUNTING_MODE, exitCode: Number(process.env.CELL_EXIT_CODE), }, null, 2)}\n`, { mode: 0o600 }); if (authenticationEvidenceFailure) throw new Error(authenticationEvidenceFailure); NODE exit "$status" - name: Retain controlled cell evidence in private storage if: always() && needs.authorize.outputs.accounting_mode == 'controlled' && steps.private_history_credentials.outcome == 'success' && steps.verify_controller.outcome == 'success' env: AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} AWS_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} CELL_ID: ${{ matrix.cellId }} run: | test -d cell-output node "/tmp/paperclip-runner-controlled-${GITHUB_RUN_ID}/runtime/private-history.mjs" upload "protocol-cell-$CELL_ID" "$GITHUB_RUN_ID" cell-output - name: Upload legacy cell attempt if: always() && needs.authorize.outputs.accounting_mode == 'legacy' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.cellId }} path: cell-output/ retention-days: 30 if-no-files-found: error eval_shard_1: name: Direct eval ${{ matrix.rosterId }} / ${{ matrix.caseId }} if: needs.catalog.outputs.matrix_1_present == 'true' needs: [authorize, catalog, build_runner, prepare_controller] runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: 18 permissions: contents: read id-token: write environment: name: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-paid' || 'runner-e2e-paid' }} strategy: fail-fast: false max-parallel: ${{ fromJSON(needs.catalog.outputs.max_parallel_per_shard) }} matrix: ${{ fromJSON(needs.catalog.outputs.matrix_1) }} steps: *direct_eval_steps report: name: Merge attempts and render canonical Evalbook if: always() && !cancelled() && needs.catalog.result == 'success' && needs.build_runner.result == 'success' && needs.prepare_controller.result == 'success' needs: [authorize, catalog, build_runner, prepare_controller, eval_shard_0, eval_shard_1] runs-on: ubuntu-latest timeout-minutes: 30 permissions: actions: read contents: read id-token: write environment: ${{ needs.authorize.outputs.accounting_mode == 'controlled' && 'runner-e2e-private-history' || '' }} outputs: public_report_ready: ${{ steps.public_report.outputs.ready }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: ref: ${{ github.sha }} persist-credentials: false - name: Generate private eval-repository token id: evals_token env: COMMITPERCLIP_KEY: ${{ secrets.COMMITPERCLIP_KEY }} GH_REPO: paperclipai/paperclip-evals run: | set -euo pipefail token="$(node .github/scripts/get-bot-token.mjs)" echo "::add-mask::$token" echo "value=$token" >> "$GITHUB_OUTPUT" - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: repository: paperclipai/paperclip-evals ref: ${{ needs.authorize.outputs.evals_sha }} path: .paperclip-evals token: ${{ steps.evals_token.outputs.value }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 env: NPM_CONFIG_AUDIT: "false" NPM_CONFIG_FUND: "false" NPM_CONFIG_UPDATE_NOTIFIER: "false" with: version: 9.15.4 - name: Download immutable campaign catalog uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-eval-catalog-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-catalog - name: Download portable runner and viewer uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-build-${{ needs.authorize.outputs.target_sha }}-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-build - name: Download the retained controller code if: needs.authorize.outputs.accounting_mode == 'controlled' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: artifact-ids: ${{ needs.prepare_controller.outputs.artifact_id }} path: ${{ runner.temp }}/runner-protocol-runtime - name: Verify retained controller bytes if: needs.authorize.outputs.accounting_mode == 'controlled' env: RUNTIME_RECEIPT_SHA256: ${{ needs.prepare_controller.outputs.runtime_receipt_sha256 }} run: | set -euo pipefail node packages/paperclip-runner/scripts/controlled-evals/verify-runtime.mjs "$RUNNER_TEMP/runner-protocol-runtime" "$GITHUB_SHA" "$RUNTIME_RECEIPT_SHA256" - name: Obtain private archive credentials for transfer steps only if: needs.authorize.outputs.accounting_mode == 'controlled' id: private_history_credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} output-env-credentials: false output-credentials: true disable-retry: true action-timeout-s: 60 - name: Restore private manifest and every available controlled cell if: needs.authorize.outputs.accounting_mode == 'controlled' env: AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} AWS_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} MANIFEST_SHA256: ${{ needs.prepare_controller.outputs.manifest_sha256 }} run: | set -euo pipefail archive="$RUNNER_TEMP/runner-protocol-runtime/private-history.mjs" control="$RUNNER_TEMP/runner-protocol-control" node "$archive" download protocol-control "$GITHUB_RUN_ID" "$control" report test "$(sha256sum "$control/controller-manifest.json" | cut -d ' ' -f 1)" = "$MANIFEST_SHA256" mkdir -p downloaded-runner-protocol-evals while IFS= read -r cell; do node "$archive" download "protocol-cell-$cell" "$GITHUB_RUN_ID" "downloaded-runner-protocol-evals/runner-protocol-eval-$GITHUB_RUN_ID-1-$cell" || echo 'A private cell is unavailable; its allocation remains held.' done < <(jq -r '.attempts[].executionId' "$control/controller-manifest.json") { echo "PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST=$control/controller-manifest.json" echo "PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI=$RUNNER_TEMP/runner-protocol-runtime/controlled-evals/archive.mjs" } >> "$GITHUB_ENV" - name: Download every legacy cell if: needs.authorize.outputs.accounting_mode == 'legacy' id: download_cells continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-* path: downloaded-runner-protocol-evals merge-multiple: false - name: Retry cell download after artifact transport failure if: needs.authorize.outputs.accounting_mode == 'legacy' && steps.download_cells.outcome == 'failure' continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: pattern: runner-protocol-eval-${{ github.run_id }}-${{ github.run_attempt }}-* path: downloaded-runner-protocol-evals merge-multiple: false - name: Materialize an empty download root when every cell failed early run: mkdir -p downloaded-runner-protocol-evals - name: Verify portable viewer run: | set -euo pipefail cd runner-protocol-build sha256sum --check runner-protocol-build.tar.gz.sha256 mkdir extracted tar --extract --gzip --file runner-protocol-build.tar.gz --directory extracted test -f extracted/dist-issue-thread/index.html - name: Aggregate every expected cell, including missing infrastructure cells env: PAPERCLIP_PROTOCOL_EVAL_SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} PAPERCLIP_PROTOCOL_EVAL_SOURCE_REF: ${{ needs.authorize.outputs.target_ref }} PAPERCLIP_PROTOCOL_EVALS_SHA: ${{ needs.authorize.outputs.evals_sha }} PAPERCLIP_PROTOCOL_EVAL_WORKFLOW_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs aggregate \ --catalog runner-protocol-catalog/runner-protocol-eval-catalog.json \ --downloads downloaded-runner-protocol-evals \ --evals-root .paperclip-evals \ --runs-out runner-protocol-merged/runs \ --campaign-out runner-protocol-merged/campaign.json - name: Render the access-controlled canonical Evalbook report run: | control_args=() if [ -n "${PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST:-}" ]; then control_args=(--controller-manifest "$PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST" --controller-archive-cli "$PAPERCLIP_RUNNER_EVAL_ARCHIVE_CLI") fi set +e python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report "${control_args[@]}" \ --runs-root runner-protocol-merged/runs \ --output runner-protocol-merged/report \ --viewer-root runner-protocol-build/extracted/dist-issue-thread \ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json > runner-protocol-merged/accounting-report.json status=$? set -e cp runner-protocol-merged/campaign.json runner-protocol-merged/report/campaign.json if [ -n "${PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST:-}" ]; then cp "$PAPERCLIP_RUNNER_EVAL_CONTROL_MANIFEST" runner-protocol-merged/controller-manifest.json fi exit "$status" - name: Render the same canonical grid from a public-safe evidence projection if: needs.authorize.outputs.accounting_mode == 'legacy' run: | node packages/paperclip-runner/scripts/runner-protocol-eval-campaign.mjs sanitize \ --runs-root runner-protocol-merged/runs \ --output runner-protocol-merged/public-runs python3 .paperclip-evals/evals/paperclip-runner/tools/eval_program.py report \ --runs-root runner-protocol-merged/public-runs \ --output runner-protocol-merged/public-report \ --viewer-root runner-protocol-build/extracted/dist-issue-thread \ --public-viewer \ --inventory .paperclip-evals/evals/paperclip-runner/inventory.json \ --coverage-matrix .paperclip-evals/evals/paperclip-runner/coverage-matrix.json cp runner-protocol-merged/campaign.json runner-protocol-merged/public-report/campaign.json - name: Set up report browser verification if: needs.authorize.outputs.accounting_mode == 'legacy' uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 with: version: 9.15.4 - name: Verify the actual chat viewer before publication if: needs.authorize.outputs.accounting_mode == 'legacy' run: | pnpm install --frozen-lockfile --ignore-scripts --store-dir "$RUNNER_TEMP/runner-report-pnpm-store" pnpm --filter @paperclipai/paperclip-runner exec playwright install --with-deps chromium node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root runner-protocol-merged/public-report --screenshots runner-protocol-merged/viewer-proof node packages/paperclip-runner/scripts/verify-runner-evalbook-viewer.mjs --report-root runner-protocol-merged/report - name: Enforce the static public allowlist if: needs.authorize.outputs.accounting_mode == 'legacy' id: public_report run: | node --input-type=module -e 'import { validatePublicProtocolEvalReport } from "./packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs"; await validatePublicProtocolEvalReport("runner-protocol-merged/public-report", { viewerRoot: "runner-protocol-build/extracted/dist-issue-thread" });' echo "ready=true" >> "$GITHUB_OUTPUT" - name: Add campaign result to the workflow summary if: always() run: | { echo '## Runner direct live protocol evals' echo jq -r '"- Cells: \(.totals.passed)/\(.totals.selected) passed\n- Behavior failures: \(.totals.behaviorFailures)\n- Infrastructure failures: \(.totals.infrastructureFailures)\n- Paperclip: `\(.source.paperclip.sha)`\n- Evals: `\(.source.evals.sha)`"' runner-protocol-merged/campaign.json } >> "$GITHUB_STEP_SUMMARY" - name: Retain controlled Evalbook and accounting in private storage if: always() && needs.authorize.outputs.accounting_mode == 'controlled' && steps.private_history_credentials.outcome == 'success' env: AWS_ACCESS_KEY_ID: ${{ steps.private_history_credentials.outputs.aws-access-key-id }} AWS_SECRET_ACCESS_KEY: ${{ steps.private_history_credentials.outputs.aws-secret-access-key }} AWS_SESSION_TOKEN: ${{ steps.private_history_credentials.outputs.aws-session-token }} AWS_REGION: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_AWS_REGION }} RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET: ${{ vars.RUNNER_E2E_PRIVATE_HISTORY_S3_BUCKET }} run: | test -d runner-protocol-merged node "$RUNNER_TEMP/runner-protocol-runtime/private-history.mjs" upload protocol-report "$GITHUB_RUN_ID" runner-protocol-merged - name: Upload legacy canonical Evalbook and raw attempts if: always() && needs.authorize.outputs.accounting_mode == 'legacy' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-eval-report-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-merged/ retention-days: 30 if-no-files-found: error - name: Upload publisher-only sanitized Evalbook if: needs.authorize.outputs.accounting_mode == 'legacy' && steps.public_report.outputs.ready == 'true' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-merged/public-report/ retention-days: 1 if-no-files-found: error - name: Enforce complete green campaign if: always() run: jq -e '.complete == true and .allPassed == true' runner-protocol-merged/campaign.json >/dev/null publish_history: name: Publish immutable Evalbook and mutable campaign index needs: [authorize, catalog, report] if: always() && needs.report.outputs.public_report_ready == 'true' runs-on: ubuntu-latest timeout-minutes: 15 concurrency: group: runner-protocol-eval-history-publish cancel-in-progress: false permissions: contents: read id-token: write environment: name: runner-e2e-history url: ${{ steps.publish.outputs.report_url }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: # AWS credentials can execute only the publisher from the trusted workflow revision. ref: ${{ github.sha }} persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: node-version: 24 - name: Download only the sanitized canonical Evalbook uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-eval-public-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-public-report - name: Download the same-run canonical viewer for byte verification uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-protocol-viewer-${{ github.run_id }}-${{ github.run_attempt }} path: runner-protocol-trusted-viewer - name: Exchange GitHub OIDC identity for scoped AWS credentials uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6 with: role-to-assume: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_ROLE_ARN || vars.RUNNER_E2E_HISTORY_AWS_ROLE_ARN }} aws-region: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_AWS_REGION || vars.RUNNER_E2E_HISTORY_AWS_REGION }} - name: Publish versioned report and refresh the root index id: publish env: PAPERCLIP_RUNNER_PROTOCOL_EVAL_PUBLIC_REPORT_DIR: ${{ github.workspace }}/runner-protocol-public-report PAPERCLIP_RUNNER_PROTOCOL_EVAL_VIEWER_DIR: ${{ github.workspace }}/runner-protocol-trusted-viewer RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_S3_BUCKET || vars.RUNNER_E2E_HISTORY_S3_BUCKET }} RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PREFIX || 'runner-protocol-evals' }} RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL: ${{ vars.RUNNER_PROTOCOL_EVAL_HISTORY_PUBLIC_BASE_URL || vars.RUNNER_E2E_HISTORY_PUBLIC_BASE_URL }} run: node packages/paperclip-runner/scripts/publish-runner-protocol-eval-history.mjs