diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index fedc277482a..a9703feb747 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -4,13 +4,20 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; +import vm from "node:vm"; const workflow = readFileSync(new URL("../../workflows/ci-routing.yml", import.meta.url), "utf8"); const jobs = workflow.split(/^jobs:\n/m)[1]; test("routing uses reserved capacity and runs no PR code or actions", () => { assert.deepEqual([...jobs.matchAll(/^ {2}([a-z_]+):$/gm)].map((match) => match[1]), ["gate"]); - assert.match(jobs, /^ {4}runs-on: runs-on\/fleet=paperclip-ci-routing-x64\/env=public-ci$/m); + const placement = jobs.match(/^ {4}runs-on: (.+)$/m)?.[1]; + assert.ok(placement, "routing placement must be defined"); + assert.doesNotMatch(placement, /fromJSON/); + assert.ok(placement.includes("vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE")); + assert.ok(placement.includes("github.event.pull_request.user.id")); + assert.ok(placement.includes("github.event.sender.id")); + assert.ok(placement.endsWith("&& 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }}")); assert.doesNotMatch(jobs, /uses:|actions\/checkout|secrets\./); assert.doesNotMatch(workflow, /^ {2}(?:actions|contents|pull-requests): write$/m); }); @@ -135,3 +142,53 @@ for (const [name, stack, base, expected] of scopeCases) { }).full_ci, expected); }); } + + +function routingPlacement(vars = {}, github = {}) { + const source = "[34892728]"; + const expression = jobs.match(/^ {4}runs-on: \$\{\{ (.+) \}\}$/m)?.[1]; + assert.ok(expression, "routing expression is missing"); + return vm.runInNewContext(expression, { + vars: { + AWS_CI_ENABLED: "true", + AWS_CI_TRUSTED_USER_IDS: source, + AWS_CI_ROUTING_USER_IDS_SOURCE: source, + AWS_CI_ROUTING_USER_IDS: "|34892728|", + ...vars, + }, + github: { + repository: "paperclipai/paperclip", + repository_id: "1170821064", + event_name: "pull_request", + ref: "refs/heads/master", + actor_id: "34892728", + event: { pull_request: { user: { id: 34892728 } }, sender: { id: 34892728 } }, + ...github, + }, + contains: (search, item) => String(search).toLowerCase().includes(String(item).toLowerCase()), + format: (template, ...args) => template.replace(/\{(\d+)\}/g, (_, index) => String(args[Number(index)])), + }, { timeout: 1000 }); +} + +const placementCases = [ + ["approved PR", {}, {}, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], + ["unapproved author", {}, { event: { pull_request: { user: { id: 999 } }, sender: { id: 34892728 } } }], + ["unapproved sender", {}, { event: { pull_request: { user: { id: 34892728 } }, sender: { id: 999 } } }], + ["partial ID cannot match", {}, { event: { pull_request: { user: { id: 4892728 } }, sender: { id: 34892728 } } }], + ["invalid JSON keeps hosted fallback", { AWS_CI_TRUSTED_USER_IDS: "[34892728,]" }, {}], + ["stale snapshot", { AWS_CI_TRUSTED_USER_IDS: "[34892728,999]" }, {}], + ["missing snapshot", { AWS_CI_ROUTING_USER_IDS_SOURCE: "" }, {}], + ["missing admission members", { AWS_CI_ROUTING_USER_IDS: "" }, {}], + ["disabled routing", { AWS_CI_ENABLED: "false" }, {}], + ["different repository", {}, { repository: "other/repo" }], + ["different repository ID", {}, { repository_id: "999" }], + ["approved manual master canary", {}, { event_name: "workflow_dispatch" }, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], + ["unapproved manual actor", {}, { event_name: "workflow_dispatch", actor_id: "999" }], + ["manual branch canary", {}, { event_name: "workflow_dispatch", ref: "refs/heads/other" }], +]; + +for (const [name, vars, github, expected = "ubuntu-latest"] of placementCases) { + test(`placement before allocation: ${name}`, () => { + assert.equal(routingPlacement(vars, github), expected); + }); +} diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index b8d2e93fa5c..5ad3e1e11e3 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -17,7 +17,7 @@ permissions: jobs: gate: name: Select trusted runner - runs-on: runs-on/fleet=paperclip-ci-routing-x64/env=public-ci + runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && vars.AWS_CI_ROUTING_USER_IDS_SOURCE != '' && vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.pull_request.user.id)) && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.sender.id))) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: runner: ${{ steps.route.outputs.runner }} diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index ec62707c417..65f72fa8c56 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -17,241 +17,7 @@ concurrency: jobs: gate: - name: Select trusted runner - runs-on: ubuntu-latest - timeout-minutes: 5 - outputs: - runner: ${{ steps.route.outputs.runner }} - full_ci: ${{ steps.scope.outputs.full_ci }} - - steps: - - name: Validate PR identity and select runner - id: route - shell: bash - env: - GH_TOKEN: ${{ github.token }} - AWS_CI_ENABLED: ${{ vars.AWS_CI_ENABLED }} - TRUSTED_USER_IDS: ${{ vars.AWS_CI_TRUSTED_USER_IDS }} - EVENT_NAME: ${{ github.event_name }} - EVENT_REPOSITORY: ${{ github.repository }} - EVENT_REPOSITORY_ID: ${{ github.repository_id }} - EVENT_ACTION: ${{ github.event.action }} - EVENT_PR_NUMBER: ${{ github.event.pull_request.number }} - EVENT_PR_AUTHOR_ID: ${{ github.event.pull_request.user.id }} - EVENT_SENDER_ID: ${{ github.event.sender.id }} - EVENT_BASE_REPOSITORY_ID: ${{ github.event.pull_request.base.repo.id }} - EVENT_BASE_REF: ${{ github.event.pull_request.base.ref }} - EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} - EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - EVENT_MERGE_SHA: ${{ github.sha }} - RUN_ID: ${{ github.run_id }} - run: | - set -u - - github_runner='ubuntu-latest' - aws_runner='runs-on/fleet=paperclip-public-pr-x64/env=public-ci' - - fail_closed() { - echo "runner=$github_runner" >> "$GITHUB_OUTPUT" - echo "::notice title=AWS CI routing::Using GitHub-hosted runner: $1" - exit 0 - } - - is_positive_integer() { - [[ "$1" =~ ^[1-9][0-9]*$ ]] - } - - is_commit_sha() { - [[ "$1" =~ ^[0-9a-f]{40}$ ]] - } - - is_allowed() { - local user_id="$1" - jq -e --argjson user_id "$user_id" 'index($user_id) != null' \ - <<< "$TRUSTED_USER_IDS" >/dev/null - } - - [[ "$AWS_CI_ENABLED" == 'true' ]] || fail_closed 'AWS_CI_ENABLED is not true' - # Reusable workflows retain the caller's github context and event - # payload, so a pull_request caller must still report pull_request. - [[ "$EVENT_NAME" == 'pull_request' ]] || fail_closed 'event is not pull_request' - [[ "$EVENT_REPOSITORY" == 'paperclipai/paperclip' ]] || fail_closed 'unexpected repository' - [[ "$EVENT_REPOSITORY_ID" == '1170821064' ]] || fail_closed 'unexpected repository ID' - [[ "$EVENT_BASE_REPOSITORY_ID" == '1170821064' ]] || fail_closed 'unexpected base repository ID' - [[ -n "$EVENT_BASE_REF" ]] || fail_closed 'missing base branch' - [[ "$EVENT_ACTION" =~ ^(opened|reopened|synchronize)$ ]] || fail_closed 'unsupported pull_request action' - - jq -e ' - type == "array" and - length > 0 and - all(.[]; type == "number" and . > 0 and floor == .) - ' <<< "$TRUSTED_USER_IDS" >/dev/null 2>&1 || fail_closed 'trusted user ID list is malformed' - - for user_id in "$EVENT_PR_AUTHOR_ID" "$EVENT_SENDER_ID"; do - is_positive_integer "$user_id" || fail_closed 'event contains a malformed user ID' - is_allowed "$user_id" || fail_closed "GitHub user ID $user_id is not allowlisted" - done - - for commit_sha in "$EVENT_BASE_SHA" "$EVENT_HEAD_SHA" "$EVENT_MERGE_SHA"; do - is_commit_sha "$commit_sha" || fail_closed 'event contains a malformed commit SHA' - done - - pr_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/pulls/$EVENT_PR_NUMBER" 2>/dev/null)" \ - || fail_closed 'could not refresh pull request state' - - jq -e \ - --argjson repository_id 1170821064 \ - --argjson author_id "$EVENT_PR_AUTHOR_ID" \ - --arg base_ref "$EVENT_BASE_REF" \ - --arg head_sha "$EVENT_HEAD_SHA" \ - ' - .state == "open" and - .user.id == $author_id and - .base.repo.id == $repository_id and - .base.ref == $base_ref and - .head.sha == $head_sha - ' <<< "$pr_json" >/dev/null 2>&1 \ - || fail_closed 'current pull request state does not match the triggering event' - - live_merge_sha="$(jq -r '.merge_commit_sha // empty' <<< "$pr_json")" - is_commit_sha "$live_merge_sha" || fail_closed 'current pull request has no valid merge SHA' - - base_ref_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/git/ref/heads/$EVENT_BASE_REF" 2>/dev/null)" \ - || fail_closed 'could not inspect the current base branch' - live_base_ref_sha="$(jq -r '.object.sha // empty' <<< "$base_ref_json")" - is_commit_sha "$live_base_ref_sha" || fail_closed 'current base branch has no valid commit SHA' - - base_comparison="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$live_base_ref_sha" 2>/dev/null)" \ - || fail_closed 'could not compare the triggering and current base branches' - jq -e \ - --arg event_base_sha "$EVENT_BASE_SHA" ' - .merge_base_commit.sha == $event_base_sha and - (.status == "ahead" or .status == "identical") - ' <<< "$base_comparison" >/dev/null 2>&1 \ - || fail_closed 'current base branch does not descend from the triggering base snapshot' - - event_merge_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/git/commits/$EVENT_MERGE_SHA" 2>/dev/null)" \ - || fail_closed 'could not inspect the event merge commit' - - live_merge_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/git/commits/$live_merge_sha" 2>/dev/null)" \ - || fail_closed 'could not inspect the current merge commit' - - validate_merge_commit() { - local merge_json="$1" - local expected_sha="$2" - jq -e \ - --arg expected_sha "$expected_sha" \ - --arg head_sha "$EVENT_HEAD_SHA" ' - .sha == $expected_sha and - (.parents | length) == 2 and - .parents[1].sha == $head_sha and - (.parents[0].sha | test("^[0-9a-f]{40}$")) and - (.tree.sha | test("^[0-9a-f]{40}$")) - ' <<< "$merge_json" >/dev/null 2>&1 - } - - validate_merge_commit "$event_merge_json" "$EVENT_MERGE_SHA" \ - || fail_closed 'event merge commit does not match the current base and head' - validate_merge_commit "$live_merge_json" "$live_merge_sha" \ - || fail_closed 'current merge commit does not match the triggering base and head' - - event_merge_parent="$(jq -r '.parents[0].sha' <<< "$event_merge_json")" - live_merge_parent="$(jq -r '.parents[0].sha' <<< "$live_merge_json")" - [[ "$event_merge_parent" == "$live_merge_parent" ]] \ - || fail_closed 'current merge commit uses a different base merge parent' - - if [[ "$event_merge_parent" != "$live_base_ref_sha" ]]; then - base_merge_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/git/commits/$event_merge_parent" 2>/dev/null)" \ - || fail_closed 'could not inspect the stacked base merge commit' - jq -e \ - --arg expected_sha "$event_merge_parent" \ - --arg live_base_ref_sha "$live_base_ref_sha" ' - .sha == $expected_sha and - (.parents | length) == 2 and - any(.parents[]; .sha == $live_base_ref_sha) and - (.tree.sha | test("^[0-9a-f]{40}$")) - ' <<< "$base_merge_json" >/dev/null 2>&1 \ - || fail_closed 'stacked base merge commit does not contain the current base branch' - fi - - event_merge_tree="$(jq -r '.tree.sha' <<< "$event_merge_json")" - live_merge_tree="$(jq -r '.tree.sha' <<< "$live_merge_json")" - [[ "$event_merge_tree" == "$live_merge_tree" ]] \ - || fail_closed 'current merge tree differs from the triggering merge tree' - - run_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/actions/runs/$RUN_ID" 2>/dev/null)" \ - || fail_closed 'could not refresh workflow run state' - - triggering_actor_id="$(jq -r '.triggering_actor.id // empty' <<< "$run_json")" - is_positive_integer "$triggering_actor_id" || fail_closed 'workflow run has no valid triggering actor ID' - is_allowed "$triggering_actor_id" || fail_closed "triggering GitHub user ID $triggering_actor_id is not allowlisted" - - jq -e \ - --argjson repository_id 1170821064 \ - --arg event_name "$EVENT_NAME" ' - .repository.id == $repository_id and - .event == $event_name - ' <<< "$run_json" >/dev/null 2>&1 \ - || fail_closed 'current workflow run does not match the expected repository event' - - echo "runner=$aws_runner" >> "$GITHUB_OUTPUT" - echo '::notice title=AWS CI routing::Using an ephemeral RunsOn Fleet runner' - - - name: Select stacked PR CI scope - id: scope - shell: bash - env: - STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }} - PR_BASE_REF: ${{ github.event.pull_request.base.ref }} - run: | - set -euo pipefail - - full_ci='true' - reason='ordinary pull request' - - if jq -e 'type == "object"' <<< "$STACK_JSON" >/dev/null 2>&1; then - stack_position="$(jq -r '.position // empty' <<< "$STACK_JSON")" - stack_size="$(jq -r '.size // empty' <<< "$STACK_JSON")" - stack_base_ref="$(jq -r '.base.ref // empty' <<< "$STACK_JSON")" - - if [[ ! "$stack_position" =~ ^[1-9][0-9]*$ ]] || - [[ ! "$stack_size" =~ ^[1-9][0-9]*$ ]] || - (( stack_position > stack_size )) || - [[ -z "$stack_base_ref" ]]; then - reason='malformed stack metadata; defaulting to full CI' - elif (( stack_position == stack_size )); then - reason='top pull request in stack' - elif [[ "$stack_base_ref" == "$PR_BASE_REF" ]]; then - reason='lowest unmerged pull request in stack' - else - full_ci='false' - reason='middle pull request in stack' - fi - fi - - echo "full_ci=$full_ci" >> "$GITHUB_OUTPUT" - echo "::notice title=Stacked PR CI scope::$reason; full_ci=$full_ci" + uses: paperclipai/paperclip/.github/workflows/ci-routing.yml@master policy: needs: [gate] diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 575b1430fa8..065eea917bf 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -35,6 +35,28 @@ New workflow versions merged into master then receive runner access without a separate SHA allowlist update. Dependabot leaves this first-party reference on master. +Runner selection calls `ci-routing.yml@master` on the dedicated +`paperclip-ci-routing-x64` AWS Fleet. The `paperclip-ci-routing` runner group +admits only `paperclipai/paperclip/.github/workflows/ci-routing.yml@refs/heads/master` +and repository ID `1170821064`. GitHub evaluates the canonical repository, author, and sender checks before +allocating reserved capacity. The validated admission snapshot (`AWS_CI_ROUTING_USER_IDS_SOURCE` and +`AWS_CI_ROUTING_USER_IDS`) must match `AWS_CI_TRUSTED_USER_IDS`. Refresh it through +the runner-group configuration helper when the roster changes. Missing, stale, +or malformed settings keep hosted routing available without evaluating JSON +in the job's placement expression. Unapproved PRs use hosted metadata routing; +approved manual canaries on master can use the reserved pool. The routing +workflow reads GitHub metadata without checking out or executing PR code. Its author, sender, rerun actor, +repository, head, and merge validation still decide whether tests can use AWS. +Unapproved test workloads continue to use GitHub-hosted runners. + +The routing Fleet has two warm On-Demand runners and a four-runner ceiling. +Test fleets cannot consume its runner slots. Hosted test queues therefore do +not block AWS runner selection. A manual dispatch of `ci-routing.yml` on master +checks placement without launching the test matrix; its non-PR event returns +the hosted test-runner output. Configure and verify the group and Fleet before +merging this caller. To roll back, restore the hosted gate and retain the routing +workflow authorization until assigned jobs finish. + Keep the `.github/**` rule in `.github/CODEOWNERS` and the active master ruleset's code-owner review requirement enabled. This covers the caller, the trusted workflow, and CODEOWNERS itself. Existing administrator pull-request bypasses diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index ddeca9c1e3d..28bf3926f3c 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -16,6 +16,7 @@ const playwrightConfig = path.join(repoRoot, "tests", "e2e", "playwright.config. const prCallerWorkflow = path.join(repoRoot, ".github", "workflows", "pr.yml"); const trustedPrWorkflowPath = ".github/workflows/pr-trusted.yml"; const trustedPrWorkflow = path.join(repoRoot, trustedPrWorkflowPath); +const routingWorkflow = path.join(repoRoot, ".github", "workflows", "ci-routing.yml"); const SHARD_COUNT = 8; @@ -54,11 +55,11 @@ function readWorkflowJobs(workflow) { } function runStackScope(stack, prBaseRef) { - const workflow = readFileSync(trustedPrWorkflow, "utf8"); + const workflow = readFileSync(routingWorkflow, "utf8"); const match = workflow.match( - / - name: Select stacked PR CI scope[\s\S]*? run: \|\n([\s\S]*?)\n\n policy:/, + / - name: Select stacked PR CI scope[\s\S]*? run: \|\n([\s\S]*?)\s*$/, ); - assert.ok(match, "trusted workflow must define the stacked PR scope script"); + assert.ok(match, "routing workflow must define the stacked PR scope script"); const script = match[1] .split("\n") .map((line) => line.replace(/^ {10}/, "")) @@ -207,10 +208,22 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the } }); +test("runner selection has reserved AWS capacity and executes metadata checks only", () => { + const workflow = readFileSync(routingWorkflow, "utf8"); + const jobs = readWorkflowJobs(workflow.split(/^jobs:\n/m)[1]); + assert.deepEqual([...jobs.keys()], ["gate"]); + assert.match(jobs.get("gate"), /^ {4}runs-on: \$\{\{ vars\.AWS_CI_ENABLED[\s\S]*paperclip-ci-routing-x64/m); + assert.doesNotMatch(jobs.get("gate"), /uses:|actions\/checkout|secrets\./); + assert.match(workflow, /^ {6}runner:\n {8}value: \$\{\{ jobs\.gate\.outputs\.runner \}\}$/m); + assert.match(workflow, /^ {6}full_ci:\n {8}value: \$\{\{ jobs\.gate\.outputs\.full_ci \}\}$/m); + assert.doesNotMatch(workflow, /^ {2}(?:actions|contents|pull-requests): write$/m); +}); + test("the trusted PR workflow limits full CI to merge-relevant stack layers", () => { const workflow = readFileSync(trustedPrWorkflow, "utf8"); const jobs = readWorkflowJobs(workflow); - const gate = jobs.get("gate"); + assert.match(jobs.get("gate"), /uses: paperclipai\/paperclip\/\.github\/workflows\/ci-routing\.yml@master/); + const gate = readWorkflowJobs(readFileSync(routingWorkflow, "utf8")).get("gate"); assert.match(gate, /^ {6}full_ci: \$\{\{ steps\.scope\.outputs\.full_ci \}\}$/m); assert.match(gate, /STACK_JSON: \$\{\{ toJSON\(github\.event\.pull_request\.stack\) \}\}/);