diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index 6d9e3d7ec0c..a9703feb747 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -4,6 +4,7 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; +import vm from "node:vm"; const workflow = readFileSync(new URL("../../workflows/ci-routing.yml", import.meta.url), "utf8"); const jobs = workflow.split(/^jobs:\n/m)[1]; @@ -12,9 +13,10 @@ test("routing uses reserved capacity and runs no PR code or actions", () => { assert.deepEqual([...jobs.matchAll(/^ {2}([a-z_]+):$/gm)].map((match) => match[1]), ["gate"]); const placement = jobs.match(/^ {4}runs-on: (.+)$/m)?.[1]; assert.ok(placement, "routing placement must be defined"); - assert.ok(placement.includes("vars.AWS_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064'")); - assert.ok(placement.includes("github.event_name == 'pull_request' && contains(fromJSON(vars.AWS_CI_TRUSTED_USER_IDS || '[]'), github.event.pull_request.user.id) && contains(fromJSON(vars.AWS_CI_TRUSTED_USER_IDS || '[]'), github.event.sender.id)")); - assert.ok(placement.includes("github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(fromJSON(vars.AWS_CI_TRUSTED_USER_IDS || '[]'), fromJSON(github.actor_id))")); + assert.doesNotMatch(placement, /fromJSON/); + assert.ok(placement.includes("vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE")); + assert.ok(placement.includes("github.event.pull_request.user.id")); + assert.ok(placement.includes("github.event.sender.id")); assert.ok(placement.endsWith("&& 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }}")); assert.doesNotMatch(jobs, /uses:|actions\/checkout|secrets\./); assert.doesNotMatch(workflow, /^ {2}(?:actions|contents|pull-requests): write$/m); @@ -140,3 +142,53 @@ for (const [name, stack, base, expected] of scopeCases) { }).full_ci, expected); }); } + + +function routingPlacement(vars = {}, github = {}) { + const source = "[34892728]"; + const expression = jobs.match(/^ {4}runs-on: \$\{\{ (.+) \}\}$/m)?.[1]; + assert.ok(expression, "routing expression is missing"); + return vm.runInNewContext(expression, { + vars: { + AWS_CI_ENABLED: "true", + AWS_CI_TRUSTED_USER_IDS: source, + AWS_CI_ROUTING_USER_IDS_SOURCE: source, + AWS_CI_ROUTING_USER_IDS: "|34892728|", + ...vars, + }, + github: { + repository: "paperclipai/paperclip", + repository_id: "1170821064", + event_name: "pull_request", + ref: "refs/heads/master", + actor_id: "34892728", + event: { pull_request: { user: { id: 34892728 } }, sender: { id: 34892728 } }, + ...github, + }, + contains: (search, item) => String(search).toLowerCase().includes(String(item).toLowerCase()), + format: (template, ...args) => template.replace(/\{(\d+)\}/g, (_, index) => String(args[Number(index)])), + }, { timeout: 1000 }); +} + +const placementCases = [ + ["approved PR", {}, {}, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], + ["unapproved author", {}, { event: { pull_request: { user: { id: 999 } }, sender: { id: 34892728 } } }], + ["unapproved sender", {}, { event: { pull_request: { user: { id: 34892728 } }, sender: { id: 999 } } }], + ["partial ID cannot match", {}, { event: { pull_request: { user: { id: 4892728 } }, sender: { id: 34892728 } } }], + ["invalid JSON keeps hosted fallback", { AWS_CI_TRUSTED_USER_IDS: "[34892728,]" }, {}], + ["stale snapshot", { AWS_CI_TRUSTED_USER_IDS: "[34892728,999]" }, {}], + ["missing snapshot", { AWS_CI_ROUTING_USER_IDS_SOURCE: "" }, {}], + ["missing admission members", { AWS_CI_ROUTING_USER_IDS: "" }, {}], + ["disabled routing", { AWS_CI_ENABLED: "false" }, {}], + ["different repository", {}, { repository: "other/repo" }], + ["different repository ID", {}, { repository_id: "999" }], + ["approved manual master canary", {}, { event_name: "workflow_dispatch" }, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], + ["unapproved manual actor", {}, { event_name: "workflow_dispatch", actor_id: "999" }], + ["manual branch canary", {}, { event_name: "workflow_dispatch", ref: "refs/heads/other" }], +]; + +for (const [name, vars, github, expected = "ubuntu-latest"] of placementCases) { + test(`placement before allocation: ${name}`, () => { + assert.equal(routingPlacement(vars, github), expected); + }); +} diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index 8baba75a70e..5ad3e1e11e3 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -17,7 +17,7 @@ permissions: jobs: gate: name: Select trusted runner - runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(fromJSON(vars.AWS_CI_TRUSTED_USER_IDS || '[]'), github.event.pull_request.user.id) && contains(fromJSON(vars.AWS_CI_TRUSTED_USER_IDS || '[]'), github.event.sender.id)) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(fromJSON(vars.AWS_CI_TRUSTED_USER_IDS || '[]'), fromJSON(github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && vars.AWS_CI_ROUTING_USER_IDS_SOURCE != '' && vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.pull_request.user.id)) && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.sender.id))) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: runner: ${{ steps.route.outputs.runner }} diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 4eedd722bb1..065eea917bf 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -39,7 +39,11 @@ Runner selection calls `ci-routing.yml@master` on the dedicated `paperclip-ci-routing-x64` AWS Fleet. The `paperclip-ci-routing` runner group admits only `paperclipai/paperclip/.github/workflows/ci-routing.yml@refs/heads/master` and repository ID `1170821064`. GitHub evaluates the canonical repository, author, and sender checks before -allocating reserved capacity. Unapproved PRs use hosted metadata routing; +allocating reserved capacity. The validated admission snapshot (`AWS_CI_ROUTING_USER_IDS_SOURCE` and +`AWS_CI_ROUTING_USER_IDS`) must match `AWS_CI_TRUSTED_USER_IDS`. Refresh it through +the runner-group configuration helper when the roster changes. Missing, stale, +or malformed settings keep hosted routing available without evaluating JSON +in the job's placement expression. Unapproved PRs use hosted metadata routing; approved manual canaries on master can use the reserved pool. The routing workflow reads GitHub metadata without checking out or executing PR code. Its author, sender, rerun actor, repository, head, and merge validation still decide whether tests can use AWS.