diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index 15fb8b2d0c2..ae43a65791f 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -4,20 +4,15 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; -import vm from "node:vm"; const workflow = readFileSync(new URL("../../workflows/ci-routing.yml", import.meta.url), "utf8"); const jobs = workflow.split(/^jobs:\n/m)[1]; -test("routing uses reserved capacity and runs no PR code or actions", () => { +test("authorization uses hosted capacity and runs no PR code or actions", () => { assert.deepEqual([...jobs.matchAll(/^ {2}([a-z_]+):$/gm)].map((match) => match[1]), ["gate"]); const placement = jobs.match(/^ {4}runs-on: (.+)$/m)?.[1]; assert.ok(placement, "routing placement must be defined"); - assert.doesNotMatch(placement, /fromJSON/); - assert.ok(placement.includes("vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE")); - assert.ok(placement.includes("github.event.pull_request.user.id")); - assert.ok(placement.includes("github.event.sender.id")); - assert.ok(placement.endsWith("&& 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }}")); + assert.equal(placement, "ubuntu-latest", "verify current owners before allocating any AWS runner"); assert.doesNotMatch(jobs, /uses:|actions\/checkout|secrets\./); assert.doesNotMatch(workflow, /^ {2}(?:actions|contents|pull-requests): write$/m); }); @@ -29,7 +24,7 @@ test("routing exports both placement and stack scope to callers", () => { assert.match(jobs, /^ {6}full_ci: \$\{\{ steps\.scope\.outputs\.full_ci \}\}$/m); }); -const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"${*: -1}\"\ncase \"$endpoint\" in\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status,\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; +const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"\"\nquery=\"\"\nfor arg in \"$@\"; do\n case \"$arg\" in /repos/*|/users/*) endpoint=\"$arg\" ;; esac\ndone\nif [[ \"$*\" == *\"--jq\"* ]]; then query=\"${*: -1}\"; fi\ncase \"$endpoint\" in\n */contents/scripts/ci-profile/authorize.mjs?ref=*)\n [[ -z \"${TEST_OWNER_HELPER_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n base64 < \"$TEST_OWNER_HELPER\"\n ;;\n */contents/.github/CODEOWNERS?ref=*)\n [[ -z \"${TEST_CODEOWNERS_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n jq -cn --arg content \"$TEST_CODEOWNERS_BASE64\" '{encoding: \"base64\", content: $content}'\n ;;\n /users/ci-owner)\n [[ -z \"${TEST_OWNER_LOOKUP_FAIL:-}\" ]] || exit 1\n jq -cn --argjson id \"${TEST_CI_OWNER_ID:-34892728}\" --arg type \"${TEST_CI_OWNER_TYPE:-User}\" '{id: $id, type: $type}'\n ;;\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n if [[ \"$query\" == \".object.sha\" ]]; then\n [[ -z \"${TEST_MASTER_FAIL:-}\" ]] || exit 1\n echo \"${TEST_TRUSTED_MASTER_SHA:-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}\"\n exit 0\n fi\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status,\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*/attempts/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --argjson run_attempt \"${TEST_RUN_ATTEMPT:-1}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n run_attempt: $run_attempt,\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; function stepScript(name) { const start = workflow.indexOf(` - name: ${name}\n`); @@ -54,6 +49,9 @@ function executeStep(name, overrides = {}) { ...process.env, PATH: `${directory}:${process.env.PATH}`, GITHUB_OUTPUT: output, + RUNNER_TEMP: directory, + TEST_OWNER_HELPER: path.resolve("scripts/ci-profile/authorize.mjs"), + TEST_CODEOWNERS_BASE64: Buffer.from(".github/** @ci-owner\n").toString("base64"), AWS_CI_ENABLED: "true", TRUSTED_USER_IDS: "[34892728]", EVENT_NAME: "pull_request", @@ -69,6 +67,7 @@ function executeStep(name, overrides = {}) { EVENT_HEAD_SHA: "1111111111111111111111111111111111111111", EVENT_MERGE_SHA: "2222222222222222222222222222222222222222", RUN_ID: "123456789", + RUN_ATTEMPT: "1", ...overrides, }, }); @@ -106,6 +105,22 @@ const routeCases = [ ["unlisted author", { EVENT_PR_AUTHOR_ID: "99999999" }], ["unlisted sender", { EVENT_SENDER_ID: "99999999" }], ["unlisted rerun actor", { TEST_TRIGGERING_ACTOR_ID: "99999999" }], + ["allowlisted user removed from CODEOWNERS", { TEST_CI_OWNER_ID: "99999999" }], + ["CODEOWNER not on numeric allowlist", { TRUSTED_USER_IDS: "[99999999]" }], + ["non-owner PR with owner rerun", { EVENT_PR_AUTHOR_ID: "99999999", TEST_PR_AUTHOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ["non-owner event sender", { EVENT_SENDER_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ["non-owner rerun actor", { TEST_TRIGGERING_ACTOR_ID: "99999999", TRUSTED_USER_IDS: "[34892728,99999999]" }], + ["CODEOWNERS unavailable", { TEST_CODEOWNERS_FAIL: "1" }], + ["owner identity unavailable", { TEST_OWNER_LOOKUP_FAIL: "1" }], + ["owner is not a person", { TEST_CI_OWNER_TYPE: "Bot" }], + ["trusted helper unavailable", { TEST_OWNER_HELPER_FAIL: "1" }], + ["trusted master unavailable", { TEST_MASTER_FAIL: "1" }], + ["invalid trusted master", { TEST_TRUSTED_MASTER_SHA: "bad" }], + ["documentation owners do not authorize CI", { TEST_CODEOWNERS_BASE64: Buffer.from("doc/** @ci-owner\n").toString("base64") }], + ["ambiguous CODEOWNERS deny access", { TEST_CODEOWNERS_BASE64: Buffer.from(".github/** @ci-owner\n/.github/** @ci-owner\n").toString("base64") }], + ["malformed run attempt", { RUN_ATTEMPT: "bad" }], + ["wrong run attempt", { TEST_RUN_ATTEMPT: "2" }], + ["owner rerun validates its own attempt", { RUN_ATTEMPT: "2", TEST_RUN_ATTEMPT: "2" }, awsRunner], ["missing base", { EVENT_BASE_REF: "" }], ["wrong base repository", { EVENT_BASE_REPOSITORY_ID: "999999999" }], ["replaced base ancestry", { TEST_BASE_COMPARISON_MERGE_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }], @@ -226,53 +241,3 @@ test("path scope skips full CI for a single-PR stack that only touches release n TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), }).full_ci, "false"); }); - - -function routingPlacement(vars = {}, github = {}) { - const source = "[34892728]"; - const expression = jobs.match(/^ {4}runs-on: \$\{\{ (.+) \}\}$/m)?.[1]; - assert.ok(expression, "routing expression is missing"); - return vm.runInNewContext(expression, { - vars: { - AWS_CI_ENABLED: "true", - AWS_CI_TRUSTED_USER_IDS: source, - AWS_CI_ROUTING_USER_IDS_SOURCE: source, - AWS_CI_ROUTING_USER_IDS: "|34892728|", - ...vars, - }, - github: { - repository: "paperclipai/paperclip", - repository_id: "1170821064", - event_name: "pull_request", - ref: "refs/heads/master", - actor_id: "34892728", - event: { pull_request: { user: { id: 34892728 } }, sender: { id: 34892728 } }, - ...github, - }, - contains: (search, item) => String(search).toLowerCase().includes(String(item).toLowerCase()), - format: (template, ...args) => template.replace(/\{(\d+)\}/g, (_, index) => String(args[Number(index)])), - }, { timeout: 1000 }); -} - -const placementCases = [ - ["approved PR", {}, {}, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], - ["unapproved author", {}, { event: { pull_request: { user: { id: 999 } }, sender: { id: 34892728 } } }], - ["unapproved sender", {}, { event: { pull_request: { user: { id: 34892728 } }, sender: { id: 999 } } }], - ["partial ID cannot match", {}, { event: { pull_request: { user: { id: 4892728 } }, sender: { id: 34892728 } } }], - ["invalid JSON keeps hosted fallback", { AWS_CI_TRUSTED_USER_IDS: "[34892728,]" }, {}], - ["stale snapshot", { AWS_CI_TRUSTED_USER_IDS: "[34892728,999]" }, {}], - ["missing snapshot", { AWS_CI_ROUTING_USER_IDS_SOURCE: "" }, {}], - ["missing admission members", { AWS_CI_ROUTING_USER_IDS: "" }, {}], - ["disabled routing", { AWS_CI_ENABLED: "false" }, {}], - ["different repository", {}, { repository: "other/repo" }], - ["different repository ID", {}, { repository_id: "999" }], - ["approved manual master canary", {}, { event_name: "workflow_dispatch" }, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], - ["unapproved manual actor", {}, { event_name: "workflow_dispatch", actor_id: "999" }], - ["manual branch canary", {}, { event_name: "workflow_dispatch", ref: "refs/heads/other" }], -]; - -for (const [name, vars, github, expected = "ubuntu-latest"] of placementCases) { - test(`placement before allocation: ${name}`, () => { - assert.equal(routingPlacement(vars, github), expected); - }); -} diff --git a/.github/workflows/ci-incremental-qualification.yml b/.github/workflows/ci-incremental-qualification.yml index 3b4d608dff8..1542dada224 100644 --- a/.github/workflows/ci-incremental-qualification.yml +++ b/.github/workflows/ci-incremental-qualification.yml @@ -58,6 +58,7 @@ jobs: for (const [file, project, related] of [ ['ui/src/pages/Inbox.tsx', '@paperclipai/ui', 'ui/src/pages/Inbox.test.tsx'], ['server/src/middleware/auth.ts', '@paperclipai/server', 'server/src/__tests__/agent-auth-middleware.test.ts'], + ['server/src/storage/config.test.ts', '@paperclipai/server', 'server/src/storage/config.test.ts'], ]) { const selected = await createTestSelection(planTasks({ config, lanes, paths: [file, "doc/DEVELOPING.md", "doc/plans/2026-10-30-ci-proof.md"] }), head); const graph = selected.graphs.find(graph => graph.project === project); diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index 1b156165c5a..8cfaebf342c 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -21,7 +21,9 @@ permissions: jobs: gate: name: Select trusted runner - runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && vars.AWS_CI_ROUTING_USER_IDS_SOURCE != '' && vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.pull_request.user.id)) && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.sender.id))) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} + # No AWS capacity is allocated until current CI CODEOWNERS are verified. + # A cached numeric allowlist alone can outlive removal from CODEOWNERS. + runs-on: ubuntu-latest timeout-minutes: 5 outputs: plan: ${{ steps.plan.outputs.plan }} @@ -51,6 +53,7 @@ jobs: EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} EVENT_MERGE_SHA: ${{ github.sha }} RUN_ID: ${{ github.run_id }} + RUN_ATTEMPT: ${{ github.run_attempt }} run: | set -u @@ -73,7 +76,8 @@ jobs: is_allowed() { local user_id="$1" - jq -e --argjson user_id "$user_id" 'index($user_id) != null' \ + jq -e --argjson user_id "$user_id" --argjson owners "$owner_ids" \ + 'index($user_id) != null and ($owners | index($user_id) != null)' \ <<< "$TRUSTED_USER_IDS" >/dev/null } @@ -94,9 +98,27 @@ jobs: all(.[]; type == "number" and . > 0 and floor == .) ' <<< "$TRUSTED_USER_IDS" >/dev/null 2>&1 || fail_closed 'trusted user ID list is malformed' + # Both the resolver and CODEOWNERS come from the same protected master + # revision. Never use the PR's CODEOWNERS or execute its source here. + trusted_sha="$(gh api /repos/paperclipai/paperclip/git/ref/heads/master --jq .object.sha)" \ + || fail_closed 'could not resolve trusted CODEOWNERS revision' + is_commit_sha "$trusted_sha" || fail_closed 'invalid trusted CODEOWNERS revision' + owner_helper="$RUNNER_TEMP/aws-ci-owners.mjs" + if ! gh api "/repos/paperclipai/paperclip/contents/scripts/ci-profile/authorize.mjs?ref=$trusted_sha" \ + --jq '.content' > "$owner_helper.base64" || + ! base64 --decode < "$owner_helper.base64" > "$owner_helper"; then + fail_closed 'could not load trusted CODEOWNERS resolver' + fi + owner_ids="$(node --input-type=module - "$owner_helper" "$trusted_sha" <<'NODE' + import { pathToFileURL } from 'node:url'; + const { trustedOwnerIds } = await import(pathToFileURL(process.argv[2])); + console.log(JSON.stringify(trustedOwnerIds(undefined, process.argv[3]))); + NODE + )" || fail_closed 'could not verify current CI CODEOWNERS' + for user_id in "$EVENT_PR_AUTHOR_ID" "$EVENT_SENDER_ID"; do is_positive_integer "$user_id" || fail_closed 'event contains a malformed user ID' - is_allowed "$user_id" || fail_closed "GitHub user ID $user_id is not allowlisted" + is_allowed "$user_id" || fail_closed "GitHub user ID $user_id is not an allowlisted CI CODEOWNER" done for commit_sha in "$EVENT_BASE_SHA" "$EVENT_HEAD_SHA" "$EVENT_MERGE_SHA"; do @@ -204,20 +226,24 @@ jobs: [[ "$event_merge_tree" == "$live_merge_tree" ]] \ || fail_closed 'current merge tree differs from the triggering merge tree' + is_positive_integer "$RUN_ID" || fail_closed 'invalid workflow run ID' + is_positive_integer "$RUN_ATTEMPT" || fail_closed 'invalid workflow run attempt' run_json="$(gh api \ -H 'Accept: application/vnd.github+json' \ -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/actions/runs/$RUN_ID" 2>/dev/null)" \ + "/repos/paperclipai/paperclip/actions/runs/$RUN_ID/attempts/$RUN_ATTEMPT" 2>/dev/null)" \ || fail_closed 'could not refresh workflow run state' triggering_actor_id="$(jq -r '.triggering_actor.id // empty' <<< "$run_json")" is_positive_integer "$triggering_actor_id" || fail_closed 'workflow run has no valid triggering actor ID' - is_allowed "$triggering_actor_id" || fail_closed "triggering GitHub user ID $triggering_actor_id is not allowlisted" + is_allowed "$triggering_actor_id" || fail_closed "triggering GitHub user ID $triggering_actor_id is not an allowlisted CI CODEOWNER" jq -e \ --argjson repository_id 1170821064 \ + --argjson run_attempt "$RUN_ATTEMPT" \ --arg event_name "$EVENT_NAME" ' .repository.id == $repository_id and + .run_attempt == $run_attempt and .event == $event_name ' <<< "$run_json" >/dev/null 2>&1 \ || fail_closed 'current workflow run does not match the expected repository event' diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 17712b94903..a59f7c04db4 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -146,19 +146,19 @@ New workflow versions merged into master then receive runner access without a separate SHA allowlist update. Dependabot leaves this first-party reference on master. -Runner selection calls `ci-routing.yml@master` on the dedicated -`paperclip-ci-routing-x64` AWS Fleet. The `paperclip-ci-routing` runner group -admits only `paperclipai/paperclip/.github/workflows/ci-routing.yml@refs/heads/master` -and repository ID `1170821064`. GitHub evaluates the canonical repository, author, and sender checks before -allocating reserved capacity. The validated admission snapshot (`AWS_CI_ROUTING_USER_IDS_SOURCE` and -`AWS_CI_ROUTING_USER_IDS`) must match `AWS_CI_TRUSTED_USER_IDS`. Refresh it through -the runner-group configuration helper when the roster changes. Missing, stale, -or malformed settings keep hosted routing available without evaluating JSON -in the job's placement expression. Unapproved PRs use hosted metadata routing; -approved manual canaries on master can use the reserved pool. The routing -workflow reads GitHub metadata without checking out or executing PR code. Its author, sender, rerun actor, -repository, head, and merge validation still decide whether tests can use AWS. -Unapproved test workloads continue to use GitHub-hosted runners. +Runner selection calls `ci-routing.yml@master` on a GitHub-hosted runner. It +verifies authorization before allocating any AWS capacity and never checks out +or executes PR code. The author, event sender and rerun actor must each be both +in `AWS_CI_TRUSTED_USER_IDS` and a current owner of `.github/**` in CODEOWNERS. +The resolver and CODEOWNERS are read from the same protected master revision; +PR edits cannot change the trusted owner list. Documentation-only owners, +unsupported owner rules and failed identity lookups do not authorize AWS. +Repository, head and merge validation also apply. Missing or malformed settings +and unverifiable authorization use GitHub-hosted runners. A maintainer rerunning +an outside contributor's PR does not grant it AWS access. +Authorized test jobs and the final `verify` and `e2e` aggregates use the selected +AWS fleet. The metadata authorization job remains hosted even for maintainers, +so stale admission snapshots cannot allocate an AWS runner before verification. The routing workflow also selects the CI scope. Middle PRs in a stack and PRs whose diff changes only release notes (`releases/v*.md` or @@ -213,7 +213,11 @@ coverage without paying for another graph traversal. Filesystem, subprocess, net dynamic-loader, unresolved, and unapproved external-package consumers are retained conservatively, including indirect consumers through local helpers. The selector explicitly tracks setup/configuration dependencies as well as test -imports. The small pure-dependency allowlist lives in +imports. Ordinary source names such as `session-config-metadata.test.ts` or +`setup-state.ts` remain eligible; the words `config` and `setup` alone do not +force full CI. Conventional Vite/Vitest config and setup files remain broad +inputs. Other resolved setup/config dependencies invalidate their entire project. +The small pure-dependency allowlist lives in `scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque until reviewed. This is deliberately broader than `vitest related` alone. For conservative retention, explicit complete local mock factories cut the @@ -236,6 +240,9 @@ existing selection/execution rules. The job summary reports files and jobs selected per lane, and the `vitest-selection` artifact records the manifest and dependency audit. Selected jobs execute exact Vitest test specifications, so similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards. +The wrapper consumes and validates the selection before launching Vitest, then +removes selection environment variables from the test process. Nested runner +fixtures therefore use their own checkout and test inventory. Missing/deleted/renamed source files, non-exempt non-source inputs, unrecognized graph inputs, setup/config changes, failed dependency installs, graph errors and timeouts use @@ -288,13 +295,10 @@ explicit. When changing shard layout, update `.github/ci-task-matrix.json` and the full fallback matrices in `pr-trusted.yml`; the contract tests require them to agree. Add new cache-affecting environment settings to `globalEnv`. -The routing Fleet has two warm On-Demand runners and a four-runner ceiling. -Test fleets cannot consume its runner slots. Hosted test queues therefore do -not block AWS runner selection. A manual dispatch of `ci-routing.yml` on master -checks placement without launching the test matrix; its non-PR event returns -the hosted test-runner output. Configure and verify the group and Fleet before -merging this caller. To roll back, restore the hosted gate and retain the routing -workflow authorization until assigned jobs finish. +The former reserved routing Fleet is no longer selected by this workflow. +Keep its existing authorization until older assigned jobs finish. A manual +dispatch of `ci-routing.yml` on master exercises the hosted metadata job without +launching the test matrix; its non-PR event returns the hosted test-runner output. Keep the `.github/**` rule in `.github/CODEOWNERS` and the active master ruleset's code-owner review requirement enabled. This covers the caller, the trusted diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index cf0ab7c636c..6ed462b77a2 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -24,10 +24,11 @@ Ordinary PRs do not write timing artifacts or export traces. The exporter job is skipped for ordinary run names. Adding another label does not run the CI matrix. Profiling preserves the existing trusted AWS runner routing. Eligible PR jobs -use `paperclip-public-pr-x64`; the routing gate uses `paperclip-ci-routing-x64`. +use `paperclip-public-pr-x64`; the routing gate uses `ubuntu-latest` to verify +current CI CODEOWNERS before allocating any AWS capacity. The existing fail-closed GitHub-hosted fallback still applies if AWS routing -cannot be authorized. Only the separate post-run Honeycomb exporter uses -`ubuntu-latest` directly; it is outside the measured PR suite. +cannot be authorized. The separate post-run Honeycomb exporter also uses +`ubuntu-latest`; it is outside the measured PR suite. ## Read the waterfall diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index 4d2296345bd..6b2cc7a6642 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -216,11 +216,11 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the } }); -test("runner selection has reserved AWS capacity and executes metadata checks only", () => { +test("runner authorization uses hosted capacity and executes metadata checks only", () => { const workflow = readFileSync(routingWorkflow, "utf8"); const jobs = readWorkflowJobs(workflow.split(/^jobs:\n/m)[1]); assert.deepEqual([...jobs.keys()], ["gate"]); - assert.match(jobs.get("gate"), /^ {4}runs-on: \$\{\{ vars\.AWS_CI_ENABLED[\s\S]*paperclip-ci-routing-x64/m); + assert.match(jobs.get("gate"), /^ {4}runs-on: ubuntu-latest$/m); assert.doesNotMatch(jobs.get("gate"), /uses:|actions\/checkout|secrets\./); assert.match(workflow, /^ {6}runner:\n {8}value: \$\{\{ jobs\.gate\.outputs\.runner \}\}$/m); assert.match(workflow, /^ {6}full_ci:\n {8}value: \$\{\{ jobs\.gate\.outputs\.full_ci \}\}$/m); diff --git a/scripts/ci/select-tests.mjs b/scripts/ci/select-tests.mjs index 5ad76a10e24..c296ccd92db 100644 --- a/scripts/ci/select-tests.mjs +++ b/scripts/ci/select-tests.mjs @@ -13,8 +13,11 @@ export function selectionEligibility(plan) { const changed = productInputs(plan.changed); if (!changed.length) return "Documentation-only diff"; for (const file of changed) { + // Business modules can contain "config"/"setup" in their names. The graph + // separately tracks actual setup and configuration dependencies. if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file) - || /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`; + || /(?:^|\/)(?:vite|vitest)\.(?:config|setup)\.[cm]?[jt]sx?$|\.d\.[cm]?ts$/i.test(file) + || file.split("/").includes("..")) return `Broad input: ${file}`; try { if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`; } catch { return `Deleted or missing source: ${file}`; } diff --git a/scripts/ci/test-selection-regression.integration.test.mjs b/scripts/ci/test-selection-regression.integration.test.mjs index a18f0f019a7..49930056cb2 100644 --- a/scripts/ci/test-selection-regression.integration.test.mjs +++ b/scripts/ci/test-selection-regression.integration.test.mjs @@ -4,9 +4,32 @@ import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; +import { stripVTControlCharacters } from "node:util"; import { selectProjectTests } from "./vitest-selection-graph.mjs"; import { selectionHash } from "./test-selection-manifest.mjs"; +test("selected server tests can launch independent chat runner fixtures", (t) => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-nested-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const manifest = { version: 1, + head: execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(), + lanes: { "general-server-without-chat": { shards: [{ "@paperclipai/server": [ + "server/src/__tests__/vitest-chat-shards.test.ts", + ] }] } }, + }; + const file = path.join(directory, "selection.json"); + writeFileSync(file, JSON.stringify(manifest)); + const result = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", + "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1"], { + encoding: "utf8", timeout: 180_000, maxBuffer: 4 * 1024 * 1024, + env: { ...process.env, CI: "true", PAPERCLIP_CI_PROFILE: "false", + PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) }, + }); + assert.equal(result.error, undefined); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.match(stripVTControlCharacters(result.stdout), /Test Files\s+1 passed/); +}); + test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => { const repo = process.cwd(); const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-"))); @@ -20,7 +43,7 @@ test("production runner catches the filesystem regression that raw Vitest select write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}'); write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}'); write("ui/src/leaf.mjs", "export const value = 42;\n"); - write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));'); + write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>{expect(value).toBe(42); expect(process.env.PAPERCLIP_CI_TEST_SELECTION).toBeUndefined(); expect(process.env.PAPERCLIP_CI_SELECTION_HASH).toBeUndefined();});'); write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");'); write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));'); git("init", "-q"); git("add", "."); diff --git a/scripts/ci/test-selection.integration.test.mjs b/scripts/ci/test-selection.integration.test.mjs index 15442fcd097..16ec7219ae0 100644 --- a/scripts/ci/test-selection.integration.test.mjs +++ b/scripts/ci/test-selection.integration.test.mjs @@ -158,6 +158,25 @@ test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" })); }); +test("ordinary source names containing config or setup remain graph eligible", (t) => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-names-")); + t.after(() => { process.chdir(root); rmSync(directory, { recursive: true, force: true }); }); + process.chdir(directory); + const plan = fixturePlan().plan; + for (const file of [ + "server/src/services/heartbeat/session-config-metadata.test.ts", + "server/src/storage/config.ts", + "ui/src/pages/apps/chat/slack/setup-state.test.ts", + ]) { + mkdirSync(path.dirname(file), { recursive: true }); + writeFileSync(file, "export {};\n"); + assert.equal(selectionEligibility({ ...plan, changed: [file] }), null, file); + } + for (const file of ["ui/src/vitest.setup.ts", "server/src/vitest.config.mts"]) { + assert.equal(selectionEligibility({ ...plan, changed: [file] }), `Broad input: ${file}`); + } +}); + test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => { const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-"))); t.after(() => rmSync(directory, { recursive: true, force: true })); diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index d72bd532e57..fb48274670f 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -400,6 +400,10 @@ async function runVitest(args, label, testShard = null, exactFiles = false) { PAPERCLIP_INSTANCE_ID: `vt-${process.pid}-${invocationIndex}`, TMPDIR: path.join(testRoot, "t"), }; + // The wrapper has already validated and consumed this shard's manifest. + // Tests may launch independent runner fixtures with a different root/inventory. + delete env.PAPERCLIP_CI_TEST_SELECTION; + delete env.PAPERCLIP_CI_SELECTION_HASH; mkdirSync(env.PAPERCLIP_HOME, { recursive: true }); mkdirSync(env.TMPDIR, { recursive: true }); if (testShard) {