diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index a9703feb747..b6cf5ff2b3e 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -29,7 +29,7 @@ test("routing exports both placement and stack scope to callers", () => { assert.match(jobs, /^ {6}full_ci: \$\{\{ steps\.scope\.outputs\.full_ci \}\}$/m); }); -const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"${*: -1}\"\ncase \"$endpoint\" in\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; +const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"${*: -1}\"\ncase \"$endpoint\" in\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status,\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; function stepScript(name) { const start = workflow.indexOf(` - name: ${name}\n`); @@ -137,12 +137,82 @@ const scopeCases = [ for (const [name, stack, base, expected] of scopeCases) { test(`scope decision: ${name}`, () => { - assert.equal(executeStep("Select stacked PR CI scope", { + assert.equal(executeStep("Select PR CI scope", { STACK_JSON: JSON.stringify(stack), PR_BASE_REF: base, }).full_ci, expected); }); } +const files = (...paths) => JSON.stringify(paths.map((filename) => ({ filename }))); +const manyReleaseNotes = JSON.stringify(Array.from({ length: 300 }, (_, index) => ({ + filename: `releases/beta/v2026.1012.${index}-beta.0.md`, +}))); + +const pathScopeCases = [ + ["stable release notes", { TEST_COMPARE_FILES: files("releases/v2026.1015.0.md") }, "false"], + ["beta release notes draft", { TEST_COMPARE_FILES: files("releases/beta/v2026.1012.1-beta.0.md") }, "false"], + ["stable and beta notes", { TEST_COMPARE_FILES: files("releases/v2026.1015.0.md", "releases/beta/v2026.1012.1-beta.0.md") }, "false"], + ["removed release notes", { TEST_COMPARE_FILES: JSON.stringify([{ filename: "releases/beta/v2026.1006.0-beta.0.md", status: "removed" }]) }, "false"], + ["release notes and code", { TEST_COMPARE_FILES: files("releases/v2026.1015.0.md", "server/src/index.ts") }, "true"], + ["code renamed into releases", { TEST_COMPARE_FILES: JSON.stringify([{ filename: "releases/v2026.1015.0.md", previous_filename: "scripts/release.sh" }]) }, "true"], + ["non-markdown file in releases", { TEST_COMPARE_FILES: files("releases/beta/v2026.1012.1-beta.0.json") }, "true"], + ["nested releases directory", { TEST_COMPARE_FILES: files("releases/beta/drafts/v2026.1012.1-beta.0.md") }, "true"], + ["markdown outside releases", { TEST_COMPARE_FILES: files("doc/RELEASING.md") }, "true"], + ["file entry without a name", { TEST_COMPARE_FILES: JSON.stringify([{ status: "modified" }]) }, "true"], + ["empty diff", { TEST_COMPARE_FILES: "[]" }, "true"], + ["possibly truncated file list", { TEST_COMPARE_FILES: manyReleaseNotes }, "true"], + ["compare lookup failure", { TEST_COMPARE_FAIL: "1", TEST_COMPARE_FILES: files("releases/v2026.1015.0.md") }, "true"], + ["non-PR event", { EVENT_NAME: "workflow_dispatch", TEST_COMPARE_FILES: files("releases/v2026.1015.0.md") }, "true"], + ["malformed head SHA", { EVENT_HEAD_SHA: "invalid", TEST_COMPARE_FILES: files("releases/v2026.1015.0.md") }, "true"], +]; + +for (const [name, env, expected] of pathScopeCases) { + test(`path scope decision: ${name}`, () => { + assert.equal(executeStep("Select PR CI scope", { + STACK_JSON: "null", PR_BASE_REF: "master", ...env, + }).full_ci, expected); + }); +} + +test("path scope keeps full CI for the top PR in a stack that touches code", () => { + assert.equal(executeStep("Select PR CI scope", { + STACK_JSON: JSON.stringify({ position: 11, size: 11, base: { ref: "master" } }), + PR_BASE_REF: "stack-10", + }).full_ci, "true"); +}); + +test("path scope keeps full CI for release notes on top of code-changing stacked PRs", () => { + assert.equal(executeStep("Select PR CI scope", { + STACK_JSON: JSON.stringify({ position: 11, size: 11, base: { ref: "master" } }), + PR_BASE_REF: "stack-10", + TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), + }).full_ci, "true"); +}); + +test("path scope keeps full CI for release notes with malformed stack metadata", () => { + assert.equal(executeStep("Select PR CI scope", { + STACK_JSON: JSON.stringify({ position: 0, size: 2, base: { ref: "master" } }), + PR_BASE_REF: "master", + TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), + }).full_ci, "true"); +}); + +test("path scope skips full CI for the lowest PR in a stack that only touches release notes", () => { + assert.equal(executeStep("Select PR CI scope", { + STACK_JSON: JSON.stringify({ position: 1, size: 3, base: { ref: "master" } }), + PR_BASE_REF: "master", + TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), + }).full_ci, "false"); +}); + +test("path scope skips full CI for a single-PR stack that only touches release notes", () => { + assert.equal(executeStep("Select PR CI scope", { + STACK_JSON: JSON.stringify({ position: 1, size: 1, base: { ref: "master" } }), + PR_BASE_REF: "master", + TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), + }).full_ci, "false"); +}); + function routingPlacement(vars = {}, github = {}) { const source = "[34892728]"; diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index 5ad3e1e11e3..a146b834bd2 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -215,10 +215,14 @@ jobs: echo "runner=$aws_runner" >> "$GITHUB_OUTPUT" echo '::notice title=AWS CI routing::Using an ephemeral RunsOn Fleet runner' - - name: Select stacked PR CI scope + - name: Select PR CI scope id: scope shell: bash env: + GH_TOKEN: ${{ github.token }} + EVENT_NAME: ${{ github.event_name }} + EVENT_BASE_SHA: ${{ github.event.pull_request.base.sha }} + EVENT_HEAD_SHA: ${{ github.event.pull_request.head.sha }} STACK_JSON: ${{ toJSON(github.event.pull_request.stack) }} PR_BASE_REF: ${{ github.event.pull_request.base.ref }} run: | @@ -226,17 +230,53 @@ jobs: full_ci='true' reason='ordinary pull request' + # True when the PR's own diff is everything it adds to the stack's + # base branch: no stack, or the lowest unmerged PR in one. Above + # that, the diff omits lower PRs' code that this run must still test. + diff_covers_stack='true' + + # Release notes PRs touch only releases/v*.md or releases/beta/v*.md, + # which no build, test, or e2e lane reads. The diff is pinned to the + # event's base and head SHAs, so a later push cannot change the + # decision for this run. Any lookup failure keeps full CI. + release_notes_only() { + [[ "${EVENT_NAME:-}" == 'pull_request' ]] || return 1 + [[ "${EVENT_BASE_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || return 1 + [[ "${EVENT_HEAD_SHA:-}" =~ ^[0-9a-f]{40}$ ]] || return 1 + + local comparison + comparison="$(gh api \ + -H 'Accept: application/vnd.github+json' \ + -H 'X-GitHub-Api-Version: 2022-11-28' \ + "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$EVENT_HEAD_SHA" 2>/dev/null)" \ + || return 1 + + # The compare API lists at most 300 files, so a full page may be + # truncated. Renames count both the old and the new path. + jq -e ' + (.files | type == "array") and + (.files | length > 0 and length < 300) and + all(.files[]; + (.filename | type == "string") and + ([.filename, .previous_filename] | map(select(. != null)) | + all(.[]; type == "string" and test("^releases/(beta/)?v[^/]+\\.md$")))) + ' <<< "$comparison" >/dev/null 2>&1 + } if jq -e 'type == "object"' <<< "$STACK_JSON" >/dev/null 2>&1; then stack_position="$(jq -r '.position // empty' <<< "$STACK_JSON")" stack_size="$(jq -r '.size // empty' <<< "$STACK_JSON")" stack_base_ref="$(jq -r '.base.ref // empty' <<< "$STACK_JSON")" + if [[ -z "$stack_base_ref" || "$stack_base_ref" != "$PR_BASE_REF" ]]; then + diff_covers_stack='false' + fi if [[ ! "$stack_position" =~ ^[1-9][0-9]*$ ]] || [[ ! "$stack_size" =~ ^[1-9][0-9]*$ ]] || (( stack_position > stack_size )) || [[ -z "$stack_base_ref" ]]; then reason='malformed stack metadata; defaulting to full CI' + diff_covers_stack='false' elif (( stack_position == stack_size )); then reason='top pull request in stack' elif [[ "$stack_base_ref" == "$PR_BASE_REF" ]]; then @@ -247,6 +287,12 @@ jobs: fi fi + if [[ "$full_ci" == 'true' && "$diff_covers_stack" == 'true' ]] && + release_notes_only; then + full_ci='false' + reason='release notes only' + fi + echo "full_ci=$full_ci" >> "$GITHUB_OUTPUT" - echo "::notice title=Stacked PR CI scope::$reason; full_ci=$full_ci" + echo "::notice title=PR CI scope::$reason; full_ci=$full_ci" diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 7ad912abaf8..2bf0ae54238 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -49,6 +49,15 @@ workflow reads GitHub metadata without checking out or executing PR code. Its au repository, head, and merge validation still decide whether tests can use AWS. Unapproved test workloads continue to use GitHub-hosted runners. +The routing workflow also selects the CI scope. Middle PRs in a stack and PRs +whose diff changes only release notes (`releases/v*.md` or +`releases/beta/v*.md`) skip the build, test, and e2e lanes. The `policy` job +and the `verify` and `e2e` aggregate checks still run. The release-notes check +reads the diff between the event's base and head commits. If that lookup +fails, or the diff is empty or may be truncated, the PR runs full CI. A +notes-only PR stacked above other PRs also runs full CI, because its own diff +does not include the lower PRs' code. + The routing Fleet has two warm On-Demand runners and a four-runner ceiling. Test fleets cannot consume its runner slots. Hosted test queues therefore do not block AWS runner selection. A manual dispatch of `ci-routing.yml` on master diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index 28bf3926f3c..b6ee3e49f2c 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -57,9 +57,9 @@ function readWorkflowJobs(workflow) { function runStackScope(stack, prBaseRef) { const workflow = readFileSync(routingWorkflow, "utf8"); const match = workflow.match( - / - name: Select stacked PR CI scope[\s\S]*? run: \|\n([\s\S]*?)\s*$/, + / - name: Select PR CI scope[\s\S]*? run: \|\n([\s\S]*?)\s*$/, ); - assert.ok(match, "routing workflow must define the stacked PR scope script"); + assert.ok(match, "routing workflow must define the PR CI scope script"); const script = match[1] .split("\n") .map((line) => line.replace(/^ {10}/, ""))