diff --git a/.github/scripts/tests/ci-routing.test.mjs b/.github/scripts/tests/ci-routing.test.mjs index ae43a65791f..3b590332789 100644 --- a/.github/scripts/tests/ci-routing.test.mjs +++ b/.github/scripts/tests/ci-routing.test.mjs @@ -4,15 +4,17 @@ import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; +import vm from "node:vm"; const workflow = readFileSync(new URL("../../workflows/ci-routing.yml", import.meta.url), "utf8"); const jobs = workflow.split(/^jobs:\n/m)[1]; -test("authorization uses hosted capacity and runs no PR code or actions", () => { +test("authorization uses reserved capacity and runs no PR code or actions", () => { assert.deepEqual([...jobs.matchAll(/^ {2}([a-z_]+):$/gm)].map((match) => match[1]), ["gate"]); const placement = jobs.match(/^ {4}runs-on: (.+)$/m)?.[1]; assert.ok(placement, "routing placement must be defined"); - assert.equal(placement, "ubuntu-latest", "verify current owners before allocating any AWS runner"); + assert.match(placement, /runs-on\/fleet=paperclip-ci-routing-x64\/env=public-ci/); + assert.doesNotMatch(placement, /fromJSON/); assert.doesNotMatch(jobs, /uses:|actions\/checkout|secrets\./); assert.doesNotMatch(workflow, /^ {2}(?:actions|contents|pull-requests): write$/m); }); @@ -24,7 +26,7 @@ test("routing exports both placement and stack scope to callers", () => { assert.match(jobs, /^ {6}full_ci: \$\{\{ steps\.scope\.outputs\.full_ci \}\}$/m); }); -const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"\"\nquery=\"\"\nfor arg in \"$@\"; do\n case \"$arg\" in /repos/*|/users/*) endpoint=\"$arg\" ;; esac\ndone\nif [[ \"$*\" == *\"--jq\"* ]]; then query=\"${*: -1}\"; fi\ncase \"$endpoint\" in\n */contents/scripts/ci-profile/authorize.mjs?ref=*)\n [[ -z \"${TEST_OWNER_HELPER_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n base64 < \"$TEST_OWNER_HELPER\"\n ;;\n */contents/.github/CODEOWNERS?ref=*)\n [[ -z \"${TEST_CODEOWNERS_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n jq -cn --arg content \"$TEST_CODEOWNERS_BASE64\" '{encoding: \"base64\", content: $content}'\n ;;\n /users/ci-owner)\n [[ -z \"${TEST_OWNER_LOOKUP_FAIL:-}\" ]] || exit 1\n jq -cn --argjson id \"${TEST_CI_OWNER_ID:-34892728}\" --arg type \"${TEST_CI_OWNER_TYPE:-User}\" '{id: $id, type: $type}'\n ;;\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n if [[ \"$query\" == \".object.sha\" ]]; then\n [[ -z \"${TEST_MASTER_FAIL:-}\" ]] || exit 1\n echo \"${TEST_TRUSTED_MASTER_SHA:-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}\"\n exit 0\n fi\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg merge_base_sha \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg status \"${TEST_BASE_COMPARISON_STATUS:-ahead}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '{\n merge_base_commit: {sha: $merge_base_sha},\n status: $status,\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*/attempts/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --argjson run_attempt \"${TEST_RUN_ATTEMPT:-1}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n run_attempt: $run_attempt,\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; +const githubStub = "#!/usr/bin/env bash\nset -Eeuo pipefail\n\nendpoint=\"\"\nquery=\"\"\nfor arg in \"$@\"; do\n case \"$arg\" in /repos/*|/users/*) endpoint=\"$arg\" ;; esac\ndone\nif [[ \"$*\" == *\"--jq\"* ]]; then query=\"${*: -1}\"; fi\ncase \"$endpoint\" in\n */contents/scripts/ci-profile/authorize.mjs?ref=*)\n [[ -z \"${TEST_OWNER_HELPER_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n base64 < \"$TEST_OWNER_HELPER\"\n ;;\n */contents/.github/CODEOWNERS?ref=*)\n [[ -z \"${TEST_CODEOWNERS_FAIL:-}\" ]] || exit 1\n [[ \"$endpoint\" == *\"ref=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\" ]] || exit 1\n jq -cn --arg content \"$TEST_CODEOWNERS_BASE64\" '{encoding: \"base64\", content: $content}'\n ;;\n /users/ci-owner)\n [[ -z \"${TEST_OWNER_LOOKUP_FAIL:-}\" ]] || exit 1\n jq -cn --argjson id \"${TEST_CI_OWNER_ID:-34892728}\" --arg type \"${TEST_CI_OWNER_TYPE:-User}\" '{id: $id, type: $type}'\n ;;\n */pulls/*)\n jq -cn \\\n --argjson author_id \"${TEST_PR_AUTHOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --arg base_ref \"${TEST_BASE_REF:-master}\" \\\n --arg base_sha \"${TEST_PR_BASE_SHA:-$EVENT_BASE_SHA}\" \\\n --arg head_sha \"${TEST_PR_HEAD_SHA:-1111111111111111111111111111111111111111}\" \\\n --arg merge_sha \"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\" '{\n state: \"open\",\n user: {id: $author_id},\n base: {repo: {id: $repository_id}, ref: $base_ref, sha: $base_sha},\n head: {sha: $head_sha},\n merge_commit_sha: $merge_sha\n }'\n ;;\n */git/ref/heads/*)\n if [[ \"$query\" == \".object.sha\" ]]; then\n [[ -z \"${TEST_MASTER_FAIL:-}\" ]] || exit 1\n echo \"${TEST_TRUSTED_MASTER_SHA:-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa}\"\n exit 0\n fi\n jq -cn \\\n --arg sha \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" '{object: {sha: $sha}}'\n ;;\n */compare/*)\n [[ -z \"${TEST_COMPARE_FAIL:-}\" ]] || exit 1\n comparison=\"${endpoint##*/}\"\n comparison_base=\"${comparison%%...*}\"\n comparison_head=\"${comparison##*...}\"\n default_files='[{\"filename\":\"server/src/index.ts\"}]'\n jq -cn \\\n --arg base \"$comparison_base\" \\\n --arg head \"$comparison_head\" \\\n --arg event_base \"$EVENT_BASE_SHA\" \\\n --arg intermediate \"${TEST_INTERMEDIATE_BASE_SHA:-}\" \\\n --arg current \"${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}\" \\\n --arg side_branch \"${TEST_SIDE_BRANCH_SHA:-}\" \\\n --arg merge_base_override \"${TEST_BASE_COMPARISON_MERGE_BASE_SHA:-}\" \\\n --arg status_override \"${TEST_BASE_COMPARISON_STATUS:-}\" \\\n --argjson files \"${TEST_COMPARE_FILES:-$default_files}\" '\n [$event_base, $intermediate, $current] | map(select(length > 0)) as $history |\n ($history | index($base)) as $from |\n ($history | index($head)) as $to |\n (($from != null and $to != null and $from <= $to) or\n ($side_branch != \"\" and $base == $event_base and $head == $side_branch)) as $ancestor |\n {\n merge_base_commit: {sha: (if $merge_base_override != \"\" then $merge_base_override\n elif $ancestor then $base\n elif $base == $side_branch and $to != null then $event_base\n else \"0000000000000000000000000000000000000000\" end)},\n status: (if $status_override != \"\" then $status_override\n elif $base == $head then \"identical\" elif $ancestor then \"ahead\" else \"diverged\" end),\n files: $files\n }'\n ;;\n */git/commits/*)\n requested_sha=\"${endpoint##*/}\"\n live_merge_sha=\"${TEST_PR_MERGE_SHA:-4444444444444444444444444444444444444444}\"\n synthetic_base_merge_sha=\"${TEST_SYNTHETIC_BASE_MERGE_SHA:-}\"\n if [[ \"$requested_sha\" == \"$EVENT_MERGE_SHA\" ]]; then\n tree_sha=\"${TEST_EVENT_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_EVENT_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_EVENT_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ \"$requested_sha\" == \"$live_merge_sha\" ]]; then\n tree_sha=\"${TEST_LIVE_MERGE_TREE_SHA:-5555555555555555555555555555555555555555}\"\n base_sha=\"${TEST_LIVE_MERGE_BASE_SHA:-$EVENT_BASE_SHA}\"\n head_sha=\"${TEST_LIVE_MERGE_HEAD_SHA:-$EVENT_HEAD_SHA}\"\n elif [[ -n \"$synthetic_base_merge_sha\" && \"$requested_sha\" == \"$synthetic_base_merge_sha\" ]]; then\n tree_sha=\"${TEST_SYNTHETIC_BASE_MERGE_TREE_SHA:-8888888888888888888888888888888888888888}\"\n base_sha=\"${TEST_SYNTHETIC_MASTER_PARENT_SHA:-9999999999999999999999999999999999999999}\"\n head_sha=\"${TEST_SYNTHETIC_BASE_REF_PARENT_SHA:-${TEST_LIVE_BASE_REF_SHA:-$EVENT_BASE_SHA}}\"\n else\n echo \"unexpected merge commit: $requested_sha\" >&2\n exit 1\n fi\n jq -cn \\\n --arg sha \"$requested_sha\" \\\n --arg tree_sha \"$tree_sha\" \\\n --arg base_sha \"$base_sha\" \\\n --arg head_sha \"$head_sha\" '{\n sha: $sha,\n tree: {sha: $tree_sha},\n parents: [{sha: $base_sha}, {sha: $head_sha}]\n }'\n ;;\n */actions/runs/*/attempts/*)\n jq -cn \\\n --argjson actor_id \"${TEST_TRIGGERING_ACTOR_ID:-34892728}\" \\\n --argjson repository_id \"${TEST_REPOSITORY_ID:-1170821064}\" \\\n --argjson run_attempt \"${TEST_RUN_ATTEMPT:-1}\" \\\n --arg event_name \"${TEST_EVENT_NAME:-pull_request}\" '{\n run_attempt: $run_attempt,\n triggering_actor: {id: $actor_id},\n repository: {id: $repository_id},\n event: $event_name\n }'\n ;;\n *)\n echo \"unexpected gh endpoint: $endpoint\" >&2\n exit 1\n ;;\nesac"; function stepScript(name) { const start = workflow.indexOf(` - name: ${name}\n`); @@ -100,6 +102,31 @@ const routeCases = [ ["unrelated label", { EVENT_ACTION: "labeled", EVENT_LABEL_NAME: "documentation" }], ["validated stacked base", stackedBase, awsRunner], ["lagging PR API base snapshot", { TEST_PR_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, awsRunner], + ["base advances while the routing job waits", { TEST_LIVE_BASE_REF_SHA: "7777777777777777777777777777777777777777" }, awsRunner], + ["merge uses a newer base snapshot than the event payload", { + TEST_LIVE_BASE_REF_SHA: "7777777777777777777777777777777777777777", + TEST_INTERMEDIATE_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + TEST_EVENT_MERGE_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, awsRunner], + ["GitHub regenerates the live merge after base advancement", { + TEST_LIVE_BASE_REF_SHA: "7777777777777777777777777777777777777777", + TEST_LIVE_MERGE_BASE_SHA: "7777777777777777777777777777777777777777", + TEST_LIVE_MERGE_TREE_SHA: "6666666666666666666666666666666666666666", + }, awsRunner], + ["live merge is temporarily unavailable", { TEST_PR_MERGE_SHA: "unavailable" }, awsRunner], + ["stacked base advances while the routing job waits", { + ...stackedBase, + TEST_SYNTHETIC_BASE_REF_PARENT_SHA: "7777777777777777777777777777777777777777", + TEST_INTERMEDIATE_BASE_SHA: "7777777777777777777777777777777777777777", + TEST_LIVE_BASE_REF_SHA: "cccccccccccccccccccccccccccccccccccccccc", + }, awsRunner], + ["merge parent outside the base branch history", { TEST_EVENT_MERGE_BASE_SHA: "cccccccccccccccccccccccccccccccccccccccc" }], + ["stacked base parent descends from the event base but diverges from the live base", { + ...stackedBase, + TEST_SIDE_BRANCH_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + TEST_SYNTHETIC_BASE_REF_PARENT_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }], + ["base history lookup unavailable", { TEST_COMPARE_FAIL: "1" }], ["disabled AWS", { AWS_CI_ENABLED: "false" }], ["missing AWS variables", { AWS_CI_ENABLED: "", TRUSTED_USER_IDS: "" }], ["unlisted author", { EVENT_PR_AUTHOR_ID: "99999999" }], @@ -125,9 +152,9 @@ const routeCases = [ ["wrong base repository", { EVENT_BASE_REPOSITORY_ID: "999999999" }], ["replaced base ancestry", { TEST_BASE_COMPARISON_MERGE_BASE_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }], ["stale head", { TEST_PR_HEAD_SHA: "3333333333333333333333333333333333333333" }], - ["changed merge head", { TEST_LIVE_MERGE_HEAD_SHA: "3333333333333333333333333333333333333333" }], - ["stacked base with wrong parent", { ...stackedBase, TEST_SYNTHETIC_BASE_REF_PARENT_SHA: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }], - ["changed merge tree", { TEST_LIVE_MERGE_TREE_SHA: "6666666666666666666666666666666666666666" }], + ["event merge has the wrong head", { TEST_EVENT_MERGE_HEAD_SHA: "3333333333333333333333333333333333333333" }], + ["stacked base with wrong parent", { ...stackedBase, TEST_SYNTHETIC_BASE_REF_PARENT_SHA: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }], + ["event merge has an invalid tree", { TEST_EVENT_MERGE_TREE_SHA: "invalid" }], ["wrong repository", { EVENT_REPOSITORY_ID: "999999999" }], ["malformed allowlist", { TRUSTED_USER_IDS: '["34892728"]' }], ["manual dispatch", { EVENT_NAME: "workflow_dispatch" }], @@ -241,3 +268,52 @@ test("path scope skips full CI for a single-PR stack that only touches release n TEST_COMPARE_FILES: files("releases/v2026.1015.0.md"), }).full_ci, "false"); }); + +function routingPlacement(vars = {}, github = {}) { + const source = "[34892728]"; + const expression = jobs.match(/^ {4}runs-on: \$\{\{ (.+) \}\}$/m)?.[1]; + assert.ok(expression, "routing expression is missing"); + return vm.runInNewContext(expression, { + vars: { + AWS_CI_ENABLED: "true", + AWS_CI_TRUSTED_USER_IDS: source, + AWS_CI_ROUTING_USER_IDS_SOURCE: source, + AWS_CI_ROUTING_USER_IDS: "|34892728|", + ...vars, + }, + github: { + repository: "paperclipai/paperclip", + repository_id: "1170821064", + event_name: "pull_request", + ref: "refs/heads/master", + actor_id: "34892728", + event: { pull_request: { user: { id: 34892728 } }, sender: { id: 34892728 } }, + ...github, + }, + contains: (search, item) => String(search).toLowerCase().includes(String(item).toLowerCase()), + format: (template, ...args) => template.replace(/\{(\d+)\}/g, (_, index) => String(args[Number(index)])), + }, { timeout: 1000 }); +} + +const placementCases = [ + ["approved PR", {}, {}, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], + ["unapproved author", {}, { event: { pull_request: { user: { id: 999 } }, sender: { id: 34892728 } } }], + ["unapproved sender", {}, { event: { pull_request: { user: { id: 34892728 } }, sender: { id: 999 } } }], + ["partial ID cannot match", {}, { event: { pull_request: { user: { id: 4892728 } }, sender: { id: 34892728 } } }], + ["invalid JSON keeps hosted fallback", { AWS_CI_TRUSTED_USER_IDS: "[34892728,]" }, {}], + ["stale snapshot", { AWS_CI_TRUSTED_USER_IDS: "[34892728,999]" }, {}], + ["missing snapshot", { AWS_CI_ROUTING_USER_IDS_SOURCE: "" }, {}], + ["missing admission members", { AWS_CI_ROUTING_USER_IDS: "" }, {}], + ["disabled routing", { AWS_CI_ENABLED: "false" }, {}], + ["different repository", {}, { repository: "other/repo" }], + ["different repository ID", {}, { repository_id: "999" }], + ["approved manual master canary", {}, { event_name: "workflow_dispatch" }, "runs-on/fleet=paperclip-ci-routing-x64/env=public-ci"], + ["unapproved manual actor", {}, { event_name: "workflow_dispatch", actor_id: "999" }], + ["manual branch canary", {}, { event_name: "workflow_dispatch", ref: "refs/heads/other" }], +]; + +for (const [name, vars, github, expected = "ubuntu-latest"] of placementCases) { + test(`placement before allocation: ${name}`, () => { + assert.equal(routingPlacement(vars, github), expected); + }); +} diff --git a/.github/scripts/tests/post-merge-runner-routing.test.mjs b/.github/scripts/tests/post-merge-runner-routing.test.mjs index d31d315c042..7c46aa47931 100644 --- a/.github/scripts/tests/post-merge-runner-routing.test.mjs +++ b/.github/scripts/tests/post-merge-runner-routing.test.mjs @@ -7,7 +7,7 @@ const fleet = "runs-on/fleet=paperclip-post-merge-x64/env=public-ci"; const sha = "a".repeat(40); const base = { repository: "paperclipai/paperclip", repository_id: "1170821064", - ref: "refs/heads/master", event_name: "push", sha, + ref: "refs/heads/master", event_name: "push", sha, run_attempt: 1, }; const expectedJobs = { "cloud-readiness.yml": [], @@ -58,6 +58,11 @@ for (const [file, expectedNames] of Object.entries(expectedJobs)) { cases.push({ name: "preview of master", inputs: { channel: "preview" } }); cases.push({ name: "stable release", inputs: { channel: "stable" } }); } + cases.push({ name: "first attempt as a string", github: { run_attempt: "1" }, expected: fleet }); + cases.push({ name: "rerun after runner interruption", github: { run_attempt: 2 } }); + cases.push({ name: "later rerun", github: { run_attempt: 3 } }); + cases.push({ name: "rerun attempt as a string", github: { run_attempt: "2" } }); + cases.push({ name: "missing attempt", github: { run_attempt: undefined } }); for (const { name, github = {}, inputs: overrides = {}, enabled = "true", expected = "ubuntu-latest" } of cases) { test(`${file}/${job}: ${name}`, () => { const context = { github: { ...defaultContext, ...github }, inputs: { ...inputs, ...overrides }, vars: { AWS_POST_MERGE_CI_ENABLED: enabled } }; diff --git a/.github/workflows/ci-routing.yml b/.github/workflows/ci-routing.yml index 8cfaebf342c..7fc7416f369 100644 --- a/.github/workflows/ci-routing.yml +++ b/.github/workflows/ci-routing.yml @@ -21,9 +21,7 @@ permissions: jobs: gate: name: Select trusted runner - # No AWS capacity is allocated until current CI CODEOWNERS are verified. - # A cached numeric allowlist alone can outlive removal from CODEOWNERS. - runs-on: ubuntu-latest + runs-on: ${{ vars.AWS_CI_ENABLED == 'true' && vars.AWS_CI_ROUTING_USER_IDS_SOURCE != '' && vars.AWS_CI_TRUSTED_USER_IDS == vars.AWS_CI_ROUTING_USER_IDS_SOURCE && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && ((github.event_name == 'pull_request' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.pull_request.user.id)) && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.event.sender.id))) || (github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/master' && contains(vars.AWS_CI_ROUTING_USER_IDS, format('|{0}|', github.actor_id)))) && 'runs-on/fleet=paperclip-ci-routing-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: plan: ${{ steps.plan.outputs.plan }} @@ -145,9 +143,6 @@ jobs: ' <<< "$pr_json" >/dev/null 2>&1 \ || fail_closed 'current pull request state does not match the triggering event' - live_merge_sha="$(jq -r '.merge_commit_sha // empty' <<< "$pr_json")" - is_commit_sha "$live_merge_sha" || fail_closed 'current pull request has no valid merge SHA' - base_ref_json="$(gh api \ -H 'Accept: application/vnd.github+json' \ -H 'X-GitHub-Api-Version: 2022-11-28' \ @@ -174,12 +169,6 @@ jobs: "/repos/paperclipai/paperclip/git/commits/$EVENT_MERGE_SHA" 2>/dev/null)" \ || fail_closed 'could not inspect the event merge commit' - live_merge_json="$(gh api \ - -H 'Accept: application/vnd.github+json' \ - -H 'X-GitHub-Api-Version: 2022-11-28' \ - "/repos/paperclipai/paperclip/git/commits/$live_merge_sha" 2>/dev/null)" \ - || fail_closed 'could not inspect the current merge commit' - validate_merge_commit() { local merge_json="$1" local expected_sha="$2" @@ -195,37 +184,50 @@ jobs: } validate_merge_commit "$event_merge_json" "$EVENT_MERGE_SHA" \ - || fail_closed 'event merge commit does not match the current base and head' - validate_merge_commit "$live_merge_json" "$live_merge_sha" \ - || fail_closed 'current merge commit does not match the triggering base and head' + || fail_closed 'event merge commit does not match the triggering head' + + is_base_snapshot() { + local snapshot_sha="$1" + local comparison + [[ "$snapshot_sha" == "$EVENT_BASE_SHA" || "$snapshot_sha" == "$live_base_ref_sha" ]] && return 0 + comparison="$(gh api "/repos/paperclipai/paperclip/compare/$EVENT_BASE_SHA...$snapshot_sha" 2>/dev/null)" || return 1 + jq -e --arg base "$EVENT_BASE_SHA" ' + .merge_base_commit.sha == $base and + (.status == "ahead" or .status == "identical") + ' <<< "$comparison" >/dev/null 2>&1 || return 1 + comparison="$(gh api "/repos/paperclipai/paperclip/compare/$snapshot_sha...$live_base_ref_sha" 2>/dev/null)" || return 1 + jq -e --arg base "$snapshot_sha" ' + .merge_base_commit.sha == $base and + (.status == "ahead" or .status == "identical") + ' <<< "$comparison" >/dev/null 2>&1 + } event_merge_parent="$(jq -r '.parents[0].sha' <<< "$event_merge_json")" - live_merge_parent="$(jq -r '.parents[0].sha' <<< "$live_merge_json")" - [[ "$event_merge_parent" == "$live_merge_parent" ]] \ - || fail_closed 'current merge commit uses a different base merge parent' - - if [[ "$event_merge_parent" != "$live_base_ref_sha" ]]; then + if ! is_base_snapshot "$event_merge_parent"; then base_merge_json="$(gh api \ -H 'Accept: application/vnd.github+json' \ -H 'X-GitHub-Api-Version: 2022-11-28' \ "/repos/paperclipai/paperclip/git/commits/$event_merge_parent" 2>/dev/null)" \ || fail_closed 'could not inspect the stacked base merge commit' jq -e \ - --arg expected_sha "$event_merge_parent" \ - --arg live_base_ref_sha "$live_base_ref_sha" ' + --arg expected_sha "$event_merge_parent" ' .sha == $expected_sha and (.parents | length) == 2 and - any(.parents[]; .sha == $live_base_ref_sha) and + all(.parents[]; .sha | test("^[0-9a-f]{40}$")) and (.tree.sha | test("^[0-9a-f]{40}$")) ' <<< "$base_merge_json" >/dev/null 2>&1 \ - || fail_closed 'stacked base merge commit does not contain the current base branch' + || fail_closed 'invalid stacked base merge commit' + base_snapshot_found=false + while read -r parent_sha; do + if is_base_snapshot "$parent_sha"; then + base_snapshot_found=true + break + fi + done < <(jq -r '.parents[].sha' <<< "$base_merge_json") + [[ "$base_snapshot_found" == 'true' ]] \ + || fail_closed 'stacked base merge commit does not contain a validated base snapshot' fi - event_merge_tree="$(jq -r '.tree.sha' <<< "$event_merge_json")" - live_merge_tree="$(jq -r '.tree.sha' <<< "$live_merge_json")" - [[ "$event_merge_tree" == "$live_merge_tree" ]] \ - || fail_closed 'current merge tree differs from the triggering merge tree' - is_positive_integer "$RUN_ID" || fail_closed 'invalid workflow run ID' is_positive_integer "$RUN_ATTEMPT" || fail_closed 'invalid workflow run attempt' run_json="$(gh api \ diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 1eecbe823b2..455512ee2de 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -421,6 +421,9 @@ jobs: PAPERCLIP_TYPECHECK_LOW_MEMORY: "1" run: pnpm --filter @paperclipai/server typecheck + - name: Typecheck test projects against the server declarations + run: node scripts/typecheck-test-projects.mjs --low-memory --reuse-declarations + - name: Verify repository-wide CLI guidance safety run: pnpm exec vitest run --project @paperclipai/server server/src/__tests__/cli-invocation-safety.test.ts diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 76128fd1a89..e36bdbccbd2 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -34,7 +34,7 @@ jobs: typecheck: if: ${{ !inputs.runner_binaries_only }} name: Typecheck - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -120,7 +120,7 @@ jobs: general_tests: if: ${{ !inputs.runner_binaries_only }} name: General tests (${{ matrix.group_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -232,7 +232,7 @@ jobs: serialized_tests: if: ${{ !inputs.runner_binaries_only }} name: Serialized tests (${{ matrix.shard_label }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -282,7 +282,7 @@ jobs: runner_workflow_evals: if: ${{ !inputs.runner_binaries_only }} name: Runner workflow eval scorer contract - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 10 permissions: contents: read @@ -313,7 +313,7 @@ jobs: verify_paperclip_runner: if: ${{ !inputs.runner_binaries_only }} name: Verify Paperclip Runner (${{ matrix.lane }}) - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read @@ -523,7 +523,7 @@ jobs: build: if: ${{ !inputs.runner_binaries_only }} name: Build - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 20 permissions: contents: read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a398e30dae..c7b310f4ee1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -94,11 +94,11 @@ jobs: # falls back to GitHub-hosted runners, including raced merge dispatches. name: Check preview artifacts if: github.ref == 'refs/heads/master' && github.event_name == 'workflow_dispatch' && (inputs.channel == 'preview' || inputs.channel == 'cloud-migrator') && !inputs.dry_run - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} permissions: contents: read # Preserve the previous hosted default; only AWS needs the Fleet limit. - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 10 || 360 }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 10 || 360 }} outputs: image: ${{ steps.plan.outputs.image }} packages: ${{ steps.plan.outputs.packages }} @@ -122,7 +122,7 @@ jobs: name: Build preview migrator needs: plan_preview if: needs.plan_preview.outputs.packages == 'true' - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.event_name == 'workflow_dispatch' && inputs.channel == 'cloud-migrator' && github.sha != '' && inputs.source_ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} timeout-minutes: 30 permissions: contents: read diff --git a/.github/workflows/runner-chaos-evals.yml b/.github/workflows/runner-chaos-evals.yml index 91438bec3a8..810d28ec2e7 100644 --- a/.github/workflows/runner-chaos-evals.yml +++ b/.github/workflows/runner-chaos-evals.yml @@ -20,8 +20,8 @@ concurrency: jobs: chaos_and_recovery: name: Restart, replay, trace, and recovery faults - runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} - timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} + runs-on: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 'runs-on/fleet=paperclip-post-merge-x64/env=public-ci' || 'ubuntu-latest' }} + timeout-minutes: ${{ vars.AWS_POST_MERGE_CI_ENABLED == 'true' && github.run_attempt == 1 && github.repository == 'paperclipai/paperclip' && github.repository_id == '1170821064' && github.ref == 'refs/heads/master' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && github.sha != '' && inputs.ref == github.sha && 40 || 45 }} permissions: contents: read diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 3cf16c21fab..182afe1d850 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -147,19 +147,26 @@ New workflow versions merged into master then receive runner access without a separate SHA allowlist update. Dependabot leaves this first-party reference on master. -Runner selection calls `ci-routing.yml@master` on a GitHub-hosted runner. It -verifies authorization before allocating any AWS capacity and never checks out -or executes PR code. The author, event sender and rerun actor must each be both +Runner selection calls `ci-routing.yml@master`. Allowlisted maintainers use the +reserved `paperclip-ci-routing-x64` AWS fleet for this bounded metadata job; +other callers use GitHub-hosted capacity. The gate never checks out or executes +PR code. The author, event sender and rerun actor must each be both in `AWS_CI_TRUSTED_USER_IDS` and a current owner of `.github/**` in CODEOWNERS. The resolver and CODEOWNERS are read from the same protected master revision; PR edits cannot change the trusted owner list. Documentation-only owners, unsupported owner rules and failed identity lookups do not authorize AWS. -Repository, head and merge validation also apply. Missing or malformed settings +Repository, head and merge validation also apply. The gate validates this run's +immutable event merge, including its PR head and a base snapshot between the +event's base and the current branch tip. Normal base advancement and regeneration +of GitHub's live merge ref do not move an authorized run to GitHub-hosted CI. +Stacked synthetic bases must contain a validated base snapshot. Rewritten base +history, changed PR heads and unverifiable ancestry still deny AWS execution. +Missing or malformed settings and unverifiable authorization use GitHub-hosted runners. A maintainer rerunning an outside contributor's PR does not grant it AWS access. Authorized test jobs and the final `verify` and `e2e` aggregates use the selected -AWS fleet. The metadata authorization job remains hosted even for maintainers, -so stale admission snapshots cannot allocate an AWS runner before verification. +AWS fleet. A stale admission roster can allocate only the short metadata gate; +the fresh CODEOWNERS check still denies PR execution for a removed owner. The routing workflow also selects the CI scope. Middle PRs in a stack and PRs whose diff changes only release notes (`releases/v*.md` or @@ -339,10 +346,16 @@ explicit. When changing shard layout, update `.github/ci-task-matrix.json` and the full fallback matrices in `pr-trusted.yml`; the contract tests require them to agree. Add new cache-affecting environment settings to `globalEnv`. -The former reserved routing Fleet is no longer selected by this workflow. -Keep its existing authorization until older assigned jobs finish. A manual -dispatch of `ci-routing.yml` on master exercises the hosted metadata job without -launching the test matrix; its non-PR event returns the hosted test-runner output. +The `paperclip-ci-routing` runner group must allow +`paperclipai/paperclip/.github/workflows/ci-routing.yml@refs/heads/master`. +Keep `AWS_CI_ROUTING_USER_IDS_SOURCE` equal to the exact JSON value of +`AWS_CI_TRUSTED_USER_IDS` and `AWS_CI_ROUTING_USER_IDS` as its delimiter-wrapped +numeric IDs (for example, `|34892728|139239|`). Missing or mismatched admission +settings use GitHub-hosted capacity for the gate. Current CODEOWNERS remain +authoritative for PR execution regardless of these admission settings. +A maintainer's manual dispatch of `ci-routing.yml` on master exercises the AWS +metadata gate without launching the test matrix; its non-PR event returns the +hosted test-runner output. Keep the `.github/**` rule in `.github/CODEOWNERS` and the active master ruleset's code-owner review requirement enabled. This covers the caller, the trusted @@ -369,6 +382,13 @@ config; an explicit shared test database retains one partition per job. The Runner transport lease-expiry and cold-admission regressions live in their own file so those real-clock waits overlap other files in the existing pool. +Exact-source release verification uses the post-merge AWS fleet on the first +attempt. Reruns of its verification and chaos jobs use GitHub-hosted runners. +Exact-source cloud-migrator planning and package builds use the same policy. +This gives interrupted jobs an independent recovery path. It keeps the same +source, commands, assertions and required gates. A failed test still needs a fix; +a successful rerun does not explain an earlier failure. + The two PR Runner Vitest jobs separate package tests from native server tests. The first runs the complete package inventory; the second runs the complete native server group. The existing `--shard=1/2` and `--shard=2/2` commands route @@ -1188,6 +1208,34 @@ slows `tsc` by up to 20 times when the live heap goes above the limit. Docker image sets `PAPERCLIP_TSC_EMIT_ONLY=1` for that build, which emits JavaScript without checks or declarations. +Test projects outside `pnpm -r` import server source through relative paths. Each +project's `tsconfig.layers.json` reads that source as layer declarations. This +command builds the declarations and then checks every project: + +```sh +node scripts/typecheck-test-projects.mjs +``` + +Name a project to check only that project. `pnpm test:e2e:runner:typecheck` and +`pnpm test:lifecycle-baseline:typecheck` do this for their projects. If the memory +available to the task is limited, add `--low-memory`, or set +`PAPERCLIP_TYPECHECK_LOW_MEMORY=1` for the pnpm commands. + +The command fails if a project loads server source, because that is what makes +these programs large. It does not limit memory. CI runs it with +`--reuse-declarations` straight after the server typecheck, so it does not build +the declarations again. Do not use that option when the declarations can be stale. + +Add each new test project that imports server source to `PROJECTS` in +`scripts/typecheck-test-projects.mjs`. + +Keep the project references out of each project's `tsconfig.json`. Vitest reads +that file, and it fails when a referenced config does not exist yet. The +`tsconfig.json` files remain single programs for editors, and +`tests/runner-e2e/native-completion-checks.mjs` still checks that project as one +program. That program peaks at about 15 GiB at default settings and about 8 GiB on +one thread. + #### Reused Runner build in the server build The server build vendors `packages/paperclip-runner/dist`, so its diff --git a/doc/ci-profiling.md b/doc/ci-profiling.md index 6ed462b77a2..9975cff7d3b 100644 --- a/doc/ci-profiling.md +++ b/doc/ci-profiling.md @@ -24,8 +24,13 @@ Ordinary PRs do not write timing artifacts or export traces. The exporter job is skipped for ordinary run names. Adding another label does not run the CI matrix. Profiling preserves the existing trusted AWS runner routing. Eligible PR jobs -use `paperclip-public-pr-x64`; the routing gate uses `ubuntu-latest` to verify -current CI CODEOWNERS before allocating any AWS capacity. +use `paperclip-public-pr-x64`. The synchronized maintainer admission roster +places the metadata gate on the reserved `paperclip-ci-routing-x64` AWS fleet. +The gate checks current CI CODEOWNERS before authorizing any PR code on AWS; +PR edits cannot grant access. A stale roster can allocate only the bounded +metadata gate. Normal base-branch advancement does not change an authorized +run's placement. See [Trusted PR Workflow](DEVELOPING.md#trusted-pr-workflow) +for the admission settings and immutable merge validation. The existing fail-closed GitHub-hosted fallback still applies if AWS routing cannot be authorized. The separate post-run Honeycomb exporter also uses `ubuntu-latest`; it is outside the measured PR suite. diff --git a/package.json b/package.json index d4e1c1afe35..597f549b436 100644 --- a/package.json +++ b/package.json @@ -78,7 +78,7 @@ "test:e2e:runner:browser-support": "playwright test --config tests/runner-e2e/playwright-support.config.ts", "test:e2e:runner:unit": "node tests/runner-e2e/run-unit.mjs", "test:e2e:runner:stock-harness": "node tests/runner-e2e/stock-harness-checks.mjs", - "test:e2e:runner:typecheck": "tsc -p tests/runner-e2e/tsconfig.json", + "test:e2e:runner:typecheck": "node scripts/typecheck-test-projects.mjs tests/runner-e2e", "test:e2e:runner:report": "node cli/node_modules/tsx/dist/cli.mjs tests/runner-e2e/report.ts", "test:runner-workflow-evals": "pnpm --filter @paperclipai/paperclip-eval-kernel build && pnpm --filter @paperclipai/paperclip-runner test:runner-workflow-evals", "test:e2e:mcp-user-stories": "node scripts/e2e-mcp-user-stories.mjs", @@ -94,7 +94,7 @@ "connections:ingest-app-definitions": "node scripts/ingest-app-definitions.mjs", "test:lifecycle-baseline": "node tests/lifecycle-baseline/run.mjs", "test:lifecycle-baseline:support": "node --test tests/lifecycle-baseline/report.test.mjs", - "test:lifecycle-baseline:typecheck": "tsc -p tests/lifecycle-baseline/tsconfig.json" + "test:lifecycle-baseline:typecheck": "node scripts/typecheck-test-projects.mjs tests/lifecycle-baseline" }, "devDependencies": { "@playwright/test": "^1.62.1", diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index 6b2cc7a6642..8a08d5427cd 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -216,11 +216,11 @@ test("the trusted PR workflow keeps a stable aggregate check named e2e over the } }); -test("runner authorization uses hosted capacity and executes metadata checks only", () => { +test("runner authorization uses reserved capacity and executes metadata checks only", () => { const workflow = readFileSync(routingWorkflow, "utf8"); const jobs = readWorkflowJobs(workflow.split(/^jobs:\n/m)[1]); assert.deepEqual([...jobs.keys()], ["gate"]); - assert.match(jobs.get("gate"), /^ {4}runs-on: ubuntu-latest$/m); + assert.match(jobs.get("gate"), /runs-on\/fleet=paperclip-ci-routing-x64\/env=public-ci/); assert.doesNotMatch(jobs.get("gate"), /uses:|actions\/checkout|secrets\./); assert.match(workflow, /^ {6}runner:\n {8}value: \$\{\{ jobs\.gate\.outputs\.runner \}\}$/m); assert.match(workflow, /^ {6}full_ci:\n {8}value: \$\{\{ jobs\.gate\.outputs\.full_ci \}\}$/m); diff --git a/scripts/typecheck-low-memory.mjs b/scripts/typecheck-low-memory.mjs index a4c54dce846..22d44f7b07f 100644 --- a/scripts/typecheck-low-memory.mjs +++ b/scripts/typecheck-low-memory.mjs @@ -15,7 +15,7 @@ export const STEPS = [ ["-r", "--workspace-concurrency=1", "typecheck"], ]; -function runPnpm(args, options) { +export function runPnpm(args, options) { const result = spawnSync("pnpm", args, { ...options, stdio: "inherit", diff --git a/scripts/typecheck-test-projects.mjs b/scripts/typecheck-test-projects.mjs new file mode 100644 index 00000000000..8fd8cef014f --- /dev/null +++ b/scripts/typecheck-test-projects.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node +// Typechecks the test projects outside `pnpm -r`. +// +// Each project imports server source through relative paths, so its tsconfig.json +// is one program over most of the server. Its tsconfig.layers.json reads that +// source as the declarations from server/scripts/typecheck-layers.mjs, which this +// script builds first. It then fails if a project still loads server source, +// because that is what makes these programs large. See "Typecheck memory" in +// doc/DEVELOPING.md. +// +// --reuse-declarations skips the build. Use it only straight after a server +// typecheck on the same commit, because stale declarations pass silently. +// +// Usage: node scripts/typecheck-test-projects.mjs [--low-memory] [--reuse-declarations] [project ...] + +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { LOW_MEMORY_VARIABLE, runTsc } from "./run-tsc.mjs"; +import { runPnpm } from "./typecheck-low-memory.mjs"; +import { isTestFile, typecheckLayers } from "../server/scripts/typecheck-layers.mjs"; + +export const PROJECTS = [ + "server/src/__tests__/helpers/maintenance", + "tests/lifecycle-baseline", + "tests/runner-e2e", + "tests/speko-native", +]; + +// The same steps as the server's typecheck script. +export const PREPARE_STEPS = [ + ["--filter", "@paperclipai/paperclip-runner", "run", "build:typescript"], + ["--filter", "@paperclipai/plugin-sdk", "ensure-build-deps"], +]; + +/** + * Finds the server source in a program's file list, as paths relative to the + * server. Test support files have no layer, and declaration and JSON files add + * little, so only other source files count. + */ +export function findServerSource(files, serverRoot) { + return files + .map((file) => path.relative(serverRoot, file).split(path.sep).join("/")) + .filter( + (file) => + file.startsWith("src/") && /\.[cm]?[jt]sx?$/.test(file) && !/\.d\.[cm]?ts$/.test(file) && !isTestFile(file), + ); +} + +function listProgramFiles(config, { cwd, env }) { + const tsc = path.join(cwd, "node_modules", "typescript", "bin", "tsc"); + const result = spawnSync(process.execPath, [tsc, "-p", config, "--listFilesOnly"], { + cwd, + env, + encoding: "utf8", + maxBuffer: 1 << 26, + }); + if (result.error) throw result.error; + return { status: result.status ?? 1, files: result.stdout.split("\n").filter(Boolean), output: result.stdout }; +} + +export function typecheckTestProjects({ + root, + projects = PROJECTS, + reuseDeclarations = false, + env = process.env, + pnpm = runPnpm, + run = runTsc, + listFiles = listProgramFiles, + log = console.log, + logError = console.error, +}) { + const serverRoot = path.join(root, "server"); + if (!reuseDeclarations) { + for (const args of PREPARE_STEPS) { + const status = pnpm(args, { cwd: root, env }); + if (status !== 0) return status; + } + + const layers = typecheckLayers({ + serverRoot, + declarationsOnly: true, + run: (args, options) => run(args, { ...options, env }), + log, + }); + // Report the server's own errors first. They would repeat in each project. + if (layers !== 0) return layers; + } + + let exitCode = 0; + for (const project of projects) { + log(`typecheck:test-projects: ${project}`); + const config = path.posix.join(project, "tsconfig.layers.json"); + const { status, signal } = run(["-p", config], { cwd: root, env }); + if (signal) return status || 1; + if (status !== 0) exitCode = status; + + const listing = listFiles(config, { cwd: root, env }); + if (listing.status !== 0) { + logError(`typecheck:test-projects: could not list the files of ${project}\n${listing.output ?? ""}`); + exitCode ||= listing.status; + continue; + } + const source = findServerSource(listing.files, serverRoot); + if (source.length > 0) { + logError( + `typecheck:test-projects: ${project} loads ${source.length} server source files. ` + + `It must read them as layer declarations. First files: ${source.slice(0, 5).join(", ")}`, + ); + exitCode ||= 1; + } + } + return exitCode; +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + const args = process.argv.slice(2); + const flags = args.filter((arg) => arg.startsWith("--")); + const projects = args.filter((arg) => !arg.startsWith("--")); + const unknownFlags = flags.filter((flag) => !["--low-memory", "--reuse-declarations"].includes(flag)); + const unknown = projects.filter((project) => !PROJECTS.includes(project)); + if (unknownFlags.length > 0) { + console.error(`Unknown option: ${unknownFlags.join(", ")}`); + process.exitCode = 2; + } else if (unknown.length > 0) { + console.error(`Unknown project: ${unknown.join(", ")}. Known projects: ${PROJECTS.join(", ")}`); + process.exitCode = 2; + } else { + process.exitCode = typecheckTestProjects({ + root: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + projects: projects.length > 0 ? projects : PROJECTS, + reuseDeclarations: flags.includes("--reuse-declarations"), + env: args.includes("--low-memory") ? { ...process.env, [LOW_MEMORY_VARIABLE]: "1" } : process.env, + }); + } +} diff --git a/server/scripts/typecheck-layers.mjs b/server/scripts/typecheck-layers.mjs index 99cfb37bfb6..95b0f22e9ee 100644 --- a/server/scripts/typecheck-layers.mjs +++ b/server/scripts/typecheck-layers.mjs @@ -20,7 +20,11 @@ // server/tsconfig.json stays the single program that editors and `pnpm build` // use. // -// Usage: node scripts/typecheck-layers.mjs +// Projects outside the server read the source layers as declarations through a +// `tsconfig.layers.json` that references tsconfig.declarations.json in the output +// directory. --declarations-only builds those layers and skips the tests layer. +// +// Usage: node scripts/typecheck-layers.mjs [--declarations-only] import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; @@ -30,7 +34,7 @@ import { runTsc } from "../../scripts/run-tsc.mjs"; const IMPORT_SPECIFIER = /(?:from\s+|import\s*\(\s*|import\s+)["'](\.[^"']+)["']/g; const OUTPUT_DIRECTORY = path.join("node_modules", ".cache", "paperclip-typecheck"); -const isTestFile = (file) => /\.test\.tsx?$/.test(file) || file.split("/").includes("__tests__"); +export const isTestFile = (file) => /\.test\.tsx?$/.test(file) || file.split("/").includes("__tests__"); // tsconfig.json excludes this tree. A test reaches its helpers through imports. const isExcludedFromProgram = (file) => file.startsWith("src/__tests__/"); @@ -212,8 +216,20 @@ export function layerConfig(layer, { emptyLayers = new Set() } = {}) { }; } +/** Builds the config other projects reference to read the source layers as declarations. */ +export function declarationsConfig(layers) { + return { + compilerOptions: { composite: true }, + files: [], + references: layers + .filter((layer) => layer.name !== "tests") + .map((layer) => ({ path: `./tsconfig.${layer.name}.json` })), + }; +} + export function typecheckLayers({ serverRoot, + declarationsOnly = false, run = runTsc, log = console.log, }) { @@ -236,8 +252,14 @@ export function typecheckLayers({ ); } + writeFileSync( + path.join(outputRoot, "tsconfig.declarations.json"), + `${JSON.stringify(declarationsConfig(layers), null, 2)}\n`, + ); + let exitCode = 0; for (const layer of layers) { + if (declarationsOnly && layer.name === "tests") continue; log(`typecheck:layers: ${layer.name} (${layer.sourceFiles.length} files)`); const config = path.join(OUTPUT_DIRECTORY, `tsconfig.${layer.name}.json`); const { status, signal } = run(["-p", config], { cwd: serverRoot }); @@ -253,5 +275,6 @@ export function typecheckLayers({ if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { process.exitCode = typecheckLayers({ serverRoot: path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."), + declarationsOnly: process.argv.includes("--declarations-only"), }); } diff --git a/server/scripts/typecheck-layers.test.mjs b/server/scripts/typecheck-layers.test.mjs index eec3f2171e8..82b782e3391 100644 --- a/server/scripts/typecheck-layers.test.mjs +++ b/server/scripts/typecheck-layers.test.mjs @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -8,7 +8,8 @@ import { afterEach, describe, expect, it } from "vitest"; import { emitOnlyTscArgs, lowMemoryTscArgs, runTsc } from "../../scripts/run-tsc.mjs"; import { STEPS, typecheckLowMemory } from "../../scripts/typecheck-low-memory.mjs"; -import { layerConfig, listSourceFiles, planLayers, readImports, typecheckLayers } from "./typecheck-layers.mjs"; +import { findServerSource, PREPARE_STEPS, PROJECTS, typecheckTestProjects } from "../../scripts/typecheck-test-projects.mjs"; +import { declarationsConfig, layerConfig, listSourceFiles, planLayers, readImports, typecheckLayers } from "./typecheck-layers.mjs"; const serverRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const sourcesOf = (layers, name) => layers.find((layer) => layer.name === name)?.sourceFiles; @@ -159,6 +160,16 @@ describe("layerConfig", () => { }); }); +describe("declarationsConfig", () => { + it("references the source layers that exist, and not tests", () => { + const config = declarationsConfig([{ name: "below" }, { name: "above" }, { name: "tests" }]); + + expect(config.compilerOptions.composite).toBe(true); + expect(config.files).toEqual([]); + expect(config.references).toEqual([{ path: "./tsconfig.below.json" }, { path: "./tsconfig.above.json" }]); + }); +}); + describe("typecheckLayers", () => { const roots = []; @@ -202,6 +213,30 @@ describe("typecheckLayers", () => { expect(JSON.parse(readFileSync(path.join(output, "package.json"), "utf8"))).toEqual({ type: "module" }); }); + it("skips the tests layer when only declarations are needed", () => { + const root = serverFixture(); + const configs = []; + + const status = typecheckLayers({ + serverRoot: root, + declarationsOnly: true, + log: () => {}, + run: (args) => { + configs.push(path.basename(args[1])); + return { status: 0, signal: null }; + }, + }); + + expect(status).toBe(0); + expect(configs).toEqual(["tsconfig.below.json", "tsconfig.cycle.json", "tsconfig.above.json"]); + const output = path.join(root, "node_modules", ".cache", "paperclip-typecheck"); + expect(JSON.parse(readFileSync(path.join(output, "tsconfig.declarations.json"), "utf8")).references).toEqual([ + { path: "./tsconfig.below.json" }, + { path: "./tsconfig.cycle.json" }, + { path: "./tsconfig.above.json" }, + ]); + }); + it("reports type errors from every layer", () => { const root = serverFixture(); const configs = []; @@ -308,6 +343,57 @@ describe("typecheckLayers with the real compiler", () => { expect(existsSync(path.join(declarations, "route.test.d.ts"))).toBe(false); }, 60_000); + it("lets a project outside the server read the source layers as declarations", () => { + const root = compilerFixture(); + const outside = path.join(root, "outside"); + mkdirSync(outside); + writeFileSync(path.join(outside, "use.ts"), `import { route } from "../src/route.js";\nexport const used: number = route();\n`); + const config = path.join(outside, "tsconfig.layers.json"); + writeFileSync( + config, + `${JSON.stringify({ + extends: "../tsconfig.json", + compilerOptions: { noEmit: true, rootDir: "..", outDir: null }, + include: ["*.ts"], + references: [{ path: "../node_modules/.cache/paperclip-typecheck/tsconfig.declarations.json" }], + })}\n`, + ); + const buildDeclarations = () => + typecheckLayers({ + serverRoot: root, + declarationsOnly: true, + log: () => {}, + run: (args, { cwd }) => { + const result = spawnSync(process.execPath, [tsc, ...args], { cwd, encoding: "utf8" }); + return { status: result.status ?? 1, signal: result.signal }; + }, + }); + const tscOutside = (...args) => spawnSync(process.execPath, [tsc, "-p", config, ...args], { cwd: root, encoding: "utf8" }); + + expect(buildDeclarations()).toBe(0); + const passing = tscOutside(); + expect(passing.stdout).toBe(""); + expect(passing.status).toBe(0); + const loaded = tscOutside("--listFilesOnly").stdout; + expect(loaded).toContain("paperclip-typecheck/dts/route.d.ts"); + expect(loaded).not.toContain("src/route.ts"); + expect(findServerSource(loaded.split("\n").filter(Boolean), root)).toEqual([]); + + const layered = readFileSync(config, "utf8"); + const { references: _references, ...withoutReferences } = JSON.parse(layered); + writeFileSync(config, `${JSON.stringify(withoutReferences)}\n`); + const allSource = tscOutside("--listFilesOnly").stdout.split("\n").filter(Boolean); + expect(findServerSource(allSource, root)).toContain("src/route.ts"); + writeFileSync(config, layered); + + writeFileSync(path.join(root, "src", "route.ts"), `import { one } from "./one.js";\nexport const route = () => String(one());\n`); + expect(buildDeclarations()).toBe(0); + const failing = tscOutside(); + expect(failing.stdout).toContain("outside/use.ts"); + expect(failing.stdout).toContain("TS2322"); + expect(failing.status).not.toBe(0); + }, 60_000); + it("reports a changed export in the layers that depend on it", () => { const root = compilerFixture(); writeFileSync(path.join(root, "src", "leaf.ts"), `export const renamedLeaf = 1;\n`); @@ -404,6 +490,203 @@ describe("typecheckLowMemory", () => { }); }); +describe("typecheckTestProjects", () => { + const roots = []; + + afterEach(() => { + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); + }); + + function repositoryFixture() { + const root = mkdtempSync(path.join(tmpdir(), "typecheck-test-projects-")); + roots.push(root); + mkdirSync(path.join(root, "server", "src"), { recursive: true }); + writeFileSync(path.join(root, "server", "src", "leaf.ts"), ""); + writeFileSync(path.join(root, "server", "src", "one.ts"), `import "./two.js"; import "./leaf.js";`); + writeFileSync(path.join(root, "server", "src", "two.ts"), `import "./one.js";`); + writeFileSync(path.join(root, "server", "src", "route.ts"), `import "./one.js";`); + writeFileSync(path.join(root, "server", "src", "route.test.ts"), `import "./route.js";`); + return root; + } + + function runWith( + root, + { + pnpmStatus = () => 0, + tscResult = () => ({ status: 0, signal: null }), + listFiles = () => ({ status: 0, files: [] }), + ...options + } = {}, + ) { + const calls = []; + const errors = []; + const status = typecheckTestProjects({ + root, + listFiles, + log: () => {}, + logError: (message) => errors.push(message), + pnpm: (args, { env }) => { + calls.push({ tool: "pnpm", args, env }); + return pnpmStatus(args); + }, + run: (args, { env }) => { + const target = path.basename(path.dirname(args[1])) === "paperclip-typecheck" ? path.basename(args[1]) : args[1]; + calls.push({ tool: "tsc", target, env }); + return tscResult(target); + }, + ...options, + }); + return { status, calls, errors }; + } + + it("prepares, builds declarations without the tests layer, and then checks the projects", () => { + const env = { MARKER: "1" }; + + const { status, calls } = runWith(repositoryFixture(), { projects: ["tests/a", "tests/b"], env }); + + expect(status).toBe(0); + expect(calls.filter((call) => call.tool === "pnpm").map((call) => call.args)).toEqual(PREPARE_STEPS); + expect(calls.filter((call) => call.tool === "tsc").map((call) => call.target)).toEqual([ + "tsconfig.below.json", + "tsconfig.cycle.json", + "tsconfig.above.json", + "tests/a/tsconfig.layers.json", + "tests/b/tsconfig.layers.json", + ]); + expect(calls.every((call) => call.env === env)).toBe(true); + }); + + it("stops when the preparation or the server layers fail", () => { + const prepare = runWith(repositoryFixture(), { pnpmStatus: () => 3 }); + expect(prepare.status).toBe(3); + expect(prepare.calls).toHaveLength(1); + + const layers = runWith(repositoryFixture(), { + projects: ["tests/a"], + tscResult: (target) => ({ status: target === "tsconfig.cycle.json" ? 2 : 0, signal: null }), + }); + expect(layers.status).toBe(2); + expect(layers.calls.some((call) => call.target === "tests/a/tsconfig.layers.json")).toBe(false); + }); + + it("checks every project after a type error, and stops when a process is killed", () => { + const errors = runWith(repositoryFixture(), { + projects: ["tests/a", "tests/b"], + tscResult: (target) => ({ status: target === "tests/a/tsconfig.layers.json" ? 2 : 0, signal: null }), + }); + expect(errors.status).toBe(2); + expect(errors.calls.at(-1).target).toBe("tests/b/tsconfig.layers.json"); + + const killed = runWith(repositoryFixture(), { + projects: ["tests/a", "tests/b"], + tscResult: (target) => + target === "tests/a/tsconfig.layers.json" ? { status: 1, signal: "SIGKILL" } : { status: 0, signal: null }, + }); + expect(killed.status).toBe(1); + expect(killed.calls.at(-1).target).toBe("tests/a/tsconfig.layers.json"); + }); + + it("reuses declarations without preparing or building layers", () => { + const { status, calls } = runWith(repositoryFixture(), { projects: ["tests/a"], reuseDeclarations: true }); + + expect(status).toBe(0); + expect(calls.map((call) => call.target ?? call.tool)).toEqual(["tests/a/tsconfig.layers.json"]); + }); + + it("fails when a project loads server source, even if its types pass", () => { + const root = repositoryFixture(); + const server = path.join(root, "server"); + + const { status, errors } = runWith(root, { + projects: ["tests/a", "tests/b"], + reuseDeclarations: true, + listFiles: (config) => ({ + status: 0, + files: config.startsWith("tests/a/") ? [path.join(server, "src", "route.ts")] : [], + }), + }); + + expect(status).toBe(1); + expect(errors).toHaveLength(1); + expect(errors[0]).toContain("tests/a loads 1 server source files"); + expect(errors[0]).toContain("src/route.ts"); + }); + + it("fails when a project's files cannot be listed, and keeps a type error's status", () => { + const listing = runWith(repositoryFixture(), { + projects: ["tests/a"], + reuseDeclarations: true, + listFiles: () => ({ status: 3, files: [], output: "TS6053" }), + }); + expect(listing.status).toBe(3); + expect(listing.errors[0]).toContain("TS6053"); + + const both = runWith(repositoryFixture(), { + projects: ["tests/a"], + reuseDeclarations: true, + tscResult: () => ({ status: 2, signal: null }), + listFiles: () => ({ status: 3, files: [], output: "" }), + }); + expect(both.status).toBe(2); + }); + + it("counts only source files that a layer should provide", () => { + const server = path.join(path.sep, "repo", "server"); + const file = (relative) => path.join(server, ...relative.split("/")); + + expect( + findServerSource( + [ + file("src/route.ts"), + file("src/view.tsx"), + file("src/legacy.js"), + file("src/types/express.d.ts"), + file("src/data.json"), + file("src/route.test.ts"), + file("src/__tests__/helpers/home.ts"), + file("node_modules/.cache/paperclip-typecheck/dts/route.d.ts"), + file("test-baselines/lifecycle.test.ts"), + path.join(path.sep, "repo", "tests", "a.ts"), + ], + server, + ), + ).toEqual(["src/route.ts", "src/view.tsx", "src/legacy.js"]); + }); + + it("covers every project under tests/, or names why it does not", () => { + // This project imports one small server file, so it stays one small program. + const singlePrograms = ["tests/runner-acceptance"]; + const testsRoot = path.join(serverRoot, "..", "tests"); + const projects = readdirSync(testsRoot, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && existsSync(path.join(testsRoot, entry.name, "tsconfig.json"))) + .map((entry) => `tests/${entry.name}`); + + expect(projects.sort()).toEqual( + [...PROJECTS.filter((project) => project.startsWith("tests/")), ...singlePrograms].sort(), + ); + }); + + it("prepares the same way as the server typecheck", () => { + const { scripts } = JSON.parse(readFileSync(path.join(serverRoot, "package.json"), "utf8")); + for (const step of PREPARE_STEPS) { + expect(scripts.typecheck).toContain(`pnpm ${step.join(" ")}`); + } + }); + + it.each(PROJECTS)("keeps the generated references out of the tsconfig.json of %s", (project) => { + const directory = path.join(serverRoot, "..", project); + const readConfig = (name) => JSON.parse(readFileSync(path.join(directory, name), "utf8")); + + // Vitest reads tsconfig.json and fails when a referenced config does not exist yet. + expect(readConfig("tsconfig.json").references).toBeUndefined(); + const layered = readConfig("tsconfig.layers.json"); + expect(layered.extends).toBe("./tsconfig.json"); + expect(layered.references.map((reference) => path.resolve(directory, reference.path))).toEqual([ + path.join(serverRoot, "node_modules", ".cache", "paperclip-typecheck", "tsconfig.declarations.json"), + ]); + }); +}); + describe("runTsc", () => { it("emits JavaScript without checks or declarations in emit-only mode", () => { const root = mkdtempSync(path.join(tmpdir(), "run-tsc-emit-only-")); diff --git a/server/src/__tests__/heartbeat-workspace-busy.test.ts b/server/src/__tests__/heartbeat-workspace-busy.test.ts index 55649bc9ba8..c01ecb082b1 100644 --- a/server/src/__tests__/heartbeat-workspace-busy.test.ts +++ b/server/src/__tests__/heartbeat-workspace-busy.test.ts @@ -209,33 +209,14 @@ describeEmbeddedPostgres("shared-workspace run serialization", () => { await db.delete(companies); } - async function waitForRunToLeaveActiveStates(runId: string, timeoutMs = 10_000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const run = await heartbeat.getRun(runId); - if (run && !["queued", "running"].includes(run.status)) return run; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - return await heartbeat.getRun(runId); + async function waitForRunToLeaveActiveStates(runId: string) { + await heartbeat.drainActiveRunExecutions(); + return heartbeat.getRun(runId); } - // A deferral does two writes in order: first it cancels the original run, - // then it inserts the scheduled-retry row. waitForRunToLeaveActiveStates - // returns after the first write, so a read of the retry row can land before - // the second write and find nothing. Poll until the retry row exists so the - // retry-row assertions never observe the gap between the two writes. - async function waitForRetryRun(originalRunId: string, timeoutMs = 10_000) { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - const retryRun = await db - .select() - .from(heartbeatRuns) - .where(eq(heartbeatRuns.retryOfRunId, originalRunId)) - .then((rows) => rows[0] ?? null); - if (retryRun) return retryRun; - await new Promise((resolve) => setTimeout(resolve, 50)); - } - return await db + async function waitForRetryRun(originalRunId: string) { + await heartbeat.drainActiveRunExecutions(); + return db .select() .from(heartbeatRuns) .where(eq(heartbeatRuns.retryOfRunId, originalRunId)) diff --git a/server/src/__tests__/helpers/maintenance/tsconfig.layers.json b/server/src/__tests__/helpers/maintenance/tsconfig.layers.json new file mode 100644 index 00000000000..589c934fec9 --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/tsconfig.layers.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "declarationMap": false + }, + "references": [{ "path": "../../../../node_modules/.cache/paperclip-typecheck/tsconfig.declarations.json" }] +} diff --git a/server/src/services/heartbeat/workspace-preparation.test.ts b/server/src/services/heartbeat/workspace-preparation.test.ts index 971ec4f4d0b..8a1d010f8a7 100644 --- a/server/src/services/heartbeat/workspace-preparation.test.ts +++ b/server/src/services/heartbeat/workspace-preparation.test.ts @@ -70,11 +70,11 @@ describe.skipIf(!support.supported && !externalDatabaseUrl)("heartbeat workspace if (external) await db.execute(sql`truncate table environments restart identity cascade`); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { - await db?.$client.end(); + await db?.$client.end({ timeout: 1 }); await database?.cleanup(); vi.unstubAllEnvs(); if (home) await rm(home, { recursive: true, force: true }); - }); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); beforeEach(async () => { persistenceBefore = new Map(persistenceKeys.map(key => [key, process.env[key]])); vi.restoreAllMocks(); diff --git a/tests/lifecycle-baseline/README.md b/tests/lifecycle-baseline/README.md index 1e9136eda3d..094b32222e4 100644 --- a/tests/lifecycle-baseline/README.md +++ b/tests/lifecycle-baseline/README.md @@ -47,7 +47,7 @@ pnpm test:lifecycle-baseline integration pnpm test:lifecycle-baseline grading pnpm test:lifecycle-baseline pnpm test:lifecycle-baseline:support -pnpm exec tsc -p tests/lifecycle-baseline/tsconfig.json +pnpm test:lifecycle-baseline:typecheck ``` All four lanes are credential-free. The integration lane uses disposable embedded diff --git a/tests/lifecycle-baseline/tsconfig.json b/tests/lifecycle-baseline/tsconfig.json index 0fa94c55b6f..e6a31f8654b 100644 --- a/tests/lifecycle-baseline/tsconfig.json +++ b/tests/lifecycle-baseline/tsconfig.json @@ -7,5 +7,5 @@ "types": ["node"], "typeRoots": ["../../cli/node_modules/@types", "../../server/node_modules/@types"] }, - "include": ["./*.ts", "../../server/test-baselines/*.ts"] + "include": ["./*.ts", "../../server/test-baselines/*.ts", "../../server/src/types/express.d.ts"] } diff --git a/tests/lifecycle-baseline/tsconfig.layers.json b/tests/lifecycle-baseline/tsconfig.layers.json new file mode 100644 index 00000000000..f1c68632282 --- /dev/null +++ b/tests/lifecycle-baseline/tsconfig.layers.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "declarationMap": false + }, + "references": [{ "path": "../../server/node_modules/.cache/paperclip-typecheck/tsconfig.declarations.json" }] +} diff --git a/tests/runner-e2e/tsconfig.layers.json b/tests/runner-e2e/tsconfig.layers.json new file mode 100644 index 00000000000..f1c68632282 --- /dev/null +++ b/tests/runner-e2e/tsconfig.layers.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "declarationMap": false + }, + "references": [{ "path": "../../server/node_modules/.cache/paperclip-typecheck/tsconfig.declarations.json" }] +} diff --git a/tests/speko-native/tsconfig.layers.json b/tests/speko-native/tsconfig.layers.json new file mode 100644 index 00000000000..f1c68632282 --- /dev/null +++ b/tests/speko-native/tsconfig.layers.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "declaration": false, + "declarationMap": false + }, + "references": [{ "path": "../../server/node_modules/.cache/paperclip-typecheck/tsconfig.declarations.json" }] +}