diff --git a/.github/scripts/tests/ci-plan.test.mjs b/.github/scripts/tests/ci-plan.test.mjs index bd50fa52559..da6dc6955fa 100644 --- a/.github/scripts/tests/ci-plan.test.mjs +++ b/.github/scripts/tests/ci-plan.test.mjs @@ -18,7 +18,8 @@ test("trusted workflow fallbacks preserve every declared partition and command", const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); for (const [job, entries] of Object.entries(lanes)) { const block = workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0]; - assert.ok(block.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job); + const normalized = block.replaceAll("(needs.test_selection.outputs.plan || needs.gate.outputs.plan)", "needs.gate.outputs.plan"); + assert.ok(normalized.includes(`fromJSON(needs.gate.outputs.plan).jobs.${job}.run`), job); if (!block.includes(" matrix:")) { assert.equal(entries.length, 1); continue; } const fallback = block.match(/matrix:.*\|\| '(.*)'\) \}\}/)?.[1]; assert.ok(fallback, `${job}: missing full fallback`); @@ -115,7 +116,7 @@ test("cache-affecting definition changes force execution across successive PR pu } } const workflow = readFileSync(new URL(".github/workflows/pr-trusted.yml", root), "utf8"); - assert.equal((workflow.match(/force:.*fromJSON\(needs.gate.outputs.plan\).forceReason != null/g) ?? []).length, 3); + assert.equal((workflow.match(/force:.*\.forceReason != null/g) ?? []).length, 3); const action = readFileSync(new URL(".github/actions/run-ci-task/action.yml", root), "utf8"); assert.match(action, /CI_FORCE_TASKS: \$\{\{ inputs.force \}\}/); assert.equal((action.match(/if:.*inputs.force != 'true'/g) ?? []).length, 2); @@ -169,6 +170,9 @@ test("required aggregates reject omitted, failed and cancelled selected work", ( env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(resultPlan), CI_RESULTS: JSON.stringify(results) } }); assert.equal(run().status, 0); for (const name of names) { + // Optional analysis can fail or be skipped; the unchanged task plan + // still has to pass. Selected-plan cases are exercised separately. + if (name === "test_selection") continue; const previous = results[name].result; for (const bad of previous === "success" ? ["failure", "cancelled", "skipped"] : ["success"]) { results[name].result = bad; diff --git a/.github/workflows/ci-incremental-qualification.yml b/.github/workflows/ci-incremental-qualification.yml index ee423086c0c..bce43fc09ab 100644 --- a/.github/workflows/ci-incremental-qualification.yml +++ b/.github/workflows/ci-incremental-qualification.yml @@ -22,6 +22,50 @@ concurrency: cancel-in-progress: true jobs: + qualify_test_selection: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + with: + version: 9.15.4 + - uses: ./.github/actions/setup-ci-dependencies + - name: Prove conservative graphs, exact partitions, manifests and failure fallbacks + run: node --test scripts/ci/test-selection*.test.mjs scripts/ci/vitest-impact-report.integration.test.mjs + - name: Qualify selection against the real Paperclip workspace + env: + NODE_ENV: test + VITEST: 'true' + run: | + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + import { readFileSync } from 'node:fs'; + import { execFileSync } from 'node:child_process'; + import { planTasks } from './.github/scripts/ci-plan.mjs'; + import { createTestSelection } from './scripts/ci/select-tests.mjs'; + const config = JSON.parse(readFileSync('turbo.json', 'utf8')); + const lanes = JSON.parse(readFileSync('.github/ci-task-matrix.json', 'utf8')); + const head = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim(); + for (const [file, project, related] of [ + ['ui/src/pages/Inbox.tsx', '@paperclipai/ui', 'ui/src/pages/Inbox.test.tsx'], + ['server/src/middleware/auth.ts', '@paperclipai/server', 'server/src/__tests__/agent-auth-middleware.test.ts'], + ]) { + const selected = await createTestSelection(planTasks({ config, lanes, paths: [file] }), head); + const graph = selected.graphs.find(graph => graph.project === project); + assert.ok(graph.related.includes(related)); + assert.ok(graph.alwaysRun.length > 0, 'Filesystem contracts must remain covered'); + assert.ok(graph.selected.length < graph.inventory.length, 'The real workspace must support narrowing'); + console.log(file, JSON.stringify(selected.plan.testSelection.stats, null, 2)); + } + NODE + qualify: runs-on: ubuntu-latest timeout-minutes: 10 diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index 8765fbfb601..7c46153bb48 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -24,6 +24,83 @@ jobs: gate: uses: paperclipai/paperclip/.github/workflows/ci-routing.yml@master + test_selection: + name: Select Vitest files + needs: [gate] + if: >- + needs.gate.outputs.full_ci == 'true' && needs.gate.outputs.plan != '' && + fromJSON(needs.gate.outputs.plan).mode == 'enabled' && + fromJSON(needs.gate.outputs.plan).forceReason == null && + (vars.CI_VITEST_SELECTION == '' || vars.CI_VITEST_SELECTION == 'enabled') + runs-on: ${{ needs.gate.outputs.runner }} + timeout-minutes: 10 + # An unavailable optimization must not suppress the original test matrix. + continue-on-error: true + outputs: + plan: ${{ steps.publish.outputs.plan }} + env: + CI_PLAN: ${{ needs.gate.outputs.plan }} + BASE_SHA: ${{ github.event.pull_request.base.sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + with: + persist-credentials: false + fetch-depth: 0 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 + with: + node-version: 24 + package-manager-cache: false + - name: Check source-diff eligibility + id: eligible + run: | + if [[ -f scripts/ci/select-tests.mjs ]]; then + node scripts/ci/select-tests.mjs --eligible + else + echo 'enabled=false' >> "$GITHUB_OUTPUT" + fi + - uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 + if: steps.eligible.outputs.enabled == 'true' + with: + version: 9.15.4 + - name: Restore dependencies for graph analysis + if: steps.eligible.outputs.enabled == 'true' + uses: ./.github/actions/setup-ci-dependencies + - name: Select files and rebalance test lanes + id: select + if: steps.eligible.outputs.enabled == 'true' + timeout-minutes: 5 + run: node scripts/ci/select-tests.mjs + - name: Upload validated selection and audit + id: upload_selection + if: steps.select.outputs.selected == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a + with: + name: vitest-selection-${{ github.run_attempt }} + path: | + .ci-test-selection/manifest.json + .ci-test-selection/audit.json + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + # Publish only after upload succeeds. Missing/failed/timed-out selectors + # produce no plan, and dependent jobs use the original gate's matrix. + - name: Publish the executable plan + id: publish + if: steps.eligible.outputs.enabled == 'true' + env: + ARTIFACT_ID: ${{ steps.upload_selection.outputs.artifact-id }} + run: | + node --input-type=module <<'NODE' + import assert from 'node:assert/strict'; + import { appendFileSync, readFileSync } from 'node:fs'; + const plan = JSON.parse(readFileSync('.ci-test-selection/plan.json', 'utf8')); + if (plan.testSelection) { + assert.match(process.env.ARTIFACT_ID ?? '', /^[0-9]+$/); + plan.testSelection.artifactId = process.env.ARTIFACT_ID; + } + appendFileSync(process.env.GITHUB_OUTPUT, `plan=${JSON.stringify(plan)}\n`); + NODE + policy: needs: [gate] runs-on: ${{ needs.gate.outputs.runner }} @@ -358,15 +435,17 @@ jobs: general_tests: name: General tests (${{ matrix.group_label }}) - needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.general_tests.run) }} + needs: [gate, test_selection] + if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false - matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }} + matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.general_tests.matrix) || '{"include":[{"group":"general-server-without-chat","group_label":"server (1/12)","shard_index":0,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (2/12)","shard_index":1,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (3/12)","shard_index":2,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (4/12)","shard_index":3,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (5/12)","shard_index":4,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (6/12)","shard_index":5,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (7/12)","shard_index":6,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (8/12)","shard_index":7,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (9/12)","shard_index":8,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (10/12)","shard_index":9,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (11/12)","shard_index":10,"shard_count":12,"task":"ci:server"},{"group":"general-server-without-chat","group_label":"server (12/12)","shard_index":11,"shard_count":12,"task":"ci:server"},{"group":"general-chat","group_label":"chat (1/3)","shard_index":0,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (2/3)","shard_index":1,"shard_count":3,"task":"ci:chat"},{"group":"general-chat","group_label":"chat (3/3)","shard_index":2,"shard_count":3,"task":"ci:chat"},{"group":"general-workspaces-a","group_label":"workspaces-a (1/2)","shard_index":0,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-a","group_label":"workspaces-a (2/2)","shard_index":1,"shard_count":2,"task":"ci:workspaces-a"},{"group":"general-workspaces-b","group_label":"workspaces-b","task":"ci:workspaces-b"},{"group":"general-workspaces-c","group_label":"workspaces-c","task":"ci:db"}]}') }} env: + PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }} + PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }} PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} PAPERCLIP_CI_PROFILE_JOB: General tests (${{ matrix.group_label }}) @@ -376,6 +455,13 @@ jobs: with: persist-credentials: false + - name: Download selected test files + if: matrix.selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }} + path: .ci-test-selection + - name: Initialize CI profile continue-on-error: true if: env.PAPERCLIP_CI_PROFILE == 'true' @@ -438,7 +524,7 @@ jobs: task: ${{ matrix.task }} args: ${{ matrix.shard_count && format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) || '[]' }} partition: ${{ strategy.job-index || 0 }} - force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }} + force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }} - name: Run grouped general test suites if: hashFiles('turbo.json') == '' @@ -521,14 +607,14 @@ jobs: verify: name: verify if: ${{ always() }} - needs: [gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run] + needs: [gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run] runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 5 steps: - name: Validate every planned suite result env: FULL_CI: ${{ needs.gate.outputs.full_ci }} - CI_PLAN: ${{ needs.gate.outputs.plan }} + CI_PLAN: ${{ needs.test_selection.outputs.plan || needs.gate.outputs.plan }} CI_RESULTS: ${{ toJSON(needs) }} run: | node --input-type=module <<'NODE' @@ -539,6 +625,7 @@ jobs: assert.ok(['true', 'false'].includes(process.env.FULL_CI), 'Valid scope required'); const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null; if (plan) assert.equal(plan.version, 1, 'Supported task plan required'); + if (plan?.testSelection) assert.equal(results.test_selection.result, 'success', 'Published selection must succeed'); for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","verify_serialized_server","canary_dry_run"]) { if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job); const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run); @@ -1021,15 +1108,17 @@ jobs: verify_serialized_server: name: Verify serialized server suites (${{ matrix.shard_label }}) - needs: [gate] - if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.run) }} + needs: [gate, test_selection] + if: ${{ always() && !cancelled() && needs.gate.result == 'success' && needs.gate.outputs.full_ci == 'true' && ((needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.run) }} runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 20 strategy: fail-fast: false - matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }} + matrix: ${{ fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan) != '' && toJSON(fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).jobs.verify_serialized_server.matrix) || '{"include":[{"shard_index":0,"shard_count":9,"shard_label":"1/9","task":"ci:serialized"},{"shard_index":1,"shard_count":9,"shard_label":"2/9","task":"ci:serialized"},{"shard_index":2,"shard_count":9,"shard_label":"3/9","task":"ci:serialized"},{"shard_index":3,"shard_count":9,"shard_label":"4/9","task":"ci:serialized"},{"shard_index":4,"shard_count":9,"shard_label":"5/9","task":"ci:serialized"},{"shard_index":5,"shard_count":9,"shard_label":"6/9","task":"ci:serialized"},{"shard_index":6,"shard_count":9,"shard_label":"7/9","task":"ci:serialized"},{"shard_index":7,"shard_count":9,"shard_label":"8/9","task":"ci:serialized"},{"shard_index":8,"shard_count":9,"shard_label":"9/9","task":"ci:serialized"}]}') }} env: + PAPERCLIP_CI_TEST_SELECTION: ${{ matrix.selection && '.ci-test-selection/manifest.json' || '' }} + PAPERCLIP_CI_SELECTION_HASH: ${{ matrix.selection && fromJSON(needs.test_selection.outputs.plan).testSelection.hash || '' }} PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} PAPERCLIP_CI_PROFILE_JOB: Verify serialized server suites (${{ matrix.shard_label }}) @@ -1039,6 +1128,13 @@ jobs: with: persist-credentials: false + - name: Download selected test files + if: matrix.selection + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c + with: + artifact-ids: ${{ fromJSON(needs.test_selection.outputs.plan).testSelection.artifactId }} + path: .ci-test-selection + - name: Initialize CI profile continue-on-error: true if: env.PAPERCLIP_CI_PROFILE == 'true' @@ -1101,7 +1197,7 @@ jobs: task: ci:serialized args: ${{ format('["--shard-index","{0}","--shard-count","{1}"]', matrix.shard_index, matrix.shard_count) }} partition: ${{ strategy.job-index || 0 }} - force: ${{ needs.gate.outputs.profile_ci == 'true' || needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).mode != 'enabled' || fromJSON(needs.gate.outputs.plan).forceReason != null }} + force: ${{ needs.gate.outputs.profile_ci == 'true' || (needs.test_selection.outputs.plan || needs.gate.outputs.plan) == '' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).mode != 'enabled' || fromJSON((needs.test_selection.outputs.plan || needs.gate.outputs.plan)).forceReason != null }} - name: Run serialized server test shard if: hashFiles('turbo.json') == '' diff --git a/.gitignore b/.gitignore index 91d676637d4..afa5f1c5db6 100644 --- a/.gitignore +++ b/.gitignore @@ -99,6 +99,7 @@ test-results/ # Opt-in CI timing artifacts .ci-profile/ +.ci-test-selection/ # Content-addressed CI task results .turbo/ diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 0f384a4bbe9..7e53190d2c1 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -193,6 +193,60 @@ canary dry run. The uncached typecheck job also runs the repository-wide CLI guidance safety guard. It still scans UI changes when unrelated server shards are omitted. +For ordinary source-only PRs, a separate `Select Vitest files` job refines the +general and serialized test lanes before their runners are allocated. It runs +on the gate-approved test fleet, loads the real Vitest projects, and follows +imports across workspace packages. Only projects with their own source changes +are narrowed; unchanged consumer projects retain their complete coarse lane +coverage without paying for another graph traversal. Filesystem, subprocess, network, CommonJS, +dynamic-loader, unresolved, and unapproved external-package consumers are +retained conservatively, including indirect consumers through local helpers. +The selector explicitly tracks setup/configuration dependencies as well as test +imports. The small pure-dependency allowlist lives in +`scripts/ci/vitest-selection-graph.mjs`; new external packages remain opaque +until reviewed. This is deliberately broader than `vitest related` alone. +For conservative retention, explicit complete local mock factories cut the +mocked implementation's I/O edges. Partial mocks, spies, automocks, mock bypasses, +and dynamic or escaped mocking APIs do not. Edits to a mocked module still use +Vitest's unmodified related-file graph. Non-isolated projects run in full. +Literal Vite globs use Vite's expanded dependency edges, including newly added +members. Consumers of generated or untracked modules remain in the conservative +set because the import graph does not describe their build inputs. +The server's temporary-home and Supertest setup has a reviewed, content-hashed +exception. It still follows local helper imports. Any change to the reviewed +contents disables the exception until reviewed again; setup dependency changes +invalidate the whole project. Unreviewed setup I/O uses the full task plan. + +Selected files are repartitioned within their existing owners using the current +per-runner budgets and server timing weights. Small selections allocate fewer +jobs; empty lanes allocate none. Serialized files remain serialized, and chat's +line shards, native Runner lanes, Rust, builds, typechecking and E2E retain their +existing selection/execution rules. The job summary reports files and jobs +selected per lane, and the `vitest-selection` artifact records the manifest and +dependency audit. Selected jobs execute exact Vitest test specifications, so +similarly named `.test.ts` and `.test.tsx` files cannot overlap between shards. + +Missing/deleted/renamed files, non-source inputs, unrecognized graph inputs, +setup/config changes, failed dependency installs, graph errors and timeouts use +the complete original task plan. Source mutations during dependency setup also +disable narrowing. A published manifest must match the checked-out +merge SHA, its plan hash and the expected shard/project ownership. Missing or +invalid manifests fail selected jobs and the required aggregate; they cannot +turn reduced matrices green. Failed-job retries download the original artifact +ID saved with the plan, even when GitHub increments the run attempt. Turbo hashes the selected file assignment through +`PAPERCLIP_CI_SELECTION_HASH`, so a passing subset cannot satisfy a full run or a +different selection. + +Set the repository variable `CI_VITEST_SELECTION=off` to disable file selection +while keeping the existing task-level optimization. Empty or `enabled` enables +it; other values disable it. `full-ci`, profiling, and task-level `full`/`shadow` +modes also bypass file selection. Qualification uses real Vitest graphs and real +Turbo cache entries, including a filesystem contract omitted by the raw graph: + +```sh +node --test scripts/ci/test-selection*.test.mjs scripts/ci/cache.integration.test.mjs +``` + General and serialized test suites and the standalone Rust suite use Turbo's content-addressed cache. An execution adapter maps selected lanes to the existing package scripts and their declared inputs. It selects tracked and non-ignored diff --git a/package.json b/package.json index b237a56c18a..9e2bb3714a6 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,7 @@ "agentmail": "^0.5.31", "cross-env": "^10.1.0", "esbuild": "^0.28.2", + "oxc-parser": "0.127.0", "typescript": "^7.0.2", "vitest": "^5.0.3", "yaml": "^2.8.1", diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index 9f16cb86763..4d2296345bd 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -249,7 +249,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () ]) { assert.match( jobs.get(jobId), - /^ {4}if: \$\{\{ needs\.gate\.outputs\.full_ci == 'true' && /m, + /^ {4}if: \$\{\{ (?:always\(\) && !cancelled\(\) && needs\.gate\.result == 'success' && )?needs\.gate\.outputs\.full_ci == 'true' && /m, `${jobId} must run only when the gate selects full CI`, ); } @@ -263,10 +263,14 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () const verify = jobs.get("verify"); assert.match( verify, - /^ {4}needs: \[gate, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m, + /^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, verify_serialized_server, canary_dry_run\]$/m, ); assert.equal(runAggregate(verify, true).status, 0); assert.equal(runAggregate(verify, false).status, 0); + for (const result of ["failure", "cancelled", "skipped"]) { + assert.equal(runAggregate(verify, true, { test_selection: result }).status, 0, + "an optional selector cannot block the successful original matrix"); + } for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner", "build", "docker_context_integrity", "verify_serialized_server", "canary_dry_run"]) { for (const result of ["failure", "cancelled", "skipped"]) { diff --git a/scripts/ci/cache.integration.test.mjs b/scripts/ci/cache.integration.test.mjs index c7fddcad772..90c94bf6c10 100644 --- a/scripts/ci/cache.integration.test.mjs +++ b/scripts/ci/cache.integration.test.mjs @@ -10,6 +10,53 @@ import { executionPlan, sourceFiles } from "./execution-plan.mjs"; const repositoryRoot = new URL("../../", import.meta.url); const turbo = process.env.PAPERCLIP_CI_TEST_TURBO ?? new URL("node_modules/.bin/turbo", repositoryRoot).pathname; +test("real Turbo keeps full runs and different selected test sets in separate cache entries", () => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-selection-cache-")); + const write = (file, value) => writeFileSync(path.join(directory, file), value); + const json = (file, value) => write(file, JSON.stringify(value)); + const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim(); + try { + git("init", "-q"); + for (const folder of ["scripts/ci", "server/src", "packages/fixture", "bin", ".ci-test-selection"]) mkdirSync(path.join(directory, folder), { recursive: true }); + for (const file of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) { + copyFileSync(new URL(`scripts/ci/${file}`, repositoryRoot), path.join(directory, "scripts/ci", file)); + } + write(".gitignore", ".turbo/\n.ci-test-selection/\nresult.txt\ncalls.txt\n"); + json("package.json", { name: "selection-cache-proof", private: true, packageManager: "npm@10.0.0", workspaces: ["packages/*"], scripts: { "test:run:general": "node check.cjs" } }); + json("packages/fixture/package.json", { name: "fixture", private: true }); + json("package-lock.json", { name: "selection-cache-proof", lockfileVersion: 3, packages: { "": { name: "selection-cache-proof" } } }); + json("turbo.json", { globalDependencies: [], globalEnv: ["CI_TASK_CACHE_SCOPE", "PAPERCLIP_*"], tasks: { + "//#ci:server": { inputs: ["check.cjs", "server/**"], outputs: ["result.txt"], cache: true }, + } }); + write("server/src/a.test.ts", "// a"); write("server/src/b.test.ts", "// b"); + write("check.cjs", `const fs=require('node:fs');fs.appendFileSync('calls.txt','call\\n');const file=process.env.PAPERCLIP_CI_TEST_SELECTION;fs.writeFileSync('result.txt',file?JSON.parse(fs.readFileSync(file)).lanes['general-server-without-chat'].shards[0]['@paperclipai/server'].join(','):'full');`); + write("bin/pnpm", `#!${process.execPath}\nconst {spawnSync}=require('node:child_process');const args=process.argv.slice(2);if(args[0]==='--version')console.log('9.15.4');else{const r=spawnSync(${JSON.stringify(turbo)},args.slice(2),{stdio:'inherit'});process.exit(r.status??1);}`); + chmodSync(path.join(directory, "bin/pnpm"), 0o755); + git("add", "."); + git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qm", "fixture"); + const head = git("rev-parse", "HEAD"); + const calls = () => readFileSync(path.join(directory, "calls.txt"), "utf8").trim().split("\n").length; + const run = (testFile) => { + const selection = path.join(directory, ".ci-test-selection/manifest.json"); + json(".ci-test-selection/manifest.json", { version: 1, head, lanes: { + "general-server-without-chat": { shards: [{ "@paperclipai/server": [testFile ?? "server/src/a.test.ts"] }] }, + } }); + const result = spawnSync(process.execPath, [path.join(directory, "scripts/ci/run-task.mjs")], { + cwd: directory, encoding: "utf8", env: { ...process.env, PATH: `${directory}/bin:${process.env.PATH}`, + CI_TASK: "ci:server", CI_TASK_ARGS: "[]", CI_FORCE_TASKS: "false", CI_TASK_DRY_RUN: "false", + PAPERCLIP_CI_TEST_SELECTION: testFile ? selection : "", PAPERCLIP_CI_SELECTION_HASH: "", + TURBO_TOKEN: "", TURBO_TEAM: "", TURBO_TELEMETRY_DISABLED: "1" }, + }); + assert.equal(result.status, 0, result.stdout + result.stderr); + assert.equal(readFileSync(path.join(directory, "result.txt"), "utf8"), testFile ?? "full"); + }; + run(); run(); assert.equal(calls(), 1); + run("server/src/a.test.ts"); run("server/src/a.test.ts"); assert.equal(calls(), 2); + run("server/src/b.test.ts"); assert.equal(calls(), 3); + run(); assert.equal(calls(), 3, "the original full result is reusable without accepting a subset result"); + } finally { rmSync(directory, { recursive: true, force: true }); } +}); + test("every cached lane maps to an existing package script with complete inputs", () => { const config = JSON.parse(readFileSync(new URL("turbo.json", repositoryRoot), "utf8")); for (const [name, definition] of Object.entries(config.tasks).filter(([, task]) => task.cache)) { @@ -83,7 +130,7 @@ test("the task launcher bypasses cache reads and writes when execution is forced mkdirSync(path.join(directory, "scripts/ci"), { recursive: true }); mkdirSync(path.join(directory, "bin")); mkdirSync(path.join(directory, "packages/fixture"), { recursive: true }); - for (const name of ["run-task.mjs", "execution-plan.mjs"]) { + for (const name of ["run-task.mjs", "execution-plan.mjs", "test-selection-manifest.mjs"]) { copyFileSync(new URL(`scripts/ci/${name}`, repositoryRoot), path.join(directory, "scripts/ci", name)); } write(".gitignore", ".turbo/\ncalls.txt\nresult.txt\nbin/\n"); diff --git a/scripts/ci/run-selected-vitest.mjs b/scripts/ci/run-selected-vitest.mjs new file mode 100644 index 00000000000..11a47b5cb2c --- /dev/null +++ b/scripts/ci/run-selected-vitest.mjs @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import path from "node:path"; +import { createVitest, parseCLI } from "vitest/node"; + +// Positional Vitest CLI filters are substrings: foo.test.ts also selects +// foo.test.tsx. Run the exact discovered specifications assigned to this shard. +const { filter, options } = parseCLI(["vitest", "run", ...process.argv.slice(2)]); +assert.ok(filter.length > 0, "Selected execution requires explicit test files"); +const expected = new Set(filter.map((file) => path.resolve(file))); +assert.equal(expected.size, filter.length, "Duplicate selected file"); +process.env.TEST = "true"; +process.env.VITEST = "true"; +process.env.NODE_ENV ??= "test"; +// Match the CLI's additive --exclude semantics without replacing project rules. +options.cliExclude = options.exclude; +delete options.exclude; +const ctx = await createVitest({ ...options, watch: false }); +try { + await ctx.standalone(); + const specifications = (await ctx.globTestSpecifications()).filter((spec) => expected.has(spec.moduleId)); + assert.deepEqual(specifications.map((spec) => spec.moduleId).sort(), [...expected].sort(), + "Selected files must match exactly one discovered specification in this project"); + await ctx.runTestSpecifications(specifications); +} catch (error) { + console.error(error); + process.exitCode = 1; +} finally { + await ctx.exit(); +} diff --git a/scripts/ci/run-task.mjs b/scripts/ci/run-task.mjs index 73e8a23985a..cc5ea03f666 100644 --- a/scripts/ci/run-task.mjs +++ b/scripts/ci/run-task.mjs @@ -3,6 +3,7 @@ import { lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from import { createHash } from "node:crypto"; import { matchesGlob } from "node:path"; import { executionPlan, sourceFiles } from "./execution-plan.mjs"; +import { loadSelection, selectionHash } from "./test-selection-manifest.mjs"; const task = process.env.CI_TASK; if (!/^ci:[a-z-]+$/.test(task ?? "")) throw new Error("Invalid CI task"); @@ -15,6 +16,11 @@ const version = (command, argv) => { const config = JSON.parse(readFileSync("turbo.json", "utf8")); const files = sourceFiles(); const execution = executionPlan(config, task, args, files); +// Validate before Turbo can return a cache hit. The hash separates selected +// executions from full executions and from every different selected file set. +if (process.env.PAPERCLIP_CI_TEST_SELECTION) { + process.env.PAPERCLIP_CI_SELECTION_HASH = selectionHash(loadSelection()); +} mkdirSync(".turbo/task-configs", { recursive: true }); const runtimeConfig = `.turbo/task-configs/${task.replace(":", "-")}.json`; writeFileSync(runtimeConfig, JSON.stringify(execution.config)); diff --git a/scripts/ci/select-tests.mjs b/scripts/ci/select-tests.mjs new file mode 100644 index 00000000000..d6a913e7356 --- /dev/null +++ b/scripts/ci/select-tests.mjs @@ -0,0 +1,119 @@ +import { execFileSync } from "node:child_process"; +import { appendFileSync, lstatSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; +import path from "node:path"; +import assert from "node:assert/strict"; +import { refineTestPlan, selectionDurations } from "./test-selection-plan.mjs"; + +export function selectionEligibility(plan) { + if (plan?.version !== 1 || plan.mode !== "enabled" || plan.forceReason) return "Full/shadow/forced task plan"; + if (!Array.isArray(plan.changed) || !plan.changed.length || plan.changed.length >= 300) return "Incomplete comparison"; + for (const file of plan.changed) { + if (!/^(?:ui\/src|server\/src|cli\/src|packages\/.+\/src)\/.+\.[cm]?[jt]sx?$/.test(file) + || /(?:config|setup)|\.d\.[cm]?ts$/i.test(file) || file.split("/").includes("..")) return `Broad input: ${file}`; + try { + if (!lstatSync(file).isFile()) return `Non-regular source: ${file}`; + } catch { return `Deleted or missing source: ${file}`; } + } + return null; +} + +function runnerInventory(group) { + const args = group === "serialized" ? ["--mode", "serialized", "--shard-index", "0", "--shard-count", "1"] + : ["--mode", "general", "--group", group, + ...(group === "general-server-without-chat" ? ["--shard-index", "0", "--shard-count", "1"] : [])]; + const env = { ...process.env, GITHUB_WORKFLOW: "PR" }; + delete env.PAPERCLIP_CI_TEST_SELECTION; + return JSON.parse(execFileSync(process.execPath, ["scripts/run-vitest-stable.mjs", ...args, "--dry-run"], + { env, encoding: "utf8", maxBuffer: 16 * 1024 * 1024 })); +} + +export async function createTestSelection(plan, head) { + const reason = selectionEligibility(plan); + if (reason) throw new Error(reason); + const groups = new Set(); + for (const job of ["general_tests", "verify_serialized_server"]) { + if (plan.jobs[job].run) for (const entry of plan.jobs[job].matrix.include) groups.add(entry.group ?? "serialized"); + } + groups.delete("general-chat"); // Keep its line sharding and DB ownership intact. + const owners = {}, projects = new Set(); + for (const group of groups) { + const inventory = runnerInventory(group); + if (group === "serialized" || group === "general-server-without-chat") { + owners[group] = { "@paperclipai/server": group === "serialized" + ? inventory.selectedSerializedSuites : inventory.selectedGeneralServerSuites }; + projects.add("@paperclipai/server"); + } else { + owners[group] = Object.fromEntries(inventory.workspaceProjects.map((project) => [project, null])); + for (const project of inventory.workspaceProjects) projects.add(project); + } + } + const { selectProjectTests } = await import("./vitest-selection-graph.mjs"); + const graphs = []; + for (const project of projects) { + const graph = await selectProjectTests({ project, changed: plan.changed, onlyOwnChanges: true }); + graphs.push(graph); + console.log(`${project}: ${graph.selected.length}/${graph.inventory.length} files, ${graph.alwaysRun.length} retained conservatively, ${graph.seconds.toFixed(1)}s`); + for (const owner of Object.values(owners)) if (owner[project] === null) owner[project] = graph.inventory; + } + const covered = new Set(graphs.flatMap((graph) => graph.coveredChanges)); + for (const file of plan.changed) assert.ok(covered.has(file), `Change outside the resolved test graphs: ${file}`); + const server = graphs.find((graph) => graph.project === "@paperclipai/server"); + if (server) { + const general = runnerInventory("general-server-without-chat"); + const native = runnerInventory("general-server-native-runner"); + assert.deepEqual([...general.selectedGeneralServerSuites, ...general.selectedSerializedSuites, + ...native.selectedGeneralServerSuites, "server/src/__tests__/chat-channels.integration.test.ts"].sort(), + server.inventory, "Server inventory must be completely covered by existing lane ownership"); + } + return { ...refineTestPlan({ plan, head, owners, graphs, durations: selectionDurations() }), graphs }; +} + +async function main() { + const plan = JSON.parse(process.env.CI_PLAN || "null"); + const output = (key, value) => { + if (process.env.GITHUB_OUTPUT) appendFileSync(process.env.GITHUB_OUTPUT, `${key}=${value}\n`); + }; + if (process.argv.includes("--eligible")) { + const reason = selectionEligibility(plan); + console.log(reason ?? "Source diff is eligible for graph selection"); + output("enabled", !reason); + return; + } + const directory = ".ci-test-selection"; + mkdirSync(directory, { recursive: true }); + const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).trim(); + let result; + try { + const head = git("rev-parse", "HEAD"); + assert.equal(head, process.env.GITHUB_SHA, "Selection must describe the tested merge commit"); + assert.match(process.env.BASE_SHA ?? "", /^[a-f0-9]{40}$/); + const changed = execFileSync("git", ["diff", "--name-only", "--no-renames", "-z", process.env.BASE_SHA, head, "--"], + { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }).split("\0").filter(Boolean).sort(); + assert.deepEqual(changed, [...new Set(plan.changed)].sort(), "Local diff disagrees with the trusted routing diff"); + assert.equal(git("status", "--porcelain", "--untracked-files=normal"), "", "Source inputs changed during dependency setup"); + process.env.NODE_ENV = "test"; + process.env.VITEST = "true"; + result = await createTestSelection(plan, head); + writeFileSync(`${directory}/manifest.json`, JSON.stringify(result.manifest)); + writeFileSync(`${directory}/audit.json`, JSON.stringify(result.graphs, null, 2)); + output("selected", "true"); + } catch (error) { + console.warn(`Using the complete task plan: ${error.message}`); + result = { plan: { ...plan, testSelectionFallback: error.message } }; + output("selected", "false"); + } + writeFileSync(`${directory}/plan.json`, JSON.stringify(result.plan)); + if (process.env.GITHUB_STEP_SUMMARY) { + const stats = result.plan.testSelection?.stats; + appendFileSync(process.env.GITHUB_STEP_SUMMARY, stats + ? ["### Vitest test selection", "", "| Lane | Test files | Jobs |", "| --- | ---: | ---: |", + ...stats.map((s) => `| ${s.group} | ${s.selected}/${s.total} | ${s.jobs}/${s.beforeJobs} |`), "", + "Filesystem/process/opaque consumers are retained. Chat and native Runner lanes keep their existing ownership.", ""].join("\n") + : `### Vitest test selection\n\nFull task plan: ${result.plan.testSelectionFallback}\n`); + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { console.error(error); process.exitCode = 1; }); +} diff --git a/scripts/ci/test-selection-manifest.mjs b/scripts/ci/test-selection-manifest.mjs new file mode 100644 index 00000000000..50b46995d78 --- /dev/null +++ b/scripts/ci/test-selection-manifest.mjs @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { execFileSync } from "node:child_process"; +import { readFileSync, statSync } from "node:fs"; + +export const selectableGroups = ["general-server-without-chat", "general-workspaces-a", + "general-workspaces-b", "general-workspaces-c", "serialized"]; + +export function selectionHash(manifest) { + return createHash("sha256").update(JSON.stringify({ version: manifest.version, lanes: manifest.lanes })).digest("hex"); +} + +export function validateSelection(manifest, { head, hash } = {}) { + assert.equal(manifest.version, 1, "Unsupported test selection"); + assert.match(manifest.head, /^[a-f0-9]{40}$/); + if (head) assert.equal(manifest.head, head, "Selection belongs to another checkout"); + assert.ok(manifest.lanes && typeof manifest.lanes === "object" && !Array.isArray(manifest.lanes)); + const seen = new Set(); + for (const [group, lane] of Object.entries(manifest.lanes)) { + assert.ok(selectableGroups.includes(group), `Unknown selection group: ${group}`); + assert.ok(Array.isArray(lane.shards) && lane.shards.length > 0 && lane.shards.length <= 12); + for (const shard of lane.shards) { + assert.ok(shard && typeof shard === "object" && !Array.isArray(shard)); + let count = 0; + for (const [project, files] of Object.entries(shard)) { + assert.ok(project && Array.isArray(files) && files.length > 0); + for (const file of files) { + assert.equal(typeof file, "string"); + assert.ok(/^(ui|server|cli|packages)\//.test(file) && !/[\0\r\n\\:*?\[\]{}]/.test(file) + && !file.split("/").some((part) => ["", ".", ".."].includes(part)), `Invalid selected path: ${file}`); + const key = `${project}:${file}`; + assert.ok(!seen.has(key), `Test assigned twice: ${key}`); + seen.add(key); + count++; + } + } + assert.ok(count > 0, "Empty selection shard"); + } + } + if (hash) assert.equal(selectionHash(manifest), hash, "Selection artifact does not match its plan"); + return manifest; +} + +export function loadSelection(file = process.env.PAPERCLIP_CI_TEST_SELECTION) { + const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + const manifest = validateSelection(JSON.parse(readFileSync(file, "utf8")), { + head, hash: process.env.PAPERCLIP_CI_SELECTION_HASH, + }); + for (const lane of Object.values(manifest.lanes)) { + for (const shard of lane.shards) for (const files of Object.values(shard)) { + for (const file of files) assert.ok(statSync(file).isFile(), `Selected test missing: ${file}`); + } + } + return manifest; +} + +export function selectedShard(manifest, options, projects) { + const group = options.mode === "serialized" ? "serialized" : options.group; + const lane = manifest.lanes[group]; + assert.ok(lane, `Missing selection for ${group}`); + assert.equal(lane.shards.length, options.shardCount ?? 1, "Selection shard count mismatch"); + const shard = lane.shards[options.shardIndex ?? 0]; + assert.ok(shard, "Missing selection shard"); + for (const project of Object.keys(shard)) assert.ok(projects.includes(project), `Wrong project for ${group}: ${project}`); + return shard; +} diff --git a/scripts/ci/test-selection-plan.mjs b/scripts/ci/test-selection-plan.mjs new file mode 100644 index 00000000000..309299c41fd --- /dev/null +++ b/scripts/ci/test-selection-plan.mjs @@ -0,0 +1,77 @@ +import assert from "node:assert/strict"; +import { loadShardDurations, partitionGeneralServerSuites } from "../general-server-shard.mjs"; +import { selectionHash, validateSelection } from "./test-selection-manifest.mjs"; + +// Preserve the existing per-job budget. Small selections use fewer runners; +// large selections retain the existing parallelism and serialized ownership. +export function partitionSelection(inventory, selected, maxShards, durations = {}) { + const totalWeight = partitionGeneralServerSuites(inventory, 1, durations)[0].totalWeight; + const selectedWeight = partitionGeneralServerSuites(selected, 1, durations)[0].totalWeight; + const budget = totalWeight / maxShards || 1; + const count = Math.min(maxShards, selected.length, Math.max(1, Math.ceil(selectedWeight / budget))); + return selected.length ? partitionGeneralServerSuites(selected, count, durations).map((shard) => shard.files) : []; +} + +export function refineTestPlan({ plan, head, owners, graphs, durations = {} }) { + const result = structuredClone(plan); + const manifest = { version: 1, head, lanes: {} }; + const stats = []; + const byProject = new Map(graphs.map((graph) => [graph.project, graph])); + for (const job of ["general_tests", "verify_serialized_server"]) { + if (!plan.jobs[job].run) continue; + const entries = plan.jobs[job].matrix.include; + const groups = [...new Set(entries.map((entry) => entry.group ?? "serialized"))]; + const matrix = []; + for (const group of groups) { + const original = entries.filter((entry) => (entry.group ?? "serialized") === group); + if (group === "general-chat") { matrix.push(...original); continue; } + const owned = owners[group]; + assert.ok(owned, `Missing inventory owner: ${group}`); + const projectByFile = new Map(); + const selected = []; + for (const [project, files] of Object.entries(owned)) { + const graph = byProject.get(project); + assert.ok(graph, `Missing project graph: ${project}`); + const inventory = new Set(graph.inventory); + const affected = new Set(graph.selected); + for (const file of files) { + assert.ok(inventory.has(file), `Runner/Vitest inventory disagreement: ${file}`); + assert.ok(!projectByFile.has(file), `Ambiguous owner: ${file}`); + projectByFile.set(file, project); + if (affected.has(file)) selected.push(file); + } + } + const partitions = partitionSelection([...projectByFile.keys()], selected, original.length, durations[group]); + if (partitions.length) { + const shards = partitions.map((files) => { + const shard = {}; + for (const file of files) (shard[projectByFile.get(file)] ??= []).push(file); + return shard; + }); + manifest.lanes[group] = { shards }; + for (let index = 0; index < shards.length; index++) { + const entry = { ...original[0], selection: true }; + if (original[0].shard_count) { + Object.assign(entry, { shard_index: index, shard_count: shards.length }); + if (entry.shard_label) entry.shard_label = `${index + 1}/${shards.length}`; + if (entry.group_label) entry.group_label = entry.group_label.replace(/\(\d+\/\d+\)/, `(${index + 1}/${shards.length})`); + } + matrix.push(entry); + } + } + stats.push({ group, total: projectByFile.size, selected: selected.length, beforeJobs: original.length, jobs: partitions.length }); + } + result.jobs[job] = { ...result.jobs[job], run: matrix.length > 0, selected: matrix.length, proposed: matrix.length, + matrix: { include: matrix.length ? matrix : entries } }; + } + validateSelection(manifest); + result.testSelection = { version: 1, head, hash: selectionHash(manifest), stats }; + return { plan: result, manifest }; +} + +export function selectionDurations() { + return { + "general-server-without-chat": loadShardDurations("scripts/general-server-shard-durations.json"), + serialized: loadShardDurations("scripts/serialized-shard-durations.json"), + }; +} diff --git a/scripts/ci/test-selection-regression.integration.test.mjs b/scripts/ci/test-selection-regression.integration.test.mjs new file mode 100644 index 00000000000..a18f0f019a7 --- /dev/null +++ b/scripts/ci/test-selection-regression.integration.test.mjs @@ -0,0 +1,56 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { selectProjectTests } from "./vitest-selection-graph.mjs"; +import { selectionHash } from "./test-selection-manifest.mjs"; + +test("production runner catches the filesystem regression that raw Vitest selection misses", async (t) => { + const repo = process.cwd(); + const root = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vr-"))); + t.after(() => rmSync(root, { recursive: true, force: true })); + for (const directory of ["ui/src", "server/src/__tests__", "server/scripts"]) mkdirSync(path.join(root, directory), { recursive: true }); + symlinkSync(path.join(repo, "node_modules"), path.join(root, "node_modules"), "dir"); + const write = (file, body) => writeFileSync(path.join(root, file), body); + const git = (...args) => execFileSync("git", args, { cwd: root, encoding: "utf8" }).trim(); + write(".gitignore", "node_modules/\nselection.json\n"); + write("package.json", '{"name":"selection-regression","private":true,"type":"module"}'); + write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}'); + write("ui/vitest.config.mjs", 'export default {test:{name:"@paperclipai/ui",include:["src/**/*.test.mjs"]}}'); + write("ui/src/leaf.mjs", "export const value = 42;\n"); + write("ui/src/import.test.mjs", 'import {test,expect} from "vitest"; import {value} from "./leaf.mjs"; test("value",()=>expect(value).toBe(42));'); + write("ui/src/import.test.mjs.test.mjs", 'throw new Error("UNSELECTED_FILE: substring filters must not execute this file");'); + write("ui/src/contract.test.mjs", 'import {test,expect} from "vitest"; import {readFileSync} from "node:fs"; test("filesystem contract",()=>expect(readFileSync(new URL("./leaf.mjs",import.meta.url),"utf8")).not.toContain("FORBIDDEN_MARKER"));'); + git("init", "-q"); git("add", "."); + const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture"); + commit(); + const run = (files) => { + const manifest = { version: 1, head: git("rev-parse", "HEAD"), lanes: { + "general-workspaces-a": { shards: [{ "@paperclipai/ui": files }] }, + } }; + write("selection.json", JSON.stringify(manifest)); + return spawnSync(process.execPath, [path.join(repo, "scripts/run-vitest-stable.mjs"), "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1"], { + cwd: root, encoding: "utf8", timeout: 60000, + env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: path.join(root, "selection.json"), + PAPERCLIP_CI_SELECTION_HASH: selectionHash(manifest) }, + }); + }; + const baseline = run(["ui/src/import.test.mjs", "ui/src/contract.test.mjs"]); + assert.equal(baseline.status, 0, baseline.stdout + baseline.stderr); + const missing = run(["ui/src/leaf.mjs"]); + assert.equal(missing.status, 1, "an existing source file is not an executable test specification"); + assert.match(missing.stdout + missing.stderr, /Selected files must match/); + write("ui/src/leaf.mjs", `${readFileSync(path.join(root, "ui/src/leaf.mjs"), "utf8")}// FORBIDDEN_MARKER\n`); + commit(); + const graph = await selectProjectTests({ root, project: "@paperclipai/ui", changed: ["ui/src/leaf.mjs"] }); + assert.deepEqual(graph.related, ["ui/src/import.test.mjs"]); + const raw = run(graph.related); + assert.equal(raw.status, 0, raw.stdout + raw.stderr); + assert.ok(graph.selected.includes("ui/src/contract.test.mjs")); + const guarded = run(graph.selected); + assert.equal(guarded.status, 1, guarded.stdout + guarded.stderr); + assert.match(guarded.stdout + guarded.stderr, /filesystem contract/); + assert.match(guarded.stdout + guarded.stderr, /FORBIDDEN_MARKER/); +}); diff --git a/scripts/ci/test-selection-workflow.test.mjs b/scripts/ci/test-selection-workflow.test.mjs new file mode 100644 index 00000000000..f0ee415a55a --- /dev/null +++ b/scripts/ci/test-selection-workflow.test.mjs @@ -0,0 +1,100 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import vm from "node:vm"; +import test from "node:test"; +import { planTasks } from "../../.github/scripts/ci-plan.mjs"; + +const workflow = readFileSync(".github/workflows/pr-trusted.yml", "utf8"); +const block = (job) => workflow.split(`\n ${job}:\n`)[1].split(/\n [a-z0-9_]+:\n/)[0]; +const plan = planTasks({ config: JSON.parse(readFileSync("turbo.json", "utf8")), + lanes: JSON.parse(readFileSync(".github/ci-task-matrix.json", "utf8")), paths: ["ui/src/pages/Inbox.tsx"] }); +const evaluate = (expression, needs, cancelled = false) => vm.runInNewContext(expression, { + needs, fromJSON: JSON.parse, toJSON: JSON.stringify, always: () => true, cancelled: () => cancelled, +}); + +test("failed, timed-out, skipped and unavailable selectors leave the original matrix runnable", () => { + for (const result of ["failure", "cancelled", "skipped", "success"]) { + const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } }, + test_selection: { result, outputs: { plan: "" } } }; + const job = block("general_tests"); + const condition = job.match(/if: \$\{\{ (.*) \}\}/)[1]; + const matrix = job.match(/matrix: \$\{\{ (.*) \}\}/)[1]; + assert.equal(evaluate(condition, needs), true, result); + assert.deepEqual(JSON.parse(JSON.stringify(evaluate(matrix, needs))), plan.jobs.general_tests.matrix); + assert.equal(evaluate(condition, needs, true), false, "cancellation must not start more runners"); + } +}); + +test("a published selection controls the matrix and the required aggregate", () => { + const selected = structuredClone(plan); + selected.testSelection = { version: 1, hash: "test" }; + selected.jobs.general_tests.matrix.include = [selected.jobs.general_tests.matrix.include[0]]; + selected.jobs.general_tests.selected = 1; + const needs = { gate: { result: "success", outputs: { full_ci: "true", plan: JSON.stringify(plan) } }, + test_selection: { result: "success", outputs: { plan: JSON.stringify(selected) } } }; + const matrix = block("general_tests").match(/matrix: \$\{\{ (.*) \}\}/)[1]; + assert.equal(evaluate(matrix, needs).include.length, 1); + const aggregate = block("verify"); + const names = aggregate.match(/needs: \[([^\]]+)\]/)[1].split(", "); + const code = aggregate.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0] + .split("\n").map((line) => line.slice(10)).join("\n"); + const results = Object.fromEntries(names.map((name) => [name, { result: + ["gate", "policy", "test_selection"].includes(name) || selected.jobs[name]?.run ? "success" : "skipped" }])); + const run = () => spawnSync(process.execPath, ["--input-type=module", "-e", code], { encoding: "utf8", + env: { ...process.env, FULL_CI: "true", CI_PLAN: JSON.stringify(selected), CI_RESULTS: JSON.stringify(results) } }); + assert.equal(run().status, 0); + for (const job of ["general_tests", "test_selection"]) { + for (const result of ["failure", "cancelled", "skipped"]) { + results[job].result = result; + assert.notEqual(run().status, 0, `${job}: ${result}`); + } + results[job].result = "success"; + } + selected.jobs.general_tests.run = false; + results.general_tests.result = "skipped"; + assert.equal(run().status, 0, "an explicitly empty selected lane may be skipped"); + assert.ok(aggregate.includes("needs.test_selection.outputs.plan || needs.gate.outputs.plan")); +}); + +test("publication follows successful upload and manifests are downloaded without ignoring failures", () => { + const selection = block("test_selection"); + assert.ok(selection.indexOf("Upload validated selection") < selection.indexOf("Publish the executable plan")); + assert.match(selection, /timeout-minutes: 5/); + assert.match(selection, /vars.CI_VITEST_SELECTION == '' \|\| vars.CI_VITEST_SELECTION == 'enabled'/); + for (const job of ["general_tests", "verify_serialized_server"]) { + assert.match(block(job), /PAPERCLIP_CI_SELECTION_HASH:.*testSelection.hash/); + assert.match(block(job), /Download selected test files\n if: matrix.selection\n uses: actions\/download-artifact@/); + assert.doesNotMatch(block(job), /continue-on-error: true\n.*download-artifact/); + } +}); + +test("failed-job retries use the original selector artifact, not the retry attempt", (t) => { + const directory = mkdtempSync(path.join(tmpdir(), "ci-retry-")); + t.after(() => rmSync(directory, { recursive: true, force: true })); + mkdirSync(path.join(directory, ".ci-test-selection")); + const output = path.join(directory, "output"); + const publication = block("test_selection").split(" - name: Publish the executable plan")[1]; + assert.match(publication, /ARTIFACT_ID: \$\{\{ steps.upload_selection.outputs.artifact-id \}\}/); + const code = publication.split(" node --input-type=module <<'NODE'\n")[1].split("\n NODE")[0] + .split("\n").map((line) => line.slice(10)).join("\n"); + const selected = { ...plan, testSelection: { version: 1, hash: "test" } }; + const run = (value, artifactId) => { + writeFileSync(path.join(directory, ".ci-test-selection/plan.json"), JSON.stringify(value)); + writeFileSync(output, ""); + return spawnSync(process.execPath, ["--input-type=module", "-e", code], { + cwd: directory, encoding: "utf8", env: { ...process.env, ARTIFACT_ID: artifactId, GITHUB_OUTPUT: output }, + }); + }; + assert.equal(run(selected, "12345").status, 0); + const savedPlan = readFileSync(output, "utf8").trim().slice("plan=".length); + for (const job of ["general_tests", "verify_serialized_server"]) { + const expression = block(job).match(/artifact-ids: \$\{\{ (.*) \}\}/)[1]; + assert.equal(vm.runInNewContext(expression, { fromJSON: JSON.parse, github: { run_attempt: 2 }, + needs: { test_selection: { outputs: { plan: savedPlan } } } }), "12345"); + } + assert.notEqual(run(selected, "").status, 0, "a selection cannot be published without its uploaded artifact"); + assert.equal(run(plan, "").status, 0, "fallback plans need no selection artifact"); +}); diff --git a/scripts/ci/test-selection.integration.test.mjs b/scripts/ci/test-selection.integration.test.mjs new file mode 100644 index 00000000000..9f74d860448 --- /dev/null +++ b/scripts/ci/test-selection.integration.test.mjs @@ -0,0 +1,183 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { selectProjectTests } from "./vitest-selection-graph.mjs"; +import { partitionSelection, refineTestPlan } from "./test-selection-plan.mjs"; +import { selectionHash, selectedShard, validateSelection } from "./test-selection-manifest.mjs"; +import { selectionEligibility } from "./select-tests.mjs"; + +const root = process.cwd(); +const head = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8" }).trim(); + +test("real graph retains direct and indirect filesystem, process and opaque dependencies", async (t) => { + const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vg-"))); + t.after(() => rmSync(directory, { recursive: true, force: true })); + mkdirSync(path.join(directory, "ui/src"), { recursive: true }); + mkdirSync(path.join(directory, "ui/src/glob")); + symlinkSync(path.join(root, "node_modules"), path.join(directory, "node_modules"), "dir"); + const write = (file, text) => writeFileSync(path.join(directory, file), text); + write("vitest.config.mjs", 'export default {test:{projects:["ui"]}}'); + write("ui/vitest.config.mjs", `export default {resolve:{alias:{"@leaf":${JSON.stringify(path.join(directory, "ui/src/leaf.ts"))}}},test:{name:"fixture",include:["src/**/*.test.ts"],setupFiles:["./setup.ts"]}}`); + write("ui/setup.ts", 'import "./src/environment.ts";'); + write("ui/src/environment.ts", "export const environment = true;"); + write("ui/src/leaf.ts", "export const value = 42;"); + write("ui/src/bridge.ts", 'export {value} from "@leaf";'); + write("ui/src/helper.ts", 'import {readFileSync} from "node:fs"; export const read = (file: string) => readFileSync(file, "utf8");'); + write("ui/src/import.test.ts", 'import {value} from "./bridge"; console.log(value); throw new Error("selection must not execute tests");'); + write("ui/src/reader.test.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));'); + write("ui/src/indirect.test.ts", 'import {read} from "./helper"; read("input"); throw new Error("selection must not execute tests");'); + write("ui/src/process.test.ts", 'import {execFileSync} from "node:child_process"; execFileSync("must-not-execute");'); + write("ui/src/dynamic.test.ts", 'const load = (file: string) => import(file);'); + write("ui/src/cjs.test.ts", 'const load = require;'); + write("ui/src/builtin-loader.test.ts", 'process.getBuiltinModule("fs").readFileSync("input");'); + write("ui/src/opaque.test.ts", 'import {parse} from "yaml"; console.log(parse("value: true"));'); + write("ui/src/independent.test.ts", 'export const independent = true;'); + write("ui/src/mocked.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); import {read} from "./helper"; read("input");'); + write("ui/src/partial.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",async (original)=>await original()); import {read} from "./helper"; read("input");'); + write("ui/src/automock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper"); import {read} from "./helper"; read("input");'); + write("ui/src/bypass.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); await vi.importActual("./helper");'); + write("ui/src/spy.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",{spy:true}); import {read} from "./helper"; read("input");'); + write("ui/src/remock.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",()=>({read:()=>"mock"})); vi.doMock("./helper",async original=>await original());'); + write("ui/src/arguments.test.ts", 'import {vi} from "vitest"; vi.mock("./helper",function(){return arguments[0]();});'); + write("ui/src/glob/one.ts", "export const value = 1;"); + write("ui/src/globber.ts", 'export const modules = import.meta.glob("./glob/*.ts", {eager:true});'); + write("ui/src/glob.test.ts", 'import {modules} from "./globber"; console.log(modules);'); + execFileSync("git", ["init", "-q"], { cwd: directory }); + execFileSync("git", ["add", "."], { cwd: directory }); + const select = (changed) => selectProjectTests({ root: directory, project: "fixture", changed }); + const result = await select(["ui/src/leaf.ts"]); + assert.deepEqual(result.related, ["ui/src/import.test.ts"]); + for (const file of ["reader", "indirect", "process", "dynamic", "cjs", "builtin-loader", "opaque", "partial", "automock", "bypass", "spy", "remock", "arguments"]) { + assert.ok(result.alwaysRun.includes(`ui/src/${file}.test.ts`), `${file} must never be dropped`); + } + assert.ok(!result.selected.includes("ui/src/independent.test.ts")); + assert.ok(!result.alwaysRun.includes("ui/src/mocked.test.ts"), "a completely replaced local dependency performs no real IO"); + assert.ok(!result.alwaysRun.includes("ui/src/glob.test.ts"), "literal Vite globs use the real dependency graph"); + assert.ok((await select(["ui/src/glob/one.ts"])).related.includes("ui/src/glob.test.ts")); + write("ui/src/glob/two.ts", "export const value = 2;"); + execFileSync("git", ["add", "."], { cwd: directory }); + assert.ok((await select(["ui/src/glob/two.ts"])).related.includes("ui/src/glob.test.ts"), "new glob members cannot be omitted"); + const setup = await select(["ui/src/environment.ts"]); + assert.equal(setup.setupChanged, true); + assert.deepEqual(setup.selected, setup.inventory, "transitive setup edits affect every test"); + write("ui/src/broken.test.ts", 'import "./missing";'); + const broken = await select(["ui/src/leaf.ts"]); + assert.ok(broken.selected.includes("ui/src/broken.test.ts"), "an unresolved dependency must retain its test"); + write("ui/src/generated.ts", "export const generated = 42;"); + write("ui/src/generated.test.ts", 'import {generated} from "./generated"; console.log(generated);'); + execFileSync("git", ["add", "ui/src/generated.test.ts"], { cwd: directory }); + assert.ok((await select(["ui/src/leaf.ts"])).alwaysRun.includes("ui/src/generated.test.ts"), + "generated inputs have no trustworthy source-to-output graph and retain their consumers"); + write("ui/src/environment.ts", 'import {readFileSync} from "node:fs"; readFileSync(new URL("./leaf.ts", import.meta.url));'); + await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup builtin/, "setup filesystem reads can affect every test outside the import graph"); + mkdirSync(path.join(directory, "server/src/__tests__/helpers"), { recursive: true }); + for (const file of ["server/src/__tests__/setup-supertest.ts", "server/src/__tests__/helpers/isolated-test-home.ts"]) { + write(file, readFileSync(path.join(root, file), "utf8")); + } + write("ui/setup.ts", 'import "../server/src/__tests__/setup-supertest.ts";'); + const reviewed = await select(["ui/src/leaf.ts"]); + assert.ok(!reviewed.selected.includes("ui/src/independent.test.ts"), "reviewed isolated setup permits file selection"); + const helper = "server/src/__tests__/helpers/isolated-test-home.ts"; + const changedSetup = await select([helper]); + assert.deepEqual(changedSetup.selected, changedSetup.inventory, "reviewed helpers still invalidate all tests when changed"); + write(helper, `${readFileSync(path.join(directory, helper), "utf8")}\n// setup edited since review\n`); + await assert.rejects(select(["ui/src/leaf.ts"]), /Opaque setup/, "a stale review cannot authorize new setup effects"); +}); + +test("partitioning reduces runners and preserves an exact, non-overlapping file cover", () => { + const all = Array.from({ length: 120 }, (_, index) => `server/src/${index}.test.ts`); + const selected = all.slice(0, 15); + const shards = partitionSelection(all, selected, 12); + assert.equal(shards.length, 2); + assert.deepEqual(shards.flat().sort(), selected.sort()); + assert.equal(new Set(shards.flat()).size, selected.length); + assert.deepEqual(partitionSelection(all, [], 12), []); + assert.equal(partitionSelection(all, all, 12).length, 12); + const durations = Object.fromEntries(all.map((file, index) => [file, index === 0 ? 1000 : 1])); + assert.equal(partitionSelection(all, [all[0]], 12, durations).length, 1); + assert.equal(partitionSelection(all, [all[0]], 12, Object.fromEntries(all.map((file) => [file, 0]))).length, 1); +}); + +function fixturePlan() { + const files = ["ui/src/pages/Inbox.test.tsx", "ui/src/pages/apps/chat/chat-ui-contract.test.ts"]; + const entries = [0, 1].map((shard_index) => ({ group: "general-workspaces-a", group_label: `workspaces-a (${shard_index + 1}/2)`, shard_index, shard_count: 2, task: "ci:workspaces-a" })); + const plan = { version: 1, mode: "enabled", forceReason: null, changed: ["ui/src/pages/Inbox.tsx"], jobs: { + general_tests: { run: true, matrix: { include: entries }, selected: 2, total: 2 }, + verify_serialized_server: { run: false, matrix: { include: [{ task: "ci:serialized" }] } }, + } }; + const owners = { "general-workspaces-a": { "@paperclipai/ui": files } }; + const graphs = [{ project: "@paperclipai/ui", inventory: files, selected: [files[1]] }]; + return { files, ...refineTestPlan({ plan, head, owners, graphs }) }; +} + +test("runner consumes the exact manifest, and rejects stale, missing, or mismatched manifests", (t) => { + const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vm-"))); + t.after(() => rmSync(directory, { recursive: true, force: true })); + const { manifest, plan } = fixturePlan(); + assert.equal(plan.jobs.general_tests.selected, 1); + const file = path.join(directory, "selection.json"); + const run = (selection = manifest, hash = selectionHash(manifest)) => { + writeFileSync(file, JSON.stringify(selection)); + return spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-workspaces-a", "--shard-index", "0", "--shard-count", "1", "--dry-run"], { + cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: hash }, + }); + }; + const result = run(); + assert.equal(result.status, 0, result.stderr); + assert.deepEqual(JSON.parse(result.stdout).selection, manifest.lanes["general-workspaces-a"].shards[0]); + assert.notEqual(run({ ...manifest, head: "0".repeat(40) }).status, 0); + assert.notEqual(run(manifest, "bad-hash").status, 0); + assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 2 }, ["@paperclipai/ui"])); + assert.throws(() => selectedShard(manifest, { mode: "general", group: "general-workspaces-a", shardCount: 1 }, ["@paperclipai/server"])); + const duplicate = structuredClone(manifest); + duplicate.lanes["general-workspaces-a"].shards.push(duplicate.lanes["general-workspaces-a"].shards[0]); + assert.throws(() => validateSelection(duplicate), /assigned twice/); + const wrongLane = { version: 1, head, lanes: { "general-server-without-chat": { + shards: [{ "@paperclipai/server": ["server/src/__tests__/issues-service.test.ts"] }], + } } }; + writeFileSync(file, JSON.stringify(wrongLane)); + const misplaced = spawnSync(process.execPath, ["scripts/run-vitest-stable.mjs", "--mode", "general", "--group", "general-server-without-chat", "--shard-index", "0", "--shard-count", "1", "--dry-run"], { + cwd: root, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_TEST_SELECTION: file, PAPERCLIP_CI_SELECTION_HASH: selectionHash(wrongLane) }, + }); + assert.notEqual(misplaced.status, 0); + assert.match(misplaced.stderr, /belongs to another server lane/); +}); + +test("forced modes, config, fixtures, deletions and unknown inputs cannot narrow CI", () => { + const plan = fixturePlan().plan; + assert.equal(selectionEligibility(plan), null); + for (const changed of [["ui/vitest.setup.ts"], ["ui/vite.config.ts"], ["package.json"], ["pnpm-lock.yaml"], + ["docs/api/issues.md"], ["ui/src/missing-deleted.ts"], ["server/fixtures/input.json"], []]) { + assert.ok(selectionEligibility({ ...plan, changed }), JSON.stringify(changed)); + } + for (const mode of ["full", "shadow"]) assert.ok(selectionEligibility({ ...plan, mode })); + assert.ok(selectionEligibility({ ...plan, forceReason: "full-ci" })); +}); + +test("the CLI publishes the complete coarse plan if setup mutates tracked inputs", (t) => { + const directory = realpathSync(mkdtempSync(path.join(tmpdir(), "ci-vd-"))); + t.after(() => rmSync(directory, { recursive: true, force: true })); + mkdirSync(path.join(directory, "ui/src"), { recursive: true }); + const file = path.join(directory, "ui/src/input.ts"); + const git = (...args) => execFileSync("git", args, { cwd: directory, encoding: "utf8" }).trim(); + const commit = () => git("-c", "user.name=CI", "-c", "user.email=ci@example.invalid", "commit", "-qam", "fixture"); + git("init", "-q"); writeFileSync(file, "export const value = 1;"); git("add", "."); commit(); + const base = git("rev-parse", "HEAD"); + writeFileSync(file, "export const value = 2;"); commit(); + const merge = git("rev-parse", "HEAD"); + writeFileSync(file, "export const value = 3; // generated during setup"); + const plan = { version: 1, mode: "enabled", forceReason: null, changed: ["ui/src/input.ts"], + jobs: { general_tests: { run: true, selected: 2, matrix: { include: [{ shard_index: 0 }, { shard_index: 1 }] } } } }; + const result = spawnSync(process.execPath, [path.join(root, "scripts/ci/select-tests.mjs")], { + cwd: directory, encoding: "utf8", env: { ...process.env, CI_PLAN: JSON.stringify(plan), BASE_SHA: base, + GITHUB_SHA: merge, GITHUB_OUTPUT: "", GITHUB_STEP_SUMMARY: "" }, + }); + assert.equal(result.status, 0, result.stderr); + const fallback = JSON.parse(readFileSync(path.join(directory, ".ci-test-selection/plan.json"), "utf8")); + assert.match(fallback.testSelectionFallback, /Source inputs changed/); + assert.deepEqual(fallback.jobs, plan.jobs); + assert.equal(fallback.testSelection, undefined); +}); diff --git a/scripts/ci/vitest-impact-report.integration.test.mjs b/scripts/ci/vitest-impact-report.integration.test.mjs new file mode 100644 index 00000000000..0ac09e37cf5 --- /dev/null +++ b/scripts/ci/vitest-impact-report.integration.test.mjs @@ -0,0 +1,38 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { measureVitestImpact } from "./vitest-impact-report.mjs"; + +test("real Vitest graph follows aliases/transitive imports but misses filesystem contracts", async (t) => { + const root = mkdtempSync(path.join(tmpdir(), "vitest-impact-")); + t.after(() => rmSync(root, { recursive: true, force: true })); + mkdirSync(path.join(root, "unit")); + const write = (file, text) => writeFileSync(path.join(root, file), text); + write("vitest.config.mjs", 'export default { test: { projects: ["unit"] } };'); + write("unit/vitest.config.mjs", `export default { + resolve: { alias: { "@leaf": ${JSON.stringify(path.join(root, "unit/leaf.mjs"))} } }, + test: { name: "fixture", include: ["**/*.test.mjs"] } + };`); + write("unit/leaf.mjs", "export const value = 42;"); + write("unit/bridge.mjs", 'export { value } from "@leaf";'); + // The graph must be collected without executing the test modules. + write("unit/related.test.mjs", 'import { value } from "./bridge.mjs"; throw new Error("must not execute");'); + write("unit/dynamic.test.mjs", 'const load = () => import("./leaf.mjs"); throw new Error("must not execute");'); + write("unit/independent.test.mjs", 'throw new Error("must not execute");'); + write("unit/reader.test.mjs", 'import { readFileSync } from "node:fs"; const body = readFileSync(new URL("./leaf.mjs", import.meta.url), "utf8");'); + + const report = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/leaf.mjs"] }); + assert.equal(report.mode, "report-only"); + assert.equal(report.projects[0].totalFiles, 4); + assert.deepEqual(report.projects[0].selectedTests, ["unit/dynamic.test.mjs", "unit/related.test.mjs"]); + assert.deepEqual(report.projects[0].omittedTests, ["unit/independent.test.mjs", "unit/reader.test.mjs"]); + assert.ok(report.limitations.some((message) => message.includes("filesystem"))); + + const direct = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unit/independent.test.mjs"] }); + assert.deepEqual(direct.projects[0].selectedTests, ["unit/independent.test.mjs"]); + const unmapped = await measureVitestImpact({ root, projects: ["fixture"], changedFiles: ["unmapped.md"] }); + assert.equal(unmapped.projects[0].selectedFiles, 0); + assert.equal(unmapped.mode, "report-only", "an empty graph is not permission to skip CI"); +}); diff --git a/scripts/ci/vitest-impact-report.mjs b/scripts/ci/vitest-impact-report.mjs new file mode 100644 index 00000000000..4502516249a --- /dev/null +++ b/scripts/ci/vitest-impact-report.mjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +import { execFileSync } from "node:child_process"; +import { realpathSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { parseArgs } from "node:util"; + +function repoPath(root, file) { + const relative = path.relative(root, path.resolve(root, file)); + if (!relative || relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) { + throw new Error(`Expected a file inside the checkout: ${file}`); + } + return relative.split(path.sep).join("/"); +} + +// This is an experiment report, deliberately not a CI execution plan. Import +// graphs cannot describe filesystem reads, subprocesses, or external services. +export async function measureVitestImpact({ root = process.cwd(), projects, changedFiles }) { + root = realpathSync(root); + if (!projects?.length || !changedFiles?.length) { + throw new Error("Specify at least one project and one changed file"); + } + const files = [...new Set(changedFiles.map((file) => repoPath(root, file)))].sort(); + const { createVitest } = await import("vitest/node"); + const results = []; + // Each project gets its own graph to bound memory usage. Cross-package imports + // are still followed, but projects omitted by the caller are NOT assessed. + for (const project of new Set(projects)) { + const started = performance.now(); + const ctx = await createVitest({ + root, + watch: false, + project: [project], + exclude: ["**/dist/**"], + reporters: [], + related: files.map((file) => path.resolve(root, file)), + }); + try { + if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) { + throw new Error(`Expected exactly one project named ${project}`); + } + const inventory = await ctx.globTestSpecifications(); + if (!inventory.length) throw new Error(`No test files found for ${project}`); + const ready = performance.now(); + const selected = await ctx.getRelevantTestSpecifications(); + const finished = performance.now(); + const selectedTests = selected.map((spec) => repoPath(root, spec.moduleId)).sort(); + const selectedSet = new Set(selectedTests); + results.push({ + project, + totalFiles: inventory.length, + selectedFiles: selectedTests.length, + startupSeconds: (ready - started) / 1000, + selectionSeconds: (finished - ready) / 1000, + selectedTests, + omittedTests: inventory.map((spec) => repoPath(root, spec.moduleId)) + .filter((file) => !selectedSet.has(file)).sort(), + }); + } finally { + await ctx.close(); + } + } + return { + mode: "report-only", + changedFiles: files, + limitations: [ + "Only the explicitly requested projects were assessed.", + "Selection follows imports, not filesystem reads, subprocesses, or external services.", + "Zero selected files is not proof that a change needs no tests.", + "This report must not be used to omit required CI jobs.", + ], + projects: results, + }; +} + +async function main() { + const { values } = parseArgs({ options: { + project: { type: "string", multiple: true }, + file: { type: "string", multiple: true }, + base: { type: "string" }, + output: { type: "string" }, + } }); + if (!values.project?.length || (!!values.base === !!values.file?.length)) { + throw new Error("Usage: node scripts/ci/vitest-impact-report.mjs --project NAME [--project NAME] (--file PATH [--file PATH] | --base REF) [--output FILE]"); + } + const git = (...args) => execFileSync("git", args, { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 }); + const head = git("rev-parse", "--verify", "HEAD^{commit}").trim(); + let comparison = { kind: "hypothetical", head }; + let changedFiles = values.file; + if (values.base) { + // Vitest reads the working tree, so do not label dirty sources as HEAD. + if (git("status", "--porcelain", "--untracked-files=no").trim()) { + throw new Error("--base requires a clean tracked working tree; use --file for a hypothetical change"); + } + const base = git("rev-parse", "--verify", "--end-of-options", `${values.base}^{commit}`).trim(); + // Compare the actual trees, including both sides of a rename. The caller + // supplies the event base; HEAD should be the tested PR merge commit in CI. + changedFiles = git("diff", "--name-only", "--no-renames", "-z", base, head, "--") + .split("\0").filter(Boolean); + comparison = { kind: "git-diff", base, head }; + } + process.env.NODE_ENV = "test"; + process.env.VITEST = "true"; + // Config/plugins are executable code. Run this only in a test job, never in + // the trusted metadata-only CI routing gate. Bound a failed graph traversal. + const timeout = setTimeout(() => { + console.error("Vitest impact analysis exceeded five minutes; report unavailable"); + process.exit(1); + }, 300_000); + try { + const report = await measureVitestImpact({ projects: values.project, changedFiles }); + const json = `${JSON.stringify({ ...report, comparison }, null, 2)}\n`; + if (values.output) writeFileSync(values.output, json); + else process.stdout.write(json); + for (const result of report.projects) { + console.error(`${result.project}: ${result.selectedFiles}/${result.totalFiles} files; ` + + `${result.selectionSeconds.toFixed(2)}s graph + ${result.startupSeconds.toFixed(2)}s startup`); + } + } finally { + clearTimeout(timeout); + } +} + +if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error) => { console.error(error); process.exitCode = 1; }); +} diff --git a/scripts/ci/vitest-impact.md b/scripts/ci/vitest-impact.md new file mode 100644 index 00000000000..7ee9c2d8930 --- /dev/null +++ b/scripts/ci/vitest-impact.md @@ -0,0 +1,113 @@ +# Vitest dependency selection experiment + +The production selector is documented in `doc/DEVELOPING.md`. This probe remains +a diagnostic for the raw import graph, without the production retention rules. + +This probe measures Vitest's real import graph against the current checkout. +It does not change the CI matrix or authorize skipping tests. It loads Vitest +configurations and Vite plugins, so run it in a normal test environment, never +in the trusted metadata-only routing job. + +Install the workspace dependencies and prepare the same shared build outputs +used by CI: + +```sh +pnpm install --frozen-lockfile +node scripts/ensure-plugin-build-deps.mjs +node --test scripts/ci/vitest-impact-report.integration.test.mjs +``` + +Probe a hypothetical change (the file need not actually be modified): + +```sh +node scripts/ci/vitest-impact-report.mjs \ + --project @paperclipai/ui --file ui/src/pages/Inbox.tsx \ + --output /tmp/inbox-impact.json +``` + +Or compare a base revision with the checked-out HEAD. This requires a clean +tracked working tree. For a PR, check out the tested merge commit and supply +the event's base SHA, as the existing CI planner does: + +```sh +node scripts/ci/vitest-impact-report.mjs \ + --project @paperclipai/ui --project @paperclipai/server \ + --base BASE_SHA --output /tmp/pr-impact.json +``` + +Every requested project is analyzed separately to bound memory usage. Imports +across package boundaries are followed within each graph, but unrequested +projects are **not assessed**. Runner's separate Vitest configurations, browser +E2E, Rust, typechecking, and builds are outside this experiment. The JSON records +selected and omitted test files, project inventory, startup and graph times, +and the comparison revisions. A failed or timed-out probe is not an empty plan. + +## Initial observations + +Measured on `5c807d11fa` with Vitest 5.0.3, macOS, and Node 26.4.0. These are +hypothetical single-file changes against that revision, not replays of historical +PR checkouts. CI uses Node 24 and different hardware. + +| Changed file | Assessed project | Selected / total test files | +| --- | --- | ---: | +| `ui/src/pages/Inbox.tsx` | UI | 6 / 844 | +| `ui/src/components/MarkdownBody.tsx` | UI | 135 / 844 | +| `server/src/middleware/auth.ts` | Server | 45 / 1,428 | +| `server/src/services/run-failure-report.ts` | Server | 787 / 1,428 | +| `packages/shared/src/issue-references.ts` | Shared only | 5 / 96 | +| `docs/api/issues.md` | Server | 0 / 1,428 — known false negative | + +Cold graph construction took 27.43 seconds for UI, 25.68 seconds for server, +and 2.79 seconds for shared. Subsequent queries on the same graph took about +0.05 seconds; fresh CI jobs must budget for cold construction. The six Inbox +files ran 81 assertions successfully in 20.92 seconds with four workers. +These figures exclude dependency installation and shared package builds. + +For a local execution comparison, the complete UI project passed all 10,043 +assertions in 844 files in 361.50 seconds with the same four-worker setting. +The initial Inbox graph plus selected execution took about 48 seconds, versus +six minutes for the full project. This is a local experiment, not a measured +change in GitHub CI latency. Later runs benefit from filesystem/transform caches. + +The CLI's Git-diff path was also exercised against the actual `HEAD^..HEAD` +change for PR #18321. Across its eleven changed files, the server graph selected +1,138 / 1,428 files and the shared graph selected 3 / 96. Other projects were +not assessed. Broad shared/service imports can therefore preserve most of a +suite; the Inbox reduction should not be generalized to every PR. + +## Why the raw graph is not yet a safe execution plan + +There are concrete missing edges in this repository: + +- `ui/src/pages/apps/chat/chat-ui-contract.test.ts` reads `Inbox.tsx` through + `readFileSync`. It is absent from the six-file selection even though changes + to Inbox can break its assertions. +- `server/src/__tests__/paperclip-skill-utils.test.ts` reads `docs/api/issues.md` + through `fs.readFile`. Changing that document selects zero server tests. + +A deliberate negative control confirmed the first missing edge: temporarily +append a comment containing `reason: "retry_failed_run"` to `Inbox.tsx`, which +violates the contract test's source assertion without changing runtime behavior. +The fresh graph still selected six files. Running those six plus the omitted +contract produced six passing files and exactly one failing assertion, in +`chat-ui-contract.test.ts` (92 assertions passed, one failed). The original source +was restored immediately afterward. The full baseline had passed this contract. + +The integration test uses actual Vitest graph traversal to verify transitive +imports, aliases, literal dynamic imports, direct test edits, and the missing +filesystem edge. Test modules deliberately throw if executed: graph discovery +must not execute the tests. Passing this test documents the limitation; it does +not prove that skipped repository tests are irrelevant. + +A production rollout needs explicit dependencies or an always-run lane for +filesystem/subprocess contract tests; broad fallbacks for configuration, setup, +lockfiles, deleted files, unknown inputs, and selector failures; and comparison +against full-suite failures. Source-only changes also need those protections: +the Inbox example is not a documentation-only edge case. Once qualified, compute +the selected test list before constructing the matrix and partition that list, +preserving the existing serialized, chat, and native-runner ownership rules. + +Verification for this experiment covers the real-graph integration test, CLI +probes, the complete UI project, and the deliberate negative control. Repository +wide typechecking, build, server execution, and the complete `test:run` suite +were not run; this is a report-only experiment, not a production CI rollout. diff --git a/scripts/ci/vitest-selection-graph.mjs b/scripts/ci/vitest-selection-graph.mjs new file mode 100644 index 00000000000..24c42a2df80 --- /dev/null +++ b/scripts/ci/vitest-selection-graph.mjs @@ -0,0 +1,264 @@ +import { readFileSync, realpathSync } from "node:fs"; +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import path from "node:path"; +import { isBuiltin } from "node:module"; +import { parseSync } from "oxc-parser"; +import { createVitest } from "vitest/node"; + +// Everything else is opaque by default. Do not add IO clients, module loaders, +// filesystem helpers or process launchers here to improve selection counts. +const purePackages = new Set([ + "vitest", "@vitest/expect", "@testing-library/react", "@testing-library/dom", + "@testing-library/user-event", "@testing-library/jest-dom", "react", "react-dom", + "react-router", "react-router-dom", "@tanstack/react-query", "@tanstack/query-core", + "zod", "lodash", "lodash-es", "date-fns", "clsx", "tailwind-merge", "class-variance-authority", + "lucide-react", "immer", "nanoid", "uuid", "semver", + "radix-ui", "motion", "@dnd-kit/core", "@dnd-kit/sortable", "@dnd-kit/utilities", + "i18next", "react-i18next", +]); +const pureBuiltins = new Set(["assert", "assert/strict", "path", "path/posix", "path/win32", + "url", "util", "util/types", "buffer", "events", "string_decoder", "crypto"]); +// Reviewed setup performs only temporary-home IO and Supertest socket setup, +// not reads of arbitrary repository inputs. Pin contents, not just paths: any +// edit disables this exception until its effects are reviewed again. Continue +// following local imports even for these files; helper changes affect all tests. +const reviewedSetup = new Map([ + ["server/src/__tests__/setup-supertest.ts", { + hash: "99d7193927468e98b41255ea9a5e37b1a681f02b259c304f3af1a52304eed2bb", + external: ["supertest/lib/test.js"], + }], + ["server/src/__tests__/helpers/isolated-test-home.ts", { + hash: "bd5645dc590087f26b473beab06d5c3ddd7956a79949f0b3e9713f11897a0c1b", + external: [], + }], +]); +const packageName = (specifier) => specifier.startsWith("@") + ? specifier.split("/").slice(0, 2).join("/") : specifier.split("/")[0]; +const inside = (root, file) => file.startsWith(`${root}${path.sep}`) && !file.includes(`${path.sep}node_modules${path.sep}`); + +// Parse source, including imports erased by mocks/transforms. Unknown loaders +// retain their consumers. This is intentionally conservative, not a regex-based +// promise that arbitrary JavaScript has no effects. +export function sourceDependencies(file, source) { + const ast = parseSync(file, source); + const imports = new Set(); + const reasons = new Set(); + const mockBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest") + .flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name)).map((s) => s.local.name))); + const hoistedBindings = new Set(ast.program.body.filter((node) => node.type === "ImportDeclaration" && node.source.value === "vitest") + .flatMap((node) => node.specifiers.filter((s) => s.type === "ImportSpecifier" && ["vi", "vitest"].includes(s.imported.name) + && s.local.name === s.imported.name).map((s) => s.local.name))); + const mocks = []; + if (ast.errors.length) reasons.add("unparsed source"); + const literal = (node) => node?.type === "Literal" && typeof node.value === "string"; + const importSource = (node) => { + if (literal(node)) imports.add(node.value); + else reasons.add("dynamic module name"); + }; + for (const statement of ast.program.body) { + const call = statement.type === "ExpressionStatement" ? statement.expression : null; + const callee = call?.type === "CallExpression" ? call.callee : null; + const factory = call?.arguments?.[1]; + if (callee?.type === "MemberExpression" && !callee.computed && callee.property.name === "mock" + && hoistedBindings.has(callee.object.name) && literal(call.arguments[0]) + && ["ArrowFunctionExpression", "FunctionExpression"].includes(factory?.type) && factory.params.length === 0) { + mocks.push(call.arguments[0].value); + } + } + function visit(node, parent = null) { + if (!node || typeof node !== "object") return; + if ((node.type === "ImportDeclaration" && node.importKind !== "type") + || (["ExportNamedDeclaration", "ExportAllDeclaration"].includes(node.type) && node.exportKind !== "type")) { + if (node.source) importSource(node.source); + } + if (node.type === "ImportExpression") importSource(node.source); + if (node.type === "TSImportEqualsDeclaration") reasons.add("CommonJS import"); + if (node.type === "Identifier" && /^(require|fetch|eval|Function|WebSocket|XMLHttpRequest|Worker)$/.test(node.name)) { + reasons.add("external input or dynamic code"); + } + if ((node.type === "Identifier" && node.name === "getBuiltinModule") + || (literal(node) && node.value === "getBuiltinModule")) reasons.add("dynamic builtin loader"); + if ((node.type === "Identifier" && /^(importActual|unmock|doUnmock|doMock|arguments)$/.test(node.name)) + || (literal(node) && /^(importActual|unmock|doUnmock|doMock)$/.test(node.value))) reasons.add("mock bypass"); + if (node.type === "Identifier" && mockBindings.has(node.name) && parent + && parent.type !== "ImportSpecifier" + && !(parent.type === "MemberExpression" && parent.object === node && !parent.computed)) { + reasons.add("escaped or dynamic mocking API"); + } + if (node.type === "CallExpression") { + const name = source.slice(node.callee.start, node.callee.end); + if (/^(?:vi|vitest)\.(?:mock|doMock|importActual|importMock)$/.test(name)) { + importSource(node.arguments[0]); + } + if (/\brequire\b/.test(name)) { + reasons.add("CommonJS loader"); + importSource(node.arguments[0]); + } + const viteGlob = name === "import.meta.glob" && (literal(node.arguments[0]) + || (node.arguments[0]?.type === "ArrayExpression" && node.arguments[0].elements.every(literal))); + // Vite expands literal globs into dependency edges. New files appear in + // that expansion; deleted/renamed and non-source inputs force full lanes. + if (!viteGlob && /\b(?:fetch|eval|Function|glob|globSync)\b/.test(name)) reasons.add("external input or dynamic code"); + } + for (const value of Object.values(node)) { + if (Array.isArray(value)) value.forEach((child) => visit(child, node)); + else if (value && typeof value === "object" && value.type) visit(value, node); + } + } + visit(ast.program); + return { imports: [...imports], reasons: [...reasons], mocks }; +} + +export async function selectProjectTests({ root = process.cwd(), project, changed, onlyOwnChanges = false }) { + root = realpathSync(root); + const started = performance.now(); + const absoluteChanges = changed.map((file) => path.resolve(root, file)); + const ctx = await createVitest({ root, watch: false, project: [project], + exclude: ["**/dist/**"], reporters: [], related: absoluteChanges }); + try { + if (ctx.projects.length !== 1 || ctx.projects[0].name !== project) throw new Error(`Unknown project: ${project}`); + const owner = ctx.projects[0]; + const environment = owner.vite.environments.ssr; + const inventory = await ctx.globTestSpecifications(); + if (!inventory.length) throw new Error(`Empty inventory: ${project}`); + const relative = (spec) => path.relative(root, spec.moduleId).split(path.sep).join("/"); + if (onlyOwnChanges && !absoluteChanges.some((file) => file.startsWith(`${owner.config.root}${path.sep}`))) { + // Keep coarse dependency coverage, but avoid spending a graph traversal + // on an unchanged consumer project merely to retain most of its IO tests. + const files = inventory.map(relative).sort(); + return { project, inventory: files, selected: files, related: [], alwaysRun: files, coveredChanges: [], + opaqueModules: {}, retainedReason: "Unchanged project retains complete lane coverage", seconds: (performance.now() - started) / 1000 }; + } + const tracked = new Set(execFileSync("git", ["ls-files", "--cached", "-z"], { + cwd: root, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, + }).split("\0").filter(Boolean)); + const audit = new Map(); + const parsed = new Map(); + const parse = (file) => { + if (!parsed.has(file)) parsed.set(file, sourceDependencies(file, readFileSync(file, "utf8"))); + return parsed.get(file); + }; + const resolve = async (specifier, importer) => { + try { + const resolved = await environment.pluginContainer.resolveId(specifier, importer); + if (!resolved || resolved.external || !path.isAbsolute(resolved.id) || resolved.id.includes("?")) return null; + return realpathSync(resolved.id); + } catch { return null; } // A conditional/require-only export is opaque. + }; + // Setup dependencies are not test-import edges. A change anywhere in their + // local import closure invalidates the entire project, including helpers. + const setup = new Set(); + async function setupClosure(file) { + if (setup.has(file)) return; + setup.add(file); + const source = parse(file); + const review = reviewedSetup.get(path.relative(root, file).split(path.sep).join("/")); + const reviewed = review?.hash === createHash("sha256").update(readFileSync(file)).digest("hex"); + if (!reviewed && source.reasons.length) throw new Error(`Opaque setup dependency: ${file}`); + for (const specifier of source.imports) { + if (isBuiltin(specifier)) { + if (!reviewed && !pureBuiltins.has(specifier.replace(/^node:/, ""))) throw new Error(`Opaque setup builtin: ${specifier}`); + continue; + } + if (reviewed && review.external.includes(specifier)) continue; + const dependency = await resolve(specifier, file); + if (dependency && inside(root, dependency)) await setupClosure(dependency); + else if (!dependency) throw new Error(`Unresolved setup dependency: ${specifier}`); + else if (!purePackages.has(packageName(specifier))) throw new Error(`Opaque setup package: ${specifier}`); + } + } + for (const file of [...(owner.config.setupFiles ?? []), ...(owner.config.globalSetup ?? []), + ...(owner.config.snapshotSerializers ?? [])]) { + const resolved = path.isAbsolute(file) ? file : await resolve(file, owner.vite.config.configFile); + if (resolved && inside(root, resolved)) await setupClosure(resolved); + else if (!resolved) throw new Error(`Unresolved setup module: ${file}`); + } + for (const file of owner.vite.config.configFileDependencies ?? []) setup.add(realpathSync(file)); + const setupChanged = absoluteChanges.some((file) => setup.has(file)); + const raw = await ctx.getRelevantTestSpecifications(); + const modules = environment.moduleGraph.idToModuleMap; + const resolvedDependencies = new Map(); + const localImports = new Map(); + const dependencyPath = (dep) => { + if (!resolvedDependencies.has(dep)) { + const file = dep.startsWith("/@fs/") ? dep.slice(process.platform === "win32" ? 5 : 4) + : path.join(owner.config.root, dep); + try { resolvedDependencies.set(dep, realpathSync(file)); } + catch { resolvedDependencies.set(dep, null); } + } + return resolvedDependencies.get(dep); + }; + for (const [id, module] of modules) { + if (!inside(root, id) || !/\.[cm]?[jt]sx?$/.test(id)) continue; + const source = parse(id); + const reasons = [...source.reasons]; + if (!tracked.has(path.relative(root, id).split(path.sep).join("/"))) reasons.push("generated or untracked source"); + const local = []; + localImports.set(id, local); + // Use the same resolved runtime edges as Vitest, including literal dynamic + // imports. Resolving the original syntax again uses the wrong conditions + // for require-only exports and invents edges for erased type imports. + const transformed = module.transformResult; + if (!Array.isArray(transformed?.deps)) reasons.push("unknown transform shape"); + for (const dependencyId of [...(transformed?.deps ?? []), ...(transformed?.dynamicDeps ?? [])]) { + if (isBuiltin(dependencyId)) { + if (!pureBuiltins.has(dependencyId.replace(/^node:/, ""))) reasons.push(`builtin ${dependencyId}`); + continue; + } + const dependency = dependencyPath(dependencyId); + if (dependency && inside(root, dependency)) { + local.push(dependency); + // Vitest intentionally skips some edges (e.g. CJS, externalized + // workspaces). Never regard an untracked local edge as independent. + if (!modules.get(dependency)?.transformResult) reasons.push(`untracked dependency ${path.relative(root, dependency)}`); + } else { + const external = dependency?.split(`${path.sep}node_modules${path.sep}`).at(-1).split(path.sep).join("/"); + if (!dependency || !dependency.includes(`${path.sep}node_modules${path.sep}`) || !purePackages.has(packageName(external))) { + reasons.push(`opaque dependency ${external ? packageName(external) : dependencyId}`); + } + } + } + if (!module.transformResult) reasons.push("untransformed module"); + if (reasons.length) audit.set(id, [...new Set(reasons)]); + } + const retained = []; + for (const spec of inventory) { + const mocked = new Set(); + // Honor only unconditional, explicitly supplied mock factories without an + // importOriginal parameter. Automocks, spies, unmock/importActual, dynamic + // mocking and escaped vi bindings stay conservative. The related-file + // graph remains uncut: edits to a mocked module can still select its tests. + if (owner.config.isolate !== false) { + for (const name of parse(spec.moduleId).mocks) { + const target = await resolve(name, spec.moduleId); + if (target && inside(root, target)) mocked.add(target); + } + } + const queue = [spec.moduleId], visited = new Set(); + let opaque = setupChanged || owner.config.isolate === false || !localImports.has(spec.moduleId); + while (queue.length && !opaque) { + const file = queue.pop(); + if (visited.has(file)) continue; + visited.add(file); + if (audit.has(file)) { opaque = true; break; } + for (const dependency of localImports.get(file) ?? []) if (!mocked.has(dependency)) queue.push(dependency); + } + if (opaque) retained.push(spec); + } + const selected = [...new Set([...raw, ...retained].map(relative))].sort(); + return { + project, + inventory: inventory.map(relative).sort(), + selected, + related: raw.map(relative).sort(), + alwaysRun: retained.map(relative).sort(), + coveredChanges: changed.filter((file, index) => modules.has(absoluteChanges[index]) || setup.has(absoluteChanges[index])), + opaqueModules: Object.fromEntries([...audit].map(([file, reasons]) => [path.relative(root, file), reasons])), + setupChanged, + seconds: (performance.now() - started) / 1000, + }; + } finally { + await ctx.close(); + } +} diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 197b1b5945c..ce3ee317239 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -8,6 +8,7 @@ import { loadShardDurations, selectGeneralServerShard } from "./general-server-s import { invocationEnv } from "./ci-profile/record.mjs"; import { assertSelectedTests, partitionTestLines } from "./test-line-shard.mjs"; +import { loadSelection, selectedShard } from "./ci/test-selection-manifest.mjs"; const repoRoot = process.cwd(); const scriptsDir = path.dirname(fileURLToPath(import.meta.url)); @@ -378,7 +379,7 @@ function selectSerializedSuites(routeTests, shardIndex, shardCount) { return shardFiles.map((file) => byRepoPath.get(file)); } -async function runVitest(args, label, testShard = null) { +async function runVitest(args, label, testShard = null, exactFiles = false) { console.log(`\n[test:run] ${label}`); invocationIndex += 1; const tempRootParent = process.platform === "win32" ? os.tmpdir() : "/tmp"; @@ -436,7 +437,9 @@ async function runVitest(args, label, testShard = null) { args = [...args.filter((arg) => arg !== chatSuite), ...runFilters, "--allowOnly=false"]; } await new Promise((resolve, reject) => { - const child = spawn("pnpm", ["exec", "vitest", "run", ...sourceOnlyVitestArgs, ...args], { + const command = exactFiles ? process.execPath : "pnpm"; + const prefix = exactFiles ? [path.join(scriptsDir, "ci/run-selected-vitest.mjs")] : ["exec", "vitest", "run"]; + const child = spawn(command, [...prefix, ...sourceOnlyVitestArgs, ...args], { cwd: repoRoot, env: invocationEnv("vitest", env), stdio: "inherit", @@ -615,6 +618,24 @@ const generalServerTestFiles = [ .sort((a, b) => a.localeCompare(b)); const options = parseCliOptions(process.argv.slice(2)); +const selectionProjects = options.mode === serializedModeName || generalServerGroups.includes(options.group) + ? ["@paperclipai/server"] + : options.group === generalWorkspacesAGroupName ? generalWorkspacesAProjects + : options.group === generalWorkspacesBGroupName ? generalWorkspacesBProjects + : options.group === generalWorkspacesCGroupName ? generalWorkspacesCProjects : []; +const selection = process.env.PAPERCLIP_CI_TEST_SELECTION + ? selectedShard(loadSelection(), options, selectionProjects) : null; +if (selection?.["@paperclipai/server"]) { + const owned = new Set(options.mode === serializedModeName ? routeTests.map((test) => test.repoPath) + : generalServerTestFiles.filter((file) => !excludedServerSuites(options.group).includes(file))); + for (const file of selection["@paperclipai/server"]) { + if (!owned.has(file)) fail(`Selected test belongs to another server lane: ${file}`); + } +} +if (selection && options.dryRun) { + writeFileSync(1, JSON.stringify({ selection })); + process.exit(0); +} if (options.dryRun) { const serializedSuites = options.mode === serializedModeName @@ -670,7 +691,16 @@ if (options.dryRun) { } try { - if (options.mode === generalModeName || options.mode === allModeName) { + if (selection) { + for (const [project, files] of Object.entries(selection)) { + const serverArgs = project !== "@paperclipai/server" ? [] + : options.mode === serializedModeName || process.env.PAPERCLIP_TEST_DATABASE_URL + ? ["--pool=forks", "--isolate", ...serializedServerVitestArgs] + : ["--pool=forks", "--isolate", "--fileParallelism", "--maxWorkers=2"]; + await runVitest(["--project", project, ...serverArgs, ...files], + `selected ${project}: ${files.length} files`, null, true); + } + } else if (options.mode === generalModeName || options.mode === allModeName) { if (options.group) { await runGeneralGroup(routeTests, options.group, options.shardIndex, options.shardCount); } else { @@ -678,7 +708,7 @@ try { } } - if (options.mode === serializedModeName || options.mode === allModeName) { + if (!selection && (options.mode === serializedModeName || options.mode === allModeName)) { await runSerializedSuites(routeTests, options.shardIndex ?? 0, options.shardCount ?? 1); } } catch (error) { diff --git a/turbo.json b/turbo.json index dfc4e2576cc..618ca30fd37 100644 --- a/turbo.json +++ b/turbo.json @@ -8,6 +8,10 @@ ".npmrc", "tsconfig*.json", "vitest.config.ts", + "**/vitest.config.*", + "**/vite.config.*", + "**/vitest.setup.*", + "**/tsconfig*.json", "scripts/**", ".github/**", "patches/**",