diff --git a/.github/ci-task-matrix.json b/.github/ci-task-matrix.json index 53d1909b158..7432f4adaed 100644 --- a/.github/ci-task-matrix.json +++ b/.github/ci-task-matrix.json @@ -158,13 +158,23 @@ "task": "ci:runner-rust" }, { - "lane_label": "vitest package", - "command": "test:typescript:vitest --shard=1/2", + "lane_label": "vitest package (1/3)", + "command": "test:typescript:vitest --package-lane=1/3", + "task": "ci:runner-vitest" + }, + { + "lane_label": "vitest package (2/3)", + "command": "test:typescript:vitest --package-lane=2/3", + "task": "ci:runner-vitest" + }, + { + "lane_label": "vitest package (3/3)", + "command": "test:typescript:vitest --package-lane=3/3", "task": "ci:runner-vitest" }, { "lane_label": "vitest native server", - "command": "test:typescript:vitest --shard=2/2", + "command": "test:typescript:vitest --native-server", "release_crate": true, "task": "ci:runner-vitest" } diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index b3b35cb0093..bc737937f70 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -825,7 +825,7 @@ jobs: timeout-minutes: 20 strategy: fail-fast: false - matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_paperclip_runner.matrix) || '{"include":[{"lane_label":"static checks","command":"check:static","task":"ci:runner-static"},{"lane_label":"rust","command":"check:runner","release_crate":true,"task":"ci:runner-rust"},{"lane_label":"vitest package","command":"test:typescript:vitest --shard=1/2","task":"ci:runner-vitest"},{"lane_label":"vitest native server","command":"test:typescript:vitest --shard=2/2","release_crate":true,"task":"ci:runner-vitest"}]}') }} + matrix: ${{ fromJSON(needs.gate.outputs.plan != '' && toJSON(fromJSON(needs.gate.outputs.plan).jobs.verify_paperclip_runner.matrix) || '{"include":[{"lane_label":"static checks","command":"check:static","task":"ci:runner-static"},{"lane_label":"rust","command":"check:runner","release_crate":true,"task":"ci:runner-rust"},{"lane_label":"vitest package (1/3)","command":"test:typescript:vitest --package-lane=1/3","task":"ci:runner-vitest"},{"lane_label":"vitest package (2/3)","command":"test:typescript:vitest --package-lane=2/3","task":"ci:runner-vitest"},{"lane_label":"vitest package (3/3)","command":"test:typescript:vitest --package-lane=3/3","task":"ci:runner-vitest"},{"lane_label":"vitest native server","command":"test:typescript:vitest --native-server","release_crate":true,"task":"ci:runner-vitest"}]}') }} env: PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} diff --git a/doc/DEVELOPING.md b/doc/DEVELOPING.md index 32302512857..bb7692a25c5 100644 --- a/doc/DEVELOPING.md +++ b/doc/DEVELOPING.md @@ -420,12 +420,18 @@ This gives interrupted jobs an independent recovery path. It keeps the same source, commands, assertions and required gates. A failed test still needs a fix; a successful rerun does not explain an earlier failure. -The two PR Runner Vitest jobs separate package tests from native server tests. -The first runs the complete package inventory; the second runs the complete -native server group. The existing `--shard=1/2` and `--shard=2/2` commands route -through `run-pr-vitest-lane.mjs` for compatibility with the master-pinned -workflow. Other workflow names and shard counts keep their existing coverage. -Both jobs still prepare debug binaries. A separate exact-key debug cache holds +The PR Runner Vitest jobs separate package tests from native server tests. +Three `--package-lane=K/3` jobs each run one duration-balanced partition of the +package inventory, and one `--native-server` job runs the complete native server +group. `run-pr-vitest-lane.mjs` asks `vitest list` for the inventory, so it +follows the `vitest.config.ts` includes. It partitions the files with the general +server partitioner, using the weights in +`packages/paperclip-runner/scripts/vitest-lane-durations.json`. Files missing +from that manifest get the median weight. The legacy `--shard=1/2` and +`--shard=2/2` commands keep their earlier PR behavior: the first runs the whole +package inventory and the second runs the native group. Other workflow names and +shard counts keep their existing coverage. Every lane still prepares debug +binaries. A separate exact-key debug cache holds the workspace library and every Cargo binary target. Its key includes source, protocol and fixture content, Cargo configuration, compiler flags, toolchain and checkout path. Only the Rust lane of a verified master push saves it. diff --git a/package.json b/package.json index 00c95e37c43..fb28fe11c50 100644 --- a/package.json +++ b/package.json @@ -61,7 +61,7 @@ "smoke:notion-generic-live": "node scripts/smoke/notion-generic-live.mjs", "smoke:posthog-live": "node scripts/smoke/posthog-live.mjs", "smoke:terminal-bench-loop-skill": "node scripts/smoke/terminal-bench-loop-skill-smoke.mjs", - "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs", + "test:release-registry": "node scripts/ci-profile/node-test.mjs scripts/ci-profile/reporters.test.mjs scripts/__tests__/runner-vitest-package-lanes.test.mjs scripts/__tests__/canary-startup-playwright.test.mjs scripts/__tests__/cursor-public-install-sandbox.test.mjs scripts/__tests__/grok-public-install-sandbox.test.mjs scripts/__tests__/public-install-closure.test.mjs scripts/__tests__/public-install-registry.test.mjs scripts/__tests__/prefetch-child.test.mjs scripts/__tests__/stage-bundled-packages.test.mjs scripts/__tests__/pack-staged-packages.test.mjs scripts/__tests__/retain-runner-qualification-packages.test.mjs scripts/__tests__/runner-release-crate-cache.test.mjs scripts/__tests__/runner-debug-crate-cache.test.mjs scripts/__tests__/agent-build-cache.test.mjs scripts/__tests__/vitest-worker-isolation.test.mjs scripts/verify-release-registry-state.test.mjs scripts/release-package-map.test.mjs scripts/check-release-package-bootstrap.test.mjs scripts/check-no-git-push.test.mjs scripts/release-lib.test.mjs scripts/release-registry-versions.test.mjs scripts/link-plugin-dev-sdk.test.js scripts/acpx-patch-packaging.test.mjs scripts/service-onboard-smoke.test.mjs scripts/docker-onboard-smoke.test.mjs scripts/preview-artifacts.test.mjs scripts/cloud-migrator-artifacts.test.mjs scripts/typecheck-test-programs.test.mjs scripts/__tests__/local-verify.test.mjs scripts/test-sandbox-providers.test.mjs", "storybook-visual:baseline": "node scripts/storybook-visual-baseline.mjs", "test:storybook-visual": "node scripts/storybook-visual-baseline.mjs download && node scripts/storybook-visual-baseline.mjs verify && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts", "test:storybook-visual:update": "node scripts/storybook-visual-baseline.mjs download && pnpm build-storybook && npx playwright test --config tests/storybook-visual/playwright.config.ts --update-snapshots && node scripts/storybook-visual-baseline.mjs pack", diff --git a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs index 40e5bd2f89b..4b13440590b 100644 --- a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs +++ b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs @@ -1,38 +1,92 @@ -// The two PR Vitest slots cover separate workloads: the first runs every -// package test and the second runs the native server integration group. -// Both need debug fixtures; only the native group also builds release units. -// Keep the existing --shard commands compatible with master-pinned workflows. -// Local/other callers retain ordinary package sharding. Other PR shard counts -// keep package shards plus native tests on the final lane, preserving coverage. +// PR Runner Vitest lanes. `--package-lane=K/N` runs the Kth of N +// duration-balanced partitions of the package inventory, and `--native-server` +// runs the native server integration group. Every lane prepares debug +// fixtures; only the native group also needs release units (see the matrix). +// +// The --shard commands stay compatible with master-pinned workflows: under +// the PR workflow with two shards, the first runs every package test and the +// second runs the native server group. Local/other callers retain ordinary +// package sharding. Other PR shard counts keep package shards plus native +// tests on the final lane, preserving coverage. import { spawnSync } from "node:child_process"; import { dirname, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { invocationEnv } from "../../../scripts/ci-profile/record.mjs"; +import { filesMatchedByFilters, listPackageVitestFiles, planPackageLanes } from "./vitest-package-lanes.mjs"; const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const workspaceRoot = resolve(packageRoot, "../.."); const args = process.argv.slice(2).filter((value) => value !== "--dry-run"); const dryRun = process.argv.includes("--dry-run"); +function fail(message) { + console.error(`[pr-vitest-lane] ${message}`); + process.exit(1); +} + const shardArg = args.find((value) => value.startsWith("--shard=")); const shardMatch = shardArg ? /^--shard=(\d+)\/(\d+)$/.exec(shardArg) : null; if (args.filter(value => value.startsWith("--shard=")).length > 1 || (shardArg && !shardMatch)) { - console.error(`[pr-vitest-lane] unrecognized shard argument: ${shardArg}`); - process.exit(1); + fail(`unrecognized shard argument: ${shardArg}`); } if (shardMatch && (Number(shardMatch[1]) < 1 || Number(shardMatch[1]) > Number(shardMatch[2]))) { throw new Error(`Invalid shard: ${shardArg}`); } + +const laneArgs = args.filter((value) => value.startsWith("--package-lane")); +const laneMatch = laneArgs.length === 1 ? /^--package-lane=(\d+)\/(\d+)$/.exec(laneArgs[0]) : null; +if (laneArgs.length > 1 || (laneArgs.length === 1 && !laneMatch)) { + fail(`unrecognized package lane argument: ${laneArgs.join(" ")}`); +} +const laneIndex = laneMatch ? Number(laneMatch[1]) : null; +const laneCount = laneMatch ? Number(laneMatch[2]) : null; +if (laneMatch && (laneCount < 1 || laneIndex < 1 || laneIndex > laneCount)) { + fail(`invalid package lane: ${laneArgs[0]}`); +} +const nativeServerOnly = args.includes("--native-server"); +if ([shardArg, laneMatch, nativeServerOnly].filter(Boolean).length > 1) { + fail("--shard, --package-lane and --native-server are mutually exclusive"); +} +if (nativeServerOnly && args.length > 1) { + fail(`--native-server takes no other arguments: ${args.join(" ")}`); +} + const isFinalShard = !shardArg || shardMatch[1] === shardMatch[2]; const isPrWorkflow = process.env.GITHUB_WORKFLOW === "PR"; -const runNativeRunnerGroup = isPrWorkflow && isFinalShard; // Reuse the two existing CI slots: all package files in the first, all native // server files in the second. Keeping the old command arguments makes this // safe while the trusted workflow still comes from master. Other shard counts // and local callers retain ordinary Vitest sharding and the final-lane append. const separateWorkloads = isPrWorkflow && shardMatch?.[2] === "2"; -const runPackageTests = !separateWorkloads || !isFinalShard; -const packageArgs = separateWorkloads ? args.filter((arg) => arg !== shardArg) : args; + +let runNativeRunnerGroup; +let runPackageTests; +let packageArgs; +let packageLane = null; +if (laneMatch) { + const inventory = listPackageVitestFiles(); + const lanes = planPackageLanes(inventory, laneCount); + const lane = lanes[laneIndex - 1]; + // A filter that also selects a file owned by another lane runs it twice; it + // never drops coverage, so warn instead of failing the lane. + const extra = filesMatchedByFilters(lane.files, inventory).filter((file) => !lane.files.includes(file)); + if (extra.length) { + console.error(`[pr-vitest-lane] package lane ${laneIndex}/${laneCount} filters also select: ${extra.join(", ")}`); + } + packageLane = { index: laneIndex, count: laneCount, inventory: inventory.length, estimatedMs: lane.estimatedMs, files: lane.files }; + runNativeRunnerGroup = false; + // An empty file list would make `vitest run` select the whole inventory. + runPackageTests = lane.files.length > 0; + packageArgs = [...args.filter((arg) => arg !== laneArgs[0]), ...lane.files]; +} else if (nativeServerOnly) { + runNativeRunnerGroup = true; + runPackageTests = false; + packageArgs = []; +} else { + runNativeRunnerGroup = isPrWorkflow && isFinalShard; + runPackageTests = !separateWorkloads || !isFinalShard; + packageArgs = separateWorkloads ? args.filter((arg) => arg !== shardArg) : args; +} const plannedCommands = [ { command: "pnpm", args: ["run", "ensure:eval-build-deps"], cwd: packageRoot }, @@ -50,10 +104,20 @@ const plannedCommands = [ ]; if (dryRun) { - console.log(JSON.stringify({ isPrWorkflow, isFinalShard, runNativeRunnerGroup, runPackageTests, plannedCommands }, null, 2)); + console.log(JSON.stringify({ + isPrWorkflow, + isFinalShard, + runNativeRunnerGroup, + runPackageTests, + ...(packageLane ? { packageLane } : {}), + plannedCommands, + }, null, 2)); process.exit(0); } +if (packageLane) { + console.log(`[pr-vitest-lane] package lane ${packageLane.index}/${packageLane.count}: ${packageLane.files.length}/${packageLane.inventory} files, estimated ${Math.round(packageLane.estimatedMs / 1000)}s`); +} for (const planned of plannedCommands) { const result = spawnSync(planned.command, planned.args, { cwd: planned.cwd, diff --git a/packages/paperclip-runner/scripts/vitest-lane-durations.json b/packages/paperclip-runner/scripts/vitest-lane-durations.json new file mode 100644 index 00000000000..f5bb05a609f --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-lane-durations.json @@ -0,0 +1,329 @@ +{ + "$comment": "Per-file Paperclip Runner package Vitest weights that packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs --package-lane=K/N uses to balance the PR package lanes. Same schema as scripts/general-server-shard-durations.json. Seeded from the test/hook durations of PR job 128144114326 (collection and setup excluded). Files missing here receive the partitioner's median weight. Timing changes only partition work; they never authorize skipping a test.", + "unit": "ms", + "sources": [ + { + "run": 42290920882, + "attempt": 1, + "sha": "d07c48a2bbe8ed5f5db73fd6d0707194183fd1eb", + "url": "https://github.com/paperclipai/paperclip/actions/runs/42290920882" + } + ], + "coverage": { + "measured": 310, + "withSetup": 0, + "inventory": 311 + }, + "durations": { + "packages/paperclip-runner/devtools/browser/src/live/transcript-model.test.ts": 366, + "packages/paperclip-runner/devtools/browser/src/live/use-live-console.test.ts": 100, + "packages/paperclip-runner/devtools/issue-thread/src/EvidencePanel.test.ts": 100, + "packages/paperclip-runner/devtools/issue-thread/src/route.test.ts": 100, + "packages/paperclip-runner/examples/scenario-explorer/src/chat.test.tsx": 190, + "packages/paperclip-runner/examples/scenario-explorer/src/explorer.test.tsx": 164, + "packages/paperclip-runner/examples/scenario-explorer/src/route.test.ts": 100, + "packages/paperclip-runner/scripts/live-console-browser-server.test.mjs": 159, + "packages/paperclip-runner/scripts/local-runner-browser-server.test.mjs": 100, + "packages/paperclip-runner/scripts/vitest-sequencer.test.mjs": 100, + "packages/paperclip-runner/src/agent-identity-environment.test.ts": 100, + "packages/paperclip-runner/src/backends/harness-driver-backend.test.ts": 100, + "packages/paperclip-runner/src/backends/native-backend-factory.test.ts": 100, + "packages/paperclip-runner/src/backends/native-instruction-measurement.test.ts": 13816, + "packages/paperclip-runner/src/backends/runtime-context.test.ts": 100, + "packages/paperclip-runner/src/browser/client.test.ts": 100, + "packages/paperclip-runner/src/browser/transcript-model.test.ts": 431, + "packages/paperclip-runner/src/catalog/catalog-docs.test.ts": 100, + "packages/paperclip-runner/src/catalog/operation-groups-doc.test.ts": 100, + "packages/paperclip-runner/src/catalog/protocol-action-contracts.test.ts": 301, + "packages/paperclip-runner/src/catalog/reconciliation.test.ts": 100, + "packages/paperclip-runner/src/catalog/semantic-action-catalog.test.ts": 334, + "packages/paperclip-runner/src/claude-subagent-lineage.test.ts": 100, + "packages/paperclip-runner/src/cli/acpx-runtime-sidecar.test.ts": 18684, + "packages/paperclip-runner/src/cli/acpx-sidecar-input.test.ts": 100, + "packages/paperclip-runner/src/cli/acpx-sidecar-locations.test.ts": 100, + "packages/paperclip-runner/src/cli/acpx-tool-detail-redaction.test.ts": 100, + "packages/paperclip-runner/src/cli/eval-failure-diagnostics.test.ts": 100, + "packages/paperclip-runner/src/cli/eval-gateway-runtime.test.ts": 100, + "packages/paperclip-runner/src/cli/eval-provider-runtime.test.ts": 100, + "packages/paperclip-runner/src/cli/eval-session-contract.test.ts": 123, + "packages/paperclip-runner/src/cli/eval-session-entrypoint.test.ts": 1800, + "packages/paperclip-runner/src/cli/opencode-app-server-proxy.test.ts": 100, + "packages/paperclip-runner/src/cli/opencode-proxy-collaboration-mode.test.ts": 100, + "packages/paperclip-runner/src/cli/opencode-proxy-completion.test.ts": 100, + "packages/paperclip-runner/src/cli/opencode-proxy-events.test.ts": 100, + "packages/paperclip-runner/src/cli/opencode-proxy-input.test.ts": 5018, + "packages/paperclip-runner/src/cli/opencode-proxy-output.test.ts": 100, + "packages/paperclip-runner/src/cli/opencode-proxy-permission-mode.test.ts": 100, + "packages/paperclip-runner/src/cli/opencode-proxy-thread-read.test.ts": 1898, + "packages/paperclip-runner/src/cli/opencode-proxy-usage.test.ts": 100, + "packages/paperclip-runner/src/compatibility.test.ts": 100, + "packages/paperclip-runner/src/configured-environment.test.ts": 100, + "packages/paperclip-runner/src/conformance/capability-eval-suite.test.ts": 100, + "packages/paperclip-runner/src/conformance/capability-semantic-conformance.test.ts": 726, + "packages/paperclip-runner/src/conformance/control-plane-port.test.ts": 100, + "packages/paperclip-runner/src/conformance/harness-driver.test.ts": 100, + "packages/paperclip-runner/src/conformance/local-provider-fault-matrix.test.ts": 116, + "packages/paperclip-runner/src/conformance/semantic-conformance.test.ts": 100, + "packages/paperclip-runner/src/contracts/completion-result.test.ts": 380, + "packages/paperclip-runner/src/contracts/native-execution.test.ts": 100, + "packages/paperclip-runner/src/contracts/provider-trace-file-sink.test.ts": 100, + "packages/paperclip-runner/src/contracts/question-adapter-conformance.test.ts": 100, + "packages/paperclip-runner/src/contracts/question-set.test.ts": 100, + "packages/paperclip-runner/src/contracts/registered-asset-instructions.test.ts": 100, + "packages/paperclip-runner/src/contracts/runtime-context.test.ts": 100, + "packages/paperclip-runner/src/control-plane/acpx-controller-recovery.test.ts": 81950, + "packages/paperclip-runner/src/control-plane/cursor-cloud-runner-launch.test.ts": 106659, + "packages/paperclip-runner/src/control-plane/durable-prp-control-plane.test.ts": 9141, + "packages/paperclip-runner/src/control-plane/runner-recovery-fixture.test.ts": 8919, + "packages/paperclip-runner/src/devtools/index.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/acp-permission-adapter.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/acp-question-adapter.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/agent-files-binding.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/antigravity-acpx-runtime.test.ts": 6563, + "packages/paperclip-runner/src/drivers/acpx/antigravity-admission.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/antigravity-auth-stream-order.test.ts": 696, + "packages/paperclip-runner/src/drivers/acpx/antigravity-auth.test.ts": 2324, + "packages/paperclip-runner/src/drivers/acpx/antigravity-credentials.test.ts": 166, + "packages/paperclip-runner/src/drivers/acpx/antigravity-models.test.ts": 32227, + "packages/paperclip-runner/src/drivers/acpx/antigravity-probe-retirement.test.ts": 1047, + "packages/paperclip-runner/src/drivers/acpx/candidate-admission.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/claude-effort.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/claude-status-banners.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/codex-acpx-driver.test.ts": 893, + "packages/paperclip-runner/src/drivers/acpx/codex-credentials.test.ts": 6558, + "packages/paperclip-runner/src/drivers/acpx/codex-runtime-adapter.test.ts": 2923, + "packages/paperclip-runner/src/drivers/acpx/command-lease-owner.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-events.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-evidence.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-extension-adapter.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-installation.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-message-identity.test.ts": 6310, + "packages/paperclip-runner/src/drivers/acpx/copilot-policy.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-profile.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-protocol-guard.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-registry.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/copilot-runtime-host.test.ts": 188, + "packages/paperclip-runner/src/drivers/acpx/copilot-tool-evidence.test.ts": 125, + "packages/paperclip-runner/src/drivers/acpx/cursor-cloud-candidate-profiles.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/cursor-extensions.test.ts": 978, + "packages/paperclip-runner/src/drivers/acpx/cursor-installation.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/cursor-instructions.test.ts": 3211, + "packages/paperclip-runner/src/drivers/acpx/cursor-launch-policy.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/cursor-mode.test.ts": 3463, + "packages/paperclip-runner/src/drivers/acpx/cursor-permission-identity.test.ts": 2748, + "packages/paperclip-runner/src/drivers/acpx/cursor-runtime-cache.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/cursor-sdk-interactions.test.ts": 181, + "packages/paperclip-runner/src/drivers/acpx/cursor-tool-evidence.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/driver-profile.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/environment.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/executable-digest-cache.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/generated-sidecar-contract.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/github-launcher.test.ts": 2628, + "packages/paperclip-runner/src/drivers/acpx/grok-credentials.test.ts": 137, + "packages/paperclip-runner/src/drivers/acpx/grok-effort.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/grok-native-session-options.test.ts": 2680, + "packages/paperclip-runner/src/drivers/acpx/grok-usage.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/grok-web-search.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/installation-integrity.test.ts": 2894, + "packages/paperclip-runner/src/drivers/acpx/installed-provider-authority.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/kimi-installation.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/kimi-instructions.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/kimi-message-boundary.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/kimi-metadata-probe.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/kimi-models.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/model-verification.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/native-distribution-integrity.test.ts": 7792, + "packages/paperclip-runner/src/drivers/acpx/native-skill-prompt.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/permission-policy.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-acp-runtime.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-extension-adapter.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-installation.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-message-projection.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-native-admission.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-provider-config.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-provision.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-rpc-deltas.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/pi-runtime-extension.test.ts": 267, + "packages/paperclip-runner/src/drivers/acpx/pi-thinking.test.ts": 1331, + "packages/paperclip-runner/src/drivers/acpx/pi-verified-runtime.test.ts": 289, + "packages/paperclip-runner/src/drivers/acpx/private-snapshot.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/profile-compatibility.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/profile-extensions.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/profile-installation.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/provider-assets-root.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/provider-mode.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/qualified-profiles.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/recovery-identity.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/response-delivery.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/runtime-host.test.ts": 7721, + "packages/paperclip-runner/src/drivers/acpx/runtime-sandbox.test.ts": 1485, + "packages/paperclip-runner/src/drivers/acpx/session-goals.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/sidecar-protocol.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/single-read-evidence.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/turn-cancellation.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/turn-controls.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/usage-accounting.test.ts": 100, + "packages/paperclip-runner/src/drivers/acpx/verified-runtime-executable.test.ts": 100, + "packages/paperclip-runner/src/drivers/agent-observability.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/app-server-transport.test.ts": 491, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.candidate-continuation.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.conformance.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.context-control.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.events.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.lifecycle.test.ts": 114, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.recovery.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.runtime-requests.test.ts": 177, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.semantic-results.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-app-server-driver.terminal-replay.test.ts": 233, + "packages/paperclip-runner/src/drivers/codex/codex-boundaries.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-history.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-notification-identity.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-protocol-cleanup.test.ts": 802, + "packages/paperclip-runner/src/drivers/codex/codex-protocol-integrity.test.ts": 3903, + "packages/paperclip-runner/src/drivers/codex/codex-question-adapter.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-resume-usage.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-security-config.test.ts": 605, + "packages/paperclip-runner/src/drivers/codex/codex-startup-trust.test.ts": 270, + "packages/paperclip-runner/src/drivers/codex/codex-thread-normalization.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-turn-diff.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/codex-usage-baseline.test.ts": 100, + "packages/paperclip-runner/src/drivers/codex/prepared-context-transport.test.ts": 100, + "packages/paperclip-runner/src/drivers/completion-continuation.test.ts": 100, + "packages/paperclip-runner/src/drivers/copilot-metadata-probe.test.ts": 5107, + "packages/paperclip-runner/src/drivers/cursor-cloud/app-server-proxy.test.ts": 1419, + "packages/paperclip-runner/src/drivers/cursor-cloud/artifact-delivery.test.ts": 4831, + "packages/paperclip-runner/src/drivers/cursor-cloud/artifact-staging-budget.test.ts": 1520, + "packages/paperclip-runner/src/drivers/cursor-cloud/checkpoint-archive.test.ts": 1065, + "packages/paperclip-runner/src/drivers/cursor-cloud/cloud-client.test.ts": 191, + "packages/paperclip-runner/src/drivers/cursor-cloud/cursor-cloud-driver.test.ts": 15764, + "packages/paperclip-runner/src/drivers/cursor-cloud/inline-assets.test.ts": 100, + "packages/paperclip-runner/src/drivers/cursor-cloud/observation-continuity.test.ts": 1948, + "packages/paperclip-runner/src/drivers/dot/dot-harness-driver.test.ts": 100, + "packages/paperclip-runner/src/drivers/dot/runnerd-dot-driver.test.ts": 11738, + "packages/paperclip-runner/src/drivers/muse/runnerd-muse-driver.test.ts": 16068, + "packages/paperclip-runner/src/drivers/opencode/company-observability.test.ts": 3033, + "packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts": 1857, + "packages/paperclip-runner/src/drivers/opencode/interrupt-empty-response.test.ts": 1401, + "packages/paperclip-runner/src/drivers/opencode/interrupt-pending-tool-call.test.ts": 1010, + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge-diagnostics.test.ts": 187, + "packages/paperclip-runner/src/drivers/opencode/mcp-bridge.test.ts": 4223, + "packages/paperclip-runner/src/drivers/opencode/opencode-command.test.ts": 100, + "packages/paperclip-runner/src/drivers/opencode/opencode-question-normalization.test.ts": 100, + "packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts": 96360, + "packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts": 532, + "packages/paperclip-runner/src/drivers/opencode/performance-diagnostics.test.ts": 100, + "packages/paperclip-runner/src/drivers/opencode/prepared-context-transport.test.ts": 2644, + "packages/paperclip-runner/src/drivers/opencode/reasoning-mode.test.ts": 2562, + "packages/paperclip-runner/src/drivers/opencode/sse-frames.test.ts": 884, + "packages/paperclip-runner/src/drivers/provider-failure.test.ts": 100, + "packages/paperclip-runner/src/drivers/provider-notices.test.ts": 100, + "packages/paperclip-runner/src/drivers/provider-rate-limits.test.ts": 100, + "packages/paperclip-runner/src/drivers/runner-tool-bridge.claude-schema.test.ts": 3441, + "packages/paperclip-runner/src/drivers/runner-tool-bridge.test.ts": 4224, + "packages/paperclip-runner/src/drivers/runtime-context-materializer.test.ts": 103, + "packages/paperclip-runner/src/drivers/semantic-tool-receipt.test.ts": 100, + "packages/paperclip-runner/src/drivers/stdio-mcp.test.ts": 100, + "packages/paperclip-runner/src/eval/eval-bundle.test.ts": 100, + "packages/paperclip-runner/src/eval/eval-execution.test.ts": 1075, + "packages/paperclip-runner/src/eval/eval-scoring.test.ts": 100, + "packages/paperclip-runner/src/eval/eval-slice.test.ts": 100, + "packages/paperclip-runner/src/eval/live-workflow-executor.test.ts": 100, + "packages/paperclip-runner/src/eval/workflow-evals.test.ts": 132, + "packages/paperclip-runner/src/evals/attempt-journal.test.ts": 100, + "packages/paperclip-runner/src/evals/compatibility.test.ts": 100, + "packages/paperclip-runner/src/evals/model-pricing.test.ts": 100, + "packages/paperclip-runner/src/evals/native-execution.test.ts": 2001, + "packages/paperclip-runner/src/evals/runnerd-artifact.test.ts": 100, + "packages/paperclip-runner/src/governance-events.test.ts": 100, + "packages/paperclip-runner/src/issue-thread/issue-thread.test.ts": 100, + "packages/paperclip-runner/src/issue-thread/public-view.test.ts": 100, + "packages/paperclip-runner/src/live/acpx-request-recovery.test.ts": 100, + "packages/paperclip-runner/src/live/antigravity-candidate-startup.test.ts": 100, + "packages/paperclip-runner/src/live/bundled-remote-provider-pack.test.ts": 100, + "packages/paperclip-runner/src/live/clean-room-server.test.ts": 25279, + "packages/paperclip-runner/src/live/clean-room.test.ts": 100, + "packages/paperclip-runner/src/live/kimi-candidate-startup.test.ts": 100, + "packages/paperclip-runner/src/live/linux-process-start.test.ts": 1114, + "packages/paperclip-runner/src/live/live-session.test.ts": 51829, + "packages/paperclip-runner/src/live/local-integrity-boundary-golden.test.ts": 100, + "packages/paperclip-runner/src/live/native-probe-cleanup.test.ts": 13509, + "packages/paperclip-runner/src/live/run-result-feedback.test.ts": 100, + "packages/paperclip-runner/src/live/runner-binary.test.ts": 100, + "packages/paperclip-runner/src/live/runner-launch-preparation-store.test.ts": 100, + "packages/paperclip-runner/src/live/runner-launch-preparation.test.ts": 100, + "packages/paperclip-runner/src/live/runner-prepared-launch-workspace.test.ts": 3935, + "packages/paperclip-runner/src/live/runner-provider-launch.test.ts": 100, + "packages/paperclip-runner/src/live/runner-tool-capacity.test.ts": 100, + "packages/paperclip-runner/src/live/runnerd-acpx-permission-forwarding.test.ts": 100, + "packages/paperclip-runner/src/live/runnerd-acpx-permission-ordering.test.ts": 237, + "packages/paperclip-runner/src/live/runnerd-acpx-permissions.test.ts": 100, + "packages/paperclip-runner/src/live/runnerd-acpx-questions.test.ts": 100, + "packages/paperclip-runner/src/live/runnerd-acpx-source-reconciliation.test.ts": 27202, + "packages/paperclip-runner/src/live/runnerd-codex-transport-adoption.test.ts": 51111, + "packages/paperclip-runner/src/live/runnerd-codex-transport-attachment-recovery.test.ts": 44665, + "packages/paperclip-runner/src/live/runnerd-codex-transport-cold-restore.test.ts": 3362, + "packages/paperclip-runner/src/live/runnerd-codex-transport-deadlines.test.ts": 101360, + "packages/paperclip-runner/src/live/runnerd-codex-transport-settlement.test.ts": 120438, + "packages/paperclip-runner/src/live/runnerd-codex-transport-suspension.test.ts": 13354, + "packages/paperclip-runner/src/live/runnerd-codex-transport-warm-attachment.test.ts": 50422, + "packages/paperclip-runner/src/live/runnerd-codex-transport.test.ts": 126679, + "packages/paperclip-runner/src/live/runnerd-contract-types.test.ts": 100, + "packages/paperclip-runner/src/live/runnerd-controller-outage.test.ts": 5960, + "packages/paperclip-runner/src/live/runnerd-credential-updates.test.ts": 4475, + "packages/paperclip-runner/src/live/runnerd-detach-durability.test.ts": 5556, + "packages/paperclip-runner/src/live/runnerd-opencode-assigned-tools.test.ts": 8593, + "packages/paperclip-runner/src/live/runnerd-opencode-streaming-tools.test.ts": 52166, + "packages/paperclip-runner/src/live/runnerd-restart-continuation.test.ts": 12270, + "packages/paperclip-runner/src/live/runnerd-runtime-activity.test.ts": 2946, + "packages/paperclip-runner/src/live/runnerd-saved-continuation.test.ts": 5314, + "packages/paperclip-runner/src/live/runnerd-semantic-handler-readiness.test.ts": 3841, + "packages/paperclip-runner/src/live/runnerd-source-reconciliation.test.ts": 39739, + "packages/paperclip-runner/src/live/runnerd-staged-handoff.test.ts": 40641, + "packages/paperclip-runner/src/live/runnerd-tool-response.test.ts": 100, + "packages/paperclip-runner/src/live/runnerd-trace-frame-index.test.ts": 219, + "packages/paperclip-runner/src/live/runnerd-welcome.integration.test.ts": 4594, + "packages/paperclip-runner/src/live/scenario-chat-demo-server.test.ts": 280, + "packages/paperclip-runner/src/live/scenario-chat-proxy-config.test.ts": 100, + "packages/paperclip-runner/src/live/tailnet-gateway.test.ts": 100, + "packages/paperclip-runner/src/live/turn-stream.test.ts": 1517, + "packages/paperclip-runner/src/live/warm-attachment-readiness.test.ts": 100, + "packages/paperclip-runner/src/live/workspace-diff.test.ts": 100, + "packages/paperclip-runner/src/live/workspace-file-reference.test.ts": 100, + "packages/paperclip-runner/src/mock-core/capability-mock-control-plane-adapter.test.ts": 100, + "packages/paperclip-runner/src/mock-core/codex-runner.test.ts": 100, + "packages/paperclip-runner/src/mock-core/live-console-demo-server.test.ts": 4421, + "packages/paperclip-runner/src/mock-core/live-console-scripted-driver.test.ts": 155, + "packages/paperclip-runner/src/mock-core/local-runner.test.ts": 150, + "packages/paperclip-runner/src/mock-core/mock-control-plane-adapter.test.ts": 100, + "packages/paperclip-runner/src/mock-core/task-creation-backlog.test.ts": 100, + "packages/paperclip-runner/src/mock-core/task-reassignment.test.ts": 100, + "packages/paperclip-runner/src/native-session-runtime.test.ts": 3455, + "packages/paperclip-runner/src/protocol-actions/case-data.test.ts": 100, + "packages/paperclip-runner/src/protocol-actions/collaboration-contracts.test.ts": 113, + "packages/paperclip-runner/src/protocol-actions/core-contracts.test.ts": 138, + "packages/paperclip-runner/src/protocol/auth-credentials.test.ts": 100, + "packages/paperclip-runner/src/protocol/conformance-fixture.test.ts": 100, + "packages/paperclip-runner/src/protocol/permission-request.test.ts": 100, + "packages/paperclip-runner/src/protocol/provider-descriptor.test.ts": 100, + "packages/paperclip-runner/src/protocol/prp-v1-expressiveness-crosswalk.test.ts": 100, + "packages/paperclip-runner/src/protocol/replay-contract.test.ts": 100, + "packages/paperclip-runner/src/protocol/result-normalization.test.ts": 100, + "packages/paperclip-runner/src/protocol/runtime-activity.test.ts": 100, + "packages/paperclip-runner/src/protocol/semantic-tool-receipts.test.ts": 100, + "packages/paperclip-runner/src/protocol/workspace-diff-events.test.ts": 100, + "packages/paperclip-runner/src/provider-events.test.ts": 100, + "packages/paperclip-runner/src/react/public-contract.test.tsx": 100, + "packages/paperclip-runner/src/reducer/session-reducer.test.ts": 100, + "packages/paperclip-runner/src/scenarios/chat-session.test.ts": 643, + "packages/paperclip-runner/src/scenarios/scenario-explorer.test.ts": 855, + "packages/paperclip-runner/src/semantic-tools/paperclip-semantic-tools.test.ts": 838, + "packages/paperclip-runner/src/semantic-tools/runner-authorized-tools.test.ts": 100, + "packages/paperclip-runner/src/semantic-tools/semantic-tools.test.ts": 1386, + "packages/paperclip-runner/src/standalone/standalone-demo.test.ts": 100, + "packages/paperclip-runner/src/test-support/owned-node-runtime.test.ts": 407, + "packages/paperclip-runner/src/tool-file-changes.test.ts": 100, + "packages/paperclip-runner/src/tools/capability-semantic-tools.test.ts": 114, + "packages/paperclip-runner/src/tracer/conformance-runner.test.ts": 100, + "packages/paperclip-runner/src/tracer/replay.test.ts": 100 + } +} diff --git a/packages/paperclip-runner/scripts/vitest-package-lanes.mjs b/packages/paperclip-runner/scripts/vitest-package-lanes.mjs new file mode 100644 index 00000000000..43961bfa781 --- /dev/null +++ b/packages/paperclip-runner/scripts/vitest-package-lanes.mjs @@ -0,0 +1,63 @@ +// Duration-balanced partitions of the Runner package Vitest inventory. The PR +// workflow runs one partition per job (`--package-lane=K/N`) so the package +// tests no longer serialize behind two workers in a single job. Vitest owns +// the inventory: it is listed from vitest.config.ts, never re-globbed here. +import { spawnSync } from "node:child_process"; +import { createRequire } from "node:module"; +import { dirname, isAbsolute, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { loadShardDurations, partitionGeneralServerSuites } from "../../../scripts/general-server-shard.mjs"; + +export const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +export const laneDurationsPath = resolve(packageRoot, "scripts/vitest-lane-durations.json"); +// Manifest keys are repository-relative, matching the server shard manifests. +export const repoPrefix = "packages/paperclip-runner/"; + +const toPosix = (value) => value.replaceAll("\\", "/"); + +// Package-relative test files exactly as vitest.config.ts includes them. +export function listPackageVitestFiles({ env = process.env } = {}) { + const vitestBin = resolve(dirname(createRequire(resolve(packageRoot, "package.json")).resolve("vitest/package.json")), "vitest.mjs"); + // Listing must not register a CI profile invocation: it runs no tests. + const listEnv = { ...env, PAPERCLIP_CI_PROFILE: "" }; + // `--json` takes an optional output path: keep it last so it prints to stdout. + const result = spawnSync(process.execPath, [vitestBin, "list", "--filesOnly", "--json"], { + cwd: packageRoot, + encoding: "utf8", + env: listEnv, + maxBuffer: 16 * 1024 * 1024, + }); + if (result.status !== 0) { + throw new Error(`vitest list failed (${result.status ?? result.signal}): ${result.stderr || result.stdout}`); + } + const start = result.stdout.indexOf("["); + const entries = JSON.parse(result.stdout.slice(start)); + const files = [...new Set(entries.map((entry) => { + const file = typeof entry === "string" ? entry : entry.file; + return toPosix(isAbsolute(file) ? relative(packageRoot, file) : file); + }))].sort((a, b) => a.localeCompare(b)); + if (files.length === 0 || files.some((file) => file.startsWith("../"))) { + throw new Error(`vitest list returned an unexpected inventory: ${JSON.stringify(files.slice(0, 5))}`); + } + return files; +} + +// Vitest treats each CLI filter as a case-insensitive substring of the file's +// root-relative path. A file whose path ends another file's path (src/x.test.ts +// within devtools/browser/src/x.test.ts) therefore also selects the longer one. +export function filesMatchedByFilters(filters, inventory) { + const lowered = filters.map((filter) => filter.toLowerCase()); + return inventory.filter((file) => lowered.some((filter) => file.toLowerCase().includes(filter))); +} + +// Splits the package inventory into `laneCount` lanes, heaviest file first, +// with the deterministic partitioner the general server shards use. Every +// runner computes the identical partition from the same checkout, so the +// lanes are a complete, non-overlapping cover of the inventory. +export function planPackageLanes(files, laneCount, durations = loadShardDurations(laneDurationsPath)) { + if (!Number.isSafeInteger(laneCount) || laneCount < 1) throw new Error(`Invalid package lane count: ${laneCount}`); + return partitionGeneralServerSuites(files.map((file) => repoPrefix + file), laneCount, durations).map((lane) => ({ + files: lane.files.map((file) => file.slice(repoPrefix.length)), + estimatedMs: lane.totalWeight, + })); +} diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index 9804de802a9..5a083997bf5 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -439,28 +439,34 @@ test("shard flags are rejected for the native-runner group", () => { // The PR-side exclusion above is safe only while the wiring it assumes holds: // pr.yml (the caller whose name reusable pr-trusted.yml jobs see as -// GITHUB_WORKFLOW) is named PR, the sharded vitest lanes partition cleanly -// with exactly one final shard, and that lane's package script routes through -// the wrapper that runs the native-runner group. +// GITHUB_WORKFLOW) is named PR, the package vitest lanes cover every +// partition exactly once, exactly one lane runs the native-runner group, and +// those lanes' package script routes through the wrapper. test("the PR workflow wiring for the native-runner lane holds", () => { const prWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr.yml"), "utf8"); assert.match(prWorkflow, /^name: PR$/m, "renaming pr.yml silently moves the native-runner suite back into the uncached server shards"); const matrix = JSON.parse(readFileSync(path.join(repoRoot, ".github/ci-task-matrix.json"), "utf8")); - const lanes = matrix.verify_paperclip_runner - .map((entry) => /test:typescript:vitest --shard=(\d+)\/(\d+)/.exec(entry.command)) + const vitestLanes = matrix.verify_paperclip_runner.filter((entry) => entry.command.startsWith("test:typescript:vitest")); + for (const entry of vitestLanes) { + assert.match(entry.command, /^test:typescript:vitest (--package-lane=\d+\/\d+|--native-server)$/, entry.lane_label); + assert.equal(entry.task, "ci:runner-vitest", entry.lane_label); + } + const lanes = vitestLanes + .map((entry) => /--package-lane=(\d+)\/(\d+)$/.exec(entry.command)) .filter(Boolean).map((match) => [Number(match[1]), Number(match[2])]); - assert.ok(lanes.length > 0, "expected sharded test:typescript:vitest lanes in pr-trusted.yml"); - assert.equal(new Set(lanes.map(([, count]) => count)).size, 1, "vitest lanes must agree on the shard count"); - const shardCount = lanes[0][1]; + assert.ok(lanes.length > 0, "expected package-lane test:typescript:vitest lanes in the task matrix"); + assert.equal(new Set(lanes.map(([, count]) => count)).size, 1, "package lanes must agree on the lane count"); + const laneCount = lanes[0][1]; assert.deepEqual( lanes.map(([index]) => index).sort((left, right) => left - right), - Array.from({ length: shardCount }, (_, index) => index + 1), - "vitest lanes must cover every shard exactly once", + Array.from({ length: laneCount }, (_, index) => index + 1), + "package lanes must cover every partition exactly once", ); - assert.equal(lanes.filter(([index, count]) => index === count).length, 1, - "exactly one final vitest shard carries the native-runner group"); + const native = vitestLanes.filter((entry) => entry.command.endsWith("--native-server")); + assert.equal(native.length, 1, "exactly one vitest lane carries the native-runner group"); + assert.equal(native[0].release_crate, true, "the native-runner lane needs the release crate"); const runnerPackage = JSON.parse( readFileSync(path.join(repoRoot, "packages/paperclip-runner/package.json"), "utf8"), @@ -508,6 +514,71 @@ test("the PR lanes separate complete package and native workloads without changi assert.notEqual(malformed.status, 0, "a malformed shard flag must fail rather than guess a lane"); }); +const runnerRoot = path.join(repoRoot, "packages/paperclip-runner"); +const prepCommands = [ + { command: "pnpm", args: ["run", "ensure:eval-build-deps"], cwd: runnerRoot }, + { command: "pnpm", args: ["run", "build:rust"], cwd: runnerRoot }, +]; +const nativeCommand = { + command: "pnpm", + args: ["test:run:general", "--", "--group", "general-server-native-runner"], + cwd: repoRoot, +}; + +// Master's trusted workflow keeps sending the --shard commands until the +// package-lane matrix merges, so their exact plans must not move. +test("the master-pinned --shard lane commands plan exactly what they did before package lanes", () => { + assert.deepEqual(wrapperPlan(["--shard=1/2"], { GITHUB_WORKFLOW: "PR" }).plannedCommands, [ + ...prepCommands, + { command: "pnpm", args: ["exec", "vitest", "run"], cwd: runnerRoot }, + ]); + assert.deepEqual(wrapperPlan(["--shard=2/2"], { GITHUB_WORKFLOW: "PR" }).plannedCommands, [...prepCommands, nativeCommand]); + for (const plan of [wrapperPlan(["--shard=1/2"], { GITHUB_WORKFLOW: "PR" }), wrapperPlan(["--shard=2/2"], { GITHUB_WORKFLOW: "PR" })]) { + assert.equal(plan.packageLane, undefined); + } +}); + +test("--native-server runs only the native group under any workflow", () => { + for (const envOverrides of [{ GITHUB_WORKFLOW: "PR" }, {}]) { + const plan = wrapperPlan(["--native-server"], envOverrides); + assert.deepEqual(plan.plannedCommands, [...prepCommands, nativeCommand]); + assert.equal(plan.runPackageTests, false); + assert.equal(plan.runNativeRunnerGroup, true); + } +}); + +test("the package lane manifest uses the general server duration schema", () => { + const manifest = JSON.parse(readFileSync(path.join(runnerRoot, "scripts/vitest-lane-durations.json"), "utf8")); + const general = JSON.parse(readFileSync(durationsManifest, "utf8")); + // The refresh script writes exactly these keys; hand-added annotations such + // as the server manifest's splitEstimates are optional. + assert.deepEqual(Object.keys(manifest), ["$comment", "unit", "sources", "coverage", "durations"]); + assert.ok(Object.keys(manifest).every((key) => key in general)); + assert.equal(manifest.unit, "ms"); + assert.deepEqual(Object.keys(manifest.coverage), Object.keys(general.coverage)); + for (const source of manifest.sources) { + assert.deepEqual(Object.keys(source), ["run", "attempt", "sha", "url"]); + assert.match(source.sha, /^[a-f0-9]{40}$/); + } + const entries = Object.entries(manifest.durations); + assert.ok(entries.length > 100); + for (const [file, ms] of entries) { + assert.match(file, /^packages\/paperclip-runner\/.+\.test\.[cm]?[jt]sx?$/); + assert.ok(Number.isFinite(ms) && ms >= 0, file); + } +}); + +test("package lane arguments fail rather than guess a lane", () => { + for (const args of [["--package-lane=0/3"], ["--package-lane=4/3"], ["--package-lane=abc"], ["--package-lane", "1/3"], + ["--package-lane=1/3", "--package-lane=2/3"], ["--package-lane=1/3", "--shard=1/2"], ["--native-server", "--shard=2/2"], + ["--native-server", "--package-lane=1/3"], ["--native-server", "src/foo.test.ts"]]) { + const result = spawnSync(process.execPath, [laneWrapper, ...args, "--dry-run"], { + cwd: repoRoot, encoding: "utf8", env: workflowEnv({ GITHUB_WORKFLOW: "PR" }), + }); + assert.notEqual(result.status, 0, args.join(" ")); + } +}); + const lineShardFile = path.join(repoRoot, "server/src/__tests__/chat-channels.integration.test.ts"); const caseAt = (line, name) => ({ name, file: lineShardFile, projectName: "@paperclipai/server", location: { line, column: 3 } }); diff --git a/scripts/__tests__/runner-release-crate-cache.test.mjs b/scripts/__tests__/runner-release-crate-cache.test.mjs index e595bcd28a9..aa242ae6bca 100644 --- a/scripts/__tests__/runner-release-crate-cache.test.mjs +++ b/scripts/__tests__/runner-release-crate-cache.test.mjs @@ -361,7 +361,9 @@ test("the Verify Paperclip Runner job restores the crate only in the lanes that assert.deepEqual(lanes, [ ["static checks", false], ["rust", true], - ["vitest package", false], + ["vitest package (1/3)", false], + ["vitest package (2/3)", false], + ["vitest package (3/3)", false], ["vitest native server", true], ]); for (const stepName of [KEY_STEP, RESTORE_STEP]) { diff --git a/scripts/__tests__/runner-vitest-package-lanes.test.mjs b/scripts/__tests__/runner-vitest-package-lanes.test.mjs new file mode 100644 index 00000000000..9ea96ef6cdf --- /dev/null +++ b/scripts/__tests__/runner-vitest-package-lanes.test.mjs @@ -0,0 +1,74 @@ +// Runs real Vitest against the Runner package, so it needs installed workspace +// dependencies. The `policy` job runs run-vitest-stable-shard.test.mjs without +// them; this file runs in `test:release-registry` (Typecheck + Release Registry), +// which installs dependencies first. +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import test from "node:test"; + +import { defaultSuiteWeight, loadShardDurations } from "../general-server-shard.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +const runnerRoot = path.join(repoRoot, "packages/paperclip-runner"); +const laneWrapper = path.join(runnerRoot, "scripts/run-pr-vitest-lane.mjs"); +const prepCommands = [ + { command: "pnpm", args: ["run", "ensure:eval-build-deps"], cwd: runnerRoot }, + { command: "pnpm", args: ["run", "build:rust"], cwd: runnerRoot }, +]; + +function wrapperPlan(args, envOverrides = {}) { + const result = spawnSync(process.execPath, [laneWrapper, ...args, "--dry-run"], { + cwd: repoRoot, + encoding: "utf8", + env: { ...process.env, ...envOverrides }, + }); + assert.equal(result.status, 0, `expected wrapper dry run to succeed for ${args.join(" ")}: ${result.stderr}`); + return JSON.parse(result.stdout); +} + +// Asks Vitest itself which files the given CLI filters select, so both the +// inventory and the lanes' substring filters follow vitest.config.ts exactly. +function vitestListFiles(filters = []) { + const listed = spawnSync(process.execPath, [path.join(runnerRoot, "node_modules/vitest/vitest.mjs"), "list", "--filesOnly", ...filters], { + cwd: runnerRoot, encoding: "utf8", env: { ...process.env, PAPERCLIP_CI_PROFILE: "" }, maxBuffer: 16 * 1024 * 1024, + }); + assert.equal(listed.status, 0, listed.stderr); + return listed.stdout.split("\n").map((line) => line.trim().replaceAll("\\", "/")) + .filter((line) => /\.test\.[cm]?[jt]sx?$/.test(line)).sort(); +} + +test("package lanes form a complete, non-overlapping, balanced cover of the Vitest inventory", () => { + const inventory = vitestListFiles(); + assert.ok(inventory.length > 100, `expected the full Runner package inventory, got ${inventory.length}`); + + const durations = loadShardDurations(path.join(runnerRoot, "scripts/vitest-lane-durations.json")); + const weight = (file) => durations[`packages/paperclip-runner/${file}`] ?? defaultSuiteWeight(durations); + const laneCount = 3; + const lanes = Array.from({ length: laneCount }, (_, index) => { + const plan = wrapperPlan([`--package-lane=${index + 1}/${laneCount}`], { GITHUB_WORKFLOW: "PR" }); + assert.equal(plan.runNativeRunnerGroup, false); + assert.deepEqual(plan.plannedCommands.slice(0, 2), prepCommands, "every package lane prepares debug fixtures"); + assert.ok(!plan.plannedCommands.some((command) => command.args.includes("general-server-native-runner"))); + assert.equal(plan.packageLane.inventory, inventory.length); + const vitest = plan.plannedCommands.filter((command) => command.args[0] === "exec"); + assert.equal(vitest.length, 1); + assert.deepEqual(vitest[0].args, ["exec", "vitest", "run", ...plan.packageLane.files]); + assert.ok(plan.packageLane.files.length > 0, "an empty filter list would run the whole inventory"); + return plan.packageLane; + }); + + const assigned = lanes.flatMap((lane) => lane.files); + assert.equal(new Set(assigned).size, assigned.length, "a file is assigned to more than one lane"); + assert.deepEqual([...assigned].sort(), inventory, "the lanes must cover the inventory exactly"); + for (const lane of lanes) { + // Vitest filters are substring matches; no lane may also select another lane's files. + assert.deepEqual(vitestListFiles(lane.files), [...lane.files].sort(), `lane ${lane.index} filters`); + assert.equal(lane.estimatedMs, lane.files.reduce((total, file) => total + weight(file), 0)); + } + // Longest-processing-time bound: no lane exceeds the mean by more than the heaviest file. + const total = inventory.reduce((sum, file) => sum + weight(file), 0); + const heaviest = Math.max(...inventory.map(weight)); + for (const lane of lanes) assert.ok(lane.estimatedMs <= total / laneCount + heaviest, `lane ${lane.index} is unbalanced`); +});