diff --git a/packages/paperclip-runner/package.json b/packages/paperclip-runner/package.json index e7fbc955dc..fbf51aa98b 100644 --- a/packages/paperclip-runner/package.json +++ b/packages/paperclip-runner/package.json @@ -75,7 +75,7 @@ "test": "pnpm run test:typescript && pnpm run test:rust", "test:typescript": "pnpm run test:typescript:prep && vitest run", "test:typescript:prep": "pnpm run ensure:eval-build-deps && pnpm run build:rust && node --test test/protocol-contract.test.mjs test/acpx-sidecar-contract.test.mjs test/acpx-codex-package-contract.test.mjs scripts/aws-agentcore-provisioning.test.mjs scripts/build-verified-provider-entrypoints.test.mjs scripts/local-provider-smoke-environment.test.mjs scripts/materialize-opencode-binary.test.mjs", - "test:typescript:vitest": "pnpm run ensure:eval-build-deps && pnpm run build:rust && vitest run", + "test:typescript:vitest": "node ./scripts/run-pr-vitest-lane.mjs", "test:rust": "cargo test --release --manifest-path runner/Cargo.toml --locked --workspace", "test:codex": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --test codex_provider", "test:durable": "cargo test --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core durable::", diff --git a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs new file mode 100644 index 0000000000..e383bd9024 --- /dev/null +++ b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs @@ -0,0 +1,69 @@ +// Runs this package's vitest shard for a Verify Paperclip Runner lane, then — +// on the final shard of the PR workflow only — the server package's +// general-server-native-runner group. +// +// That server suite rebuilds the Runner release binaries with cargo in +// beforeAll. The PR workflow's plain General tests server shards carry no +// Rust cache, so hosting it there cold-compiled every third-party crate on +// each run (277s of a 291s shard vitest step, actions run 35246999382, +// 2026-09-17) and made that shard the slowest check of the whole run. The +// Verify Paperclip Runner lanes already restore the shared release-runner-v1 +// Rust cache read-only, which turns that build into an incremental rebuild, +// and the workflow files themselves list this lane's command as a package +// script — so the suite moves here without a workflow-file change. +// +// Contract, mirrored in scripts/run-vitest-stable.mjs (prWorkflowName) and +// pinned by scripts/__tests__/run-vitest-stable-shard.test.mjs: +// - Only the PR workflow (pr.yml, whose GITHUB_WORKFLOW the reusable +// pr-trusted.yml jobs inherit) excludes the suite from the server shards, +// and only there does this wrapper run it. Any other caller — local runs, +// release-verify.yml — keeps the suite in the server group, so a renamed +// workflow degrades to the slower covered path instead of losing coverage. +// - The suite runs on the lane whose --shard=N/M has N === M (or an unsharded +// invocation), so exactly one PR lane carries it. +import { spawnSync } from "node:child_process"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const workspaceRoot = resolve(packageRoot, "../.."); +const args = process.argv.slice(2).filter((value) => value !== "--dry-run"); +const dryRun = process.argv.includes("--dry-run"); + +const shardArg = args.find((value) => value.startsWith("--shard=")); +const shardMatch = shardArg ? /^--shard=(\d+)\/(\d+)$/.exec(shardArg) : null; +if (shardArg && !shardMatch) { + console.error(`[pr-vitest-lane] unrecognized shard argument: ${shardArg}`); + process.exit(1); +} +const isFinalShard = !shardArg || shardMatch[1] === shardMatch[2]; +const isPrWorkflow = process.env.GITHUB_WORKFLOW === "PR"; +const runNativeRunnerGroup = isPrWorkflow && isFinalShard; + +const plannedCommands = [ + { command: "pnpm", args: ["run", "ensure:eval-build-deps"], cwd: packageRoot }, + { command: "pnpm", args: ["run", "build:rust"], cwd: packageRoot }, + { command: "pnpm", args: ["exec", "vitest", "run", ...args], cwd: packageRoot }, + ...(runNativeRunnerGroup + ? [{ + command: "pnpm", + args: ["test:run:general", "--", "--group", "general-server-native-runner"], + cwd: workspaceRoot, + }] + : []), +]; + +if (dryRun) { + console.log(JSON.stringify({ isPrWorkflow, isFinalShard, runNativeRunnerGroup, plannedCommands }, null, 2)); + process.exit(0); +} + +for (const planned of plannedCommands) { + const result = spawnSync(planned.command, planned.args, { + cwd: planned.cwd, + stdio: "inherit", + }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index f320fa4859..ffe7fb2d0b 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -1,5 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { readFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; import test from "node:test"; @@ -21,16 +22,29 @@ const serializedDurationsManifest = path.join( "serialized-shard-durations.json", ); -function dryRun(args) { +// Membership of general-server-without-chat depends on GITHUB_WORKFLOW (see +// prWorkflowName in run-vitest-stable.mjs), so strip the ambient value and +// make every test pin the caller it mirrors explicitly — these tests +// themselves run inside a workflow on CI. +function workflowEnv(envOverrides = {}) { + const env = { ...process.env, ...envOverrides }; + if (!("GITHUB_WORKFLOW" in envOverrides)) { + delete env.GITHUB_WORKFLOW; + } + return env; +} + +function dryRun(args, envOverrides = {}) { const result = spawnSync(process.execPath, [script, ...args, "--dry-run"], { cwd: repoRoot, encoding: "utf8", + env: workflowEnv(envOverrides), }); return result; } -function dryRunJson(args) { - const result = dryRun(args); +function dryRunJson(args, envOverrides = {}) { + const result = dryRun(args, envOverrides); assert.equal(result.status, 0, `expected success for ${args.join(" ")}: ${result.stderr}`); return JSON.parse(result.stdout); } @@ -222,11 +236,15 @@ test("the real serialized shard partition is duration-balanced", () => { test("the real shard partition is duration-balanced", () => { // Mirrors the PR matrix: general-server-without-chat across SHARD_COUNT - // runners, with the chat suite carried by the dedicated general-chat lanes. + // runners, with the chat suite carried by the dedicated general-chat lanes + // and the native-runner suite by the Rust-cached PR vitest lane. const durations = loadShardDurations(durationsManifest); const fallback = defaultSuiteWeight(durations); const shards = Array.from({ length: SHARD_COUNT }, (_, index) => - dryRunJson(["--mode", "general", "--group", "general-server-without-chat", "--shard-index", String(index), "--shard-count", String(SHARD_COUNT)]), + dryRunJson( + ["--mode", "general", "--group", "general-server-without-chat", "--shard-index", String(index), "--shard-count", String(SHARD_COUNT)], + { GITHUB_WORKFLOW: "PR" }, + ), ); const totals = shards.map((shard) => @@ -235,10 +253,15 @@ test("the real shard partition is duration-balanced", () => { const maxTotal = Math.max(...totals); const minTotal = Math.min(...totals); // LPT keeps the spread within the heaviest single suite; use that as the - // bound. The chat suite runs in its own lanes, so exclude it here. + // bound. The chat and native-runner suites run in their own lanes, so + // exclude them here. const chat = "server/src/__tests__/chat-channels.integration.test.ts"; + const nativeRunner = + "server/src/services/native-runtime/native-codex-runner.integration.test.ts"; const heaviest = Math.max( - ...Object.entries(durations).filter(([file]) => file !== chat).map(([, ms]) => ms), + ...Object.entries(durations) + .filter(([file]) => file !== chat && file !== nativeRunner) + .map(([, ms]) => ms), ); assert.ok( maxTotal - minTotal <= heaviest, @@ -247,24 +270,131 @@ test("the real shard partition is duration-balanced", () => { }); -// 12 mirrors pr-trusted.yml, 10 mirrors release-verify.yml. -for (const withoutChatShardCount of [10, 12]) { - test(`${withoutChatShardCount} without-chat shards plus the dedicated chat file cover the original server group exactly`, () => { - const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"]); - const shards = Array.from({ length: withoutChatShardCount }, (_, index) => dryRunJson([ +const chatSuitePath = "server/src/__tests__/chat-channels.integration.test.ts"; +const nativeRunnerSuitePath = + "server/src/services/native-runtime/native-codex-runner.integration.test.ts"; + +// Mirrors pr-trusted.yml (12 shards, called by pr.yml so GITHUB_WORKFLOW is +// "PR"): the chat suite runs in its dedicated lanes and the cargo-dependent +// native-runner suite in the Rust-cached final Verify Paperclip Runner vitest +// shard, so together the three cover the full server group exactly. +test("12 PR without-chat shards plus the dedicated chat and native-runner lanes cover the original server group exactly", () => { + const prEnv = { GITHUB_WORKFLOW: "PR" }; + const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], prEnv); + const shards = Array.from({ length: 12 }, (_, index) => dryRunJson([ + "--mode", "general", "--group", "general-server-without-chat", + "--shard-index", String(index), "--shard-count", "12", + ], prEnv)); + const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites); + assert.ok(!files.includes(chatSuitePath)); + assert.ok(!files.includes(nativeRunnerSuitePath)); + assert.deepEqual([...files, chatSuitePath, nativeRunnerSuitePath].sort(), full.selectedGeneralServerSuites.sort()); + assert.equal(new Set(files).size, files.length); + const defaultRun = dryRunJson([], prEnv); + assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount); +}); + +// Mirrors release-verify.yml (10 shards, called by the Release and Cloud +// readiness workflows) and local runs: no Rust-cached vitest lane exists +// there, so the native-runner suite must stay in the server shards. +for (const [caller, envOverrides] of [["Release", { GITHUB_WORKFLOW: "Release" }], ["no ambient workflow", {}]]) { + test(`10 without-chat shards under ${caller} keep the native-runner suite and cover the server group with chat alone`, () => { + const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], envOverrides); + const shards = Array.from({ length: 10 }, (_, index) => dryRunJson([ "--mode", "general", "--group", "general-server-without-chat", - "--shard-index", String(index), "--shard-count", String(withoutChatShardCount), - ])); + "--shard-index", String(index), "--shard-count", "10", + ], envOverrides)); const files = shards.flatMap((shard) => shard.selectedGeneralServerSuites); - const chat = "server/src/__tests__/chat-channels.integration.test.ts"; - assert.ok(!files.includes(chat)); - assert.deepEqual([...files, chat].sort(), full.selectedGeneralServerSuites.sort()); + assert.ok(!files.includes(chatSuitePath)); + assert.ok(files.includes(nativeRunnerSuitePath)); + assert.deepEqual([...files, chatSuitePath].sort(), full.selectedGeneralServerSuites.sort()); assert.equal(new Set(files).size, files.length); - const defaultRun = dryRunJson([]); - assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount); }); } +test("the native-runner lane runs exactly the cargo-dependent vertical-slice suite", () => { + const lane = dryRunJson(["--mode", "general", "--group", "general-server-native-runner"]); + assert.deepEqual(lane.selectedGeneralServerSuites, [ + "server/src/services/native-runtime/native-codex-runner.integration.test.ts", + ]); +}); + +test("shard flags are rejected for the native-runner group", () => { + const result = dryRun(["--mode", "general", "--group", "general-server-native-runner", "--shard-index", "0", "--shard-count", "2"]); + assert.notEqual(result.status, 0, "the native-runner lane is a single suite and must not accept shard flags"); +}); + +// The PR-side exclusion above is safe only while the wiring it assumes holds: +// pr.yml (the caller whose name reusable pr-trusted.yml jobs see as +// GITHUB_WORKFLOW) is named PR, the sharded vitest lanes partition cleanly +// with exactly one final shard, and that lane's package script routes through +// the wrapper that runs the native-runner group. +test("the PR workflow wiring for the native-runner lane holds", () => { + const prWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr.yml"), "utf8"); + assert.match(prWorkflow, /^name: PR$/m, + "renaming pr.yml silently moves the native-runner suite back into the uncached server shards"); + + const trustedWorkflow = readFileSync(path.join(repoRoot, ".github/workflows/pr-trusted.yml"), "utf8"); + const lanes = [...trustedWorkflow.matchAll(/command: test:typescript:vitest --shard=(\d+)\/(\d+)/g)] + .map((match) => [Number(match[1]), Number(match[2])]); + assert.ok(lanes.length > 0, "expected sharded test:typescript:vitest lanes in pr-trusted.yml"); + assert.equal(new Set(lanes.map(([, count]) => count)).size, 1, "vitest lanes must agree on the shard count"); + const shardCount = lanes[0][1]; + assert.deepEqual( + lanes.map(([index]) => index).sort((left, right) => left - right), + Array.from({ length: shardCount }, (_, index) => index + 1), + "vitest lanes must cover every shard exactly once", + ); + assert.equal(lanes.filter(([index, count]) => index === count).length, 1, + "exactly one final vitest shard carries the native-runner group"); + + const runnerPackage = JSON.parse( + readFileSync(path.join(repoRoot, "packages/paperclip-runner/package.json"), "utf8"), + ); + assert.equal(runnerPackage.scripts["test:typescript:vitest"], "node ./scripts/run-pr-vitest-lane.mjs"); +}); + +const laneWrapper = path.join(repoRoot, "packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs"); + +function wrapperPlan(args, envOverrides = {}) { + const result = spawnSync(process.execPath, [laneWrapper, ...args, "--dry-run"], { + cwd: repoRoot, + encoding: "utf8", + env: workflowEnv(envOverrides), + }); + assert.equal(result.status, 0, `expected wrapper dry run to succeed for ${args.join(" ")}: ${result.stderr}`); + return JSON.parse(result.stdout); +} + +test("the PR vitest lane wrapper runs the native-runner group exactly on the final PR shard", () => { + for (const [args, envOverrides, expected] of [ + [["--shard=1/2"], { GITHUB_WORKFLOW: "PR" }, false], + [["--shard=2/2"], { GITHUB_WORKFLOW: "PR" }, true], + [[], { GITHUB_WORKFLOW: "PR" }, true], + [["--shard=2/2"], { GITHUB_WORKFLOW: "Release" }, false], + [["--shard=2/2"], {}, false], + [[], {}, false], + ]) { + const plan = wrapperPlan(args, envOverrides); + assert.equal(plan.runNativeRunnerGroup, expected, + `args ${JSON.stringify(args)} env ${JSON.stringify(envOverrides)}`); + const commands = plan.plannedCommands.map((planned) => planned.args.join(" ")); + assert.ok(commands.some((command) => command.startsWith(`exec vitest run${args.length ? ` ${args.join(" ")}` : ""}`)), + "the wrapper must pass shard flags through to vitest"); + assert.equal( + commands.some((command) => command.includes("--group general-server-native-runner")), + expected, + ); + } + + const malformed = spawnSync(process.execPath, [laneWrapper, "--shard=nonsense", "--dry-run"], { + cwd: repoRoot, + encoding: "utf8", + env: workflowEnv({ GITHUB_WORKFLOW: "PR" }), + }); + assert.notEqual(malformed.status, 0, "a malformed shard flag must fail rather than guess a lane"); +}); + const lineShardFile = path.join(repoRoot, "server/src/__tests__/chat-channels.integration.test.ts"); const caseAt = (line, name) => ({ name, file: lineShardFile, projectName: "@paperclipai/server", location: { line, column: 3 } }); diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 1e7a91df35..d98a814a12 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -70,13 +70,39 @@ const allModeName = "all"; const generalServerGroupName = "general-server"; const generalServerWithoutChatGroupName = "general-server-without-chat"; const generalChatGroupName = "general-chat"; +const generalServerNativeRunnerGroupName = "general-server-native-runner"; const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts"; +// This suite rebuilds the Runner release binaries with cargo in beforeAll. +// Inside the PR workflow's plain server shards, which carry no Rust cache, +// that build was a ~4m30s cold compile of every third-party crate on each run +// (277s of a 291s shard vitest step, actions run 35246999382, 2026-09-17). +const nativeRunnerSuite = + "server/src/services/native-runtime/native-codex-runner.integration.test.ts"; +// In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable +// workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip +// Runner vitest shard runs the native-runner group instead, because those +// lanes restore the shared release-runner-v1 Rust cache (see +// packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs). Every other +// caller — local runs, release-verify.yml under the Release and Cloud +// readiness workflows — keeps the suite in the server shards, so a renamed or +// unknown workflow degrades to today's slower-but-covered behavior rather +// than dropping the suite. +const prWorkflowName = "PR"; +const nativeRunnerSuiteRunsInRustCachedLane = process.env.GITHUB_WORKFLOW === prWorkflowName; +const withoutChatExcludedSuites = nativeRunnerSuiteRunsInRustCachedLane + ? [chatSuite, nativeRunnerSuite] + : [chatSuite]; const generalWorkspacesAGroupName = "general-workspaces-a"; const generalWorkspacesBGroupName = "general-workspaces-b"; const generalWorkspacesAProjects = ["@paperclipai/ui", "paperclipai"]; const generalWorkspacesBProjects = nonServerProjects.filter((project) => !generalWorkspacesAProjects.includes(project)); const generalGroupNames = [generalServerGroupName, generalWorkspacesAGroupName, generalWorkspacesBGroupName]; -const allowedGeneralGroupNames = [...generalGroupNames, generalServerWithoutChatGroupName, generalChatGroupName]; +const allowedGeneralGroupNames = [ + ...generalGroupNames, + generalServerWithoutChatGroupName, + generalChatGroupName, + generalServerNativeRunnerGroupName, +]; const serializedServerVitestArgs = [ "--no-file-parallelism", "--maxWorkers=1", @@ -355,9 +381,21 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount = "chat integration test shard", { index: shardIndex ?? 0, count: shardCount ?? 1 }); return; } + if (groupName === generalServerNativeRunnerGroupName) { + runVitest( + ["--project", "@paperclipai/server", ...serializedServerVitestArgs, nativeRunnerSuite], + "native runner vertical-slice suite", + ); + return; + } if (groupName === generalServerGroupName || groupName === generalServerWithoutChatGroupName) { + // In the PR workflow the without-chat group also leaves the native-runner + // suite to the Rust-cached vitest lane; the full general-server group + // (local runs) keeps both. const withoutChat = groupName === generalServerWithoutChatGroupName; - const files = withoutChat ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles; + const files = withoutChat + ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file)) + : generalServerTestFiles; if (shardCount !== null && shardCount > 1) { const shardFiles = selectGeneralServerShard( files, @@ -385,7 +423,11 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount = } const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]); - if (withoutChat) excludeRouteArgs.push("--exclude", "src/__tests__/chat-channels.integration.test.ts"); + if (withoutChat) { + for (const suite of withoutChatExcludedSuites) { + excludeRouteArgs.push("--exclude", suite.replace(/^server\//, "")); + } + } runVitest( [ "--project", @@ -474,16 +516,20 @@ if (options.dryRun) { selectedSerializedSuites: serializedSuites.map((routeTest) => routeTest.repoPath), generalServerSuiteCount: generalServerTestFiles.length, selectedGeneralServerSuites: - options.mode === generalModeName && - [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) && - options.shardCount !== null - ? selectGeneralServerShard( - options.group === generalServerWithoutChatGroupName ? generalServerTestFiles.filter((file) => file !== chatSuite) : generalServerTestFiles, - options.shardIndex, - options.shardCount, - generalServerShardDurations, - ) - : null, + options.mode === generalModeName && options.group === generalServerNativeRunnerGroupName + ? [nativeRunnerSuite] + : options.mode === generalModeName && + [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) && + options.shardCount !== null + ? selectGeneralServerShard( + options.group === generalServerWithoutChatGroupName + ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file)) + : generalServerTestFiles, + options.shardIndex, + options.shardCount, + generalServerShardDurations, + ) + : null, workspaceProjects: options.group === generalWorkspacesAGroupName ? generalWorkspacesAProjects