diff --git a/.github/scripts/tests/release-runner-cache.test.mjs b/.github/scripts/tests/release-runner-cache.test.mjs index 96bba1f325..b6f478122e 100644 --- a/.github/scripts/tests/release-runner-cache.test.mjs +++ b/.github/scripts/tests/release-runner-cache.test.mjs @@ -38,14 +38,19 @@ test("parallel lanes cover check:all exactly once and never bypass verification" const scripts = JSON.parse(readFileSync(new URL("../../../packages/paperclip-runner/package.json", import.meta.url))).scripts; const checks = [...runner.matchAll(/^ checks: (.+)$/gm)].flatMap(([, value]) => value.split(" ")); assert.deepEqual(checks, scripts["check:all"].split(" && ").map((command) => command.replace(/^pnpm run /, ""))); - assert.deepEqual([...runner.matchAll(/^ - lane: (.+)$/gm)].map(([, value]) => value), ["protocol", "rust"]); + assert.deepEqual([...runner.matchAll(/^ - lane: (.+)$/gm)].map(([, value]) => value), ["protocol", "rust", "server-integration"]); assert.match(runner, /fail-fast: false/); assert.doesNotMatch(runner, /max-parallel: 1|^ needs:|continue-on-error:/m); const verify = runner.split(" - name: Verify Paperclip Runner\n")[1].split(" - name: Warm debug")[0]; assert.match(verify, /RUNNER_CHECKS: \$\{\{ matrix.checks \}\}/); assert.match(verify, /set -euo pipefail/); assert.match(verify, /for check in \$RUNNER_CHECKS; do\s+pnpm --filter @paperclipai\/paperclip-runner "\$check"\s+done/); - assert.doesNotMatch(verify, /if:|cache-hit/); + const verifyIf = verify.match(/^\s*if: \$\{\{ (.+) \}\}$/m)?.[1]; + assert.equal(verifyIf, "matrix.lane != 'server-integration'"); + for (const lane of ["protocol", "rust", "server-integration"]) { + assert.equal(runInNewContext(verifyIf, { matrix: { lane } }), lane !== "server-integration"); + } + assert.doesNotMatch(verify, /cache-hit/); assert.doesNotMatch(runner, /id-token: write|packages: write|secrets: inherit/); }); @@ -57,7 +62,7 @@ test("only the trusted Rust lane writes, and warms both build profiles before sa assert.match(warm, /run: pnpm --filter @paperclipai\/paperclip-runner build:rust/); const sha = "a".repeat(40); const base = { repository: "paperclipai/paperclip", event_name: "push", ref: "refs/heads/master", sha }; - for (const lane of ["protocol", "rust"]) { + for (const lane of ["protocol", "rust", "server-integration"]) { for (const [overrides, ref, trusted] of [ [{}, sha, true], [{ event_name: "pull_request", ref: "refs/pull/1/merge" }, sha, false], diff --git a/.github/workflows/release-verify.yml b/.github/workflows/release-verify.yml index 76e5cd28bb..492b7e62b4 100644 --- a/.github/workflows/release-verify.yml +++ b/.github/workflows/release-verify.yml @@ -107,45 +107,46 @@ jobs: matrix: include: # Split the long chat file by collected test locations, and balance - # the remaining server files across ten runners. Normal PR/local - # invocations retain their complete general-server group. - - group: general-server-without-chat + # the remaining server files across ten runners. Rust-backed server + # suites run in the cached Runner lane below. Local invocations retain + # their complete general-server group. + - group: general-server-without-chat-or-native-runner group_label: server (1/10) shard_index: 0 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (2/10) shard_index: 1 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (3/10) shard_index: 2 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (4/10) shard_index: 3 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (5/10) shard_index: 4 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (6/10) shard_index: 5 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (7/10) shard_index: 6 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (8/10) shard_index: 7 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (9/10) shard_index: 8 shard_count: 10 - - group: general-server-without-chat + - group: general-server-without-chat-or-native-runner group_label: server (10/10) shard_index: 9 shard_count: 10 @@ -297,6 +298,7 @@ jobs: checks: check:eval-kernel check:protocol - lane: rust checks: check:runner check:api-authority + - lane: server-integration steps: - name: Checkout repository @@ -373,14 +375,28 @@ jobs: # dependencies; never restore installed executables from cargo/bin. cache-workspace-crates: false cache-bin: false - # Both lanes restore the existing dependency cache. Only the Rust + # All lanes restore the existing dependency cache. Only the Rust # lane saves it, after warming both release and debug dependencies. save-if: ${{ matrix.lane == 'rust' && github.repository == 'paperclipai/paperclip' && github.event_name == 'push' && github.ref == 'refs/heads/master' && inputs.ref == github.sha }} - name: Install dependencies run: pnpm install --no-frozen-lockfile + - name: Build native server test binaries + if: ${{ matrix.lane == 'server-integration' }} + timeout-minutes: 10 + working-directory: packages/paperclip-runner + run: | + set -euo pipefail + pnpm build:rust + cargo build --release --manifest-path runner/Cargo.toml --locked -p paperclip-runner-core --bin paperclip-runnerd --bin fake-codex-app-server + + - name: Run native server integration suites + if: ${{ matrix.lane == 'server-integration' }} + run: pnpm test:run:general -- --group general-server-native-runner + - name: Verify Paperclip Runner + if: ${{ matrix.lane != 'server-integration' }} env: RUNNER_CHECKS: ${{ matrix.checks }} run: | diff --git a/doc/cloud-build-readiness.md b/doc/cloud-build-readiness.md index df143f11ec..18611b833b 100644 --- a/doc/cloud-build-readiness.md +++ b/doc/cloud-build-readiness.md @@ -33,9 +33,14 @@ test jobs on standard runners. Measure queue time to assess the timing gain. Release verification spreads the general server suites across ten standard hosted runners, with the long chat suite split separately across three jobs. Each server -job still runs one test worker. The partition covers every suite exactly once; -normal PR and local test groups keep their existing shape. More jobs increase -concurrent runner demand, so compare queue time as well as test duration. +job still runs one test worker. The three Rust-backed server suites run in a +separate Runner lane with the shared dependency cache. That lane builds both +debug and release test binaries in a visible, bounded step before Vitest starts; +the suites still check Cargo freshness and run against the current source. A +cache miss builds from source. Native test failures block source verification. +The partition covers every suite exactly once; normal PR and local test groups +keep their existing shape. More jobs increase concurrent runner demand, so compare +queue time as well as test duration. All release verification installs, including the Runner scorer and chaos evals, allow pnpm to refresh an outdated lockfile. Contributor PRs leave lockfile updates diff --git a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs index e383bd9024..cc49d2a94e 100644 --- a/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs +++ b/packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs @@ -15,10 +15,10 @@ // Contract, mirrored in scripts/run-vitest-stable.mjs (prWorkflowName) and // pinned by scripts/__tests__/run-vitest-stable-shard.test.mjs: // - Only the PR workflow (pr.yml, whose GITHUB_WORKFLOW the reusable -// pr-trusted.yml jobs inherit) excludes the suite from the server shards, -// and only there does this wrapper run it. Any other caller — local runs, -// release-verify.yml — keeps the suite in the server group, so a renamed -// workflow degrades to the slower covered path instead of losing coverage. +// pr-trusted.yml jobs inherit) excludes these suites from the existing +// without-chat group, and only there does this wrapper run them. Other +// callers keep that group complete. Release verification instead selects +// an explicit partition and its own required cached native lane. // - The suite runs on the lane whose --shard=N/M has N === M (or an unsharded // invocation), so exactly one PR lane carries it. import { spawnSync } from "node:child_process"; diff --git a/scripts/__tests__/release-verify-workflow.test.mjs b/scripts/__tests__/release-verify-workflow.test.mjs index 293280c2d1..98717aa8af 100644 --- a/scripts/__tests__/release-verify-workflow.test.mjs +++ b/scripts/__tests__/release-verify-workflow.test.mjs @@ -244,10 +244,10 @@ test("release verify workflow covers the same split test surface as stable PR ve assert.match(buildJob, /persist-credentials: false/); assert.doesNotMatch(buildJob, /cache: pnpm/); - for (const group of ["general-server-without-chat", "general-chat", "general-workspaces-a", "general-workspaces-b"]) { + for (const group of ["general-server-without-chat-or-native-runner", "general-chat", "general-workspaces-a", "general-workspaces-b"]) { assert.match(verifyWorkflow, new RegExp(`group: ${group}`)); } - for (const [group, count] of [["general-server-without-chat", 10], ["general-chat", 3]]) { + for (const [group, count] of [["general-server-without-chat-or-native-runner", 10], ["general-chat", 3]]) { const rows = [...verifyWorkflow.matchAll(new RegExp(`group: ${group}\\n\\s+group_label: [^\\n]+\\n\\s+shard_index: (\\d+)\\n\\s+shard_count: (\\d+)`, "g"))]; assert.deepEqual(rows.map((row) => [Number(row[1]), Number(row[2])]), Array.from({ length: count }, (_, index) => [index, count])); @@ -271,6 +271,30 @@ test("release verify workflow covers the same split test surface as stable PR ve assert.match(verifyWorkflow, /pnpm test:run:serialized -- --shard-index/); }); +test("release verification builds native test binaries in a required Rust-cached lane", () => { + const workflow = readWorkflow("release-verify.yml"); + const runnerJob = workflow.match(/ verify_paperclip_runner:\n[\s\S]*?(?=\n [A-Za-z0-9_-]+:|$)/)?.[0] ?? ""; + assert.equal((runnerJob.match(/- lane: server-integration/g) ?? []).length, 1); + assert.doesNotMatch(runnerJob, /continue-on-error/); + assert.match(runnerJob, /timeout-minutes: 20/); + assert.match(runnerJob, /shared-key: release-runner-v2/); + assert.match(runnerJob, /cache-workspace-crates: false/); + assert.match(runnerJob, /cache-bin: false/); + assert.match(runnerJob, /save-if: \$\{\{ matrix\.lane == 'rust'/); + assert.match(runnerJob, /Build native server test binaries\n\s+if: \$\{\{ matrix\.lane == 'server-integration' \}\}\n\s+timeout-minutes: 10/); + assert.match(runnerJob, /pnpm build:rust\n\s+cargo build --release --manifest-path runner\/Cargo\.toml --locked -p paperclip-runner-core --bin paperclip-runnerd --bin fake-codex-app-server/); + assert.match(runnerJob, /Run native server integration suites\n\s+if: \$\{\{ matrix\.lane == 'server-integration' \}\}\n\s+run: pnpm test:run:general -- --group general-server-native-runner/); + assert.ok(runnerJob.indexOf("Cache Runner Rust dependencies") < runnerJob.indexOf("Build native server test binaries")); + assert.ok(runnerJob.indexOf("Build native server test binaries") < runnerJob.indexOf("Run native server integration suites")); + + // The reusable workflow result includes every matrix child. Its caller must + // await that result without an override that can certify a failed native lane. + const cloud = readWorkflow("cloud-readiness.yml"); + assert.match(cloud, /verify:[\s\S]*?uses: \.\/\.github\/workflows\/release-verify\.yml/); + assert.match(cloud, /source_verified:[\s\S]*?needs: \[verify\]/); + assert.doesNotMatch(cloud, /continue-on-error|always\(\)/); +}); + test("Runner eval workflows pin actions and gate paid live execution", () => { const actionPinWorkflows = [ readWorkflow("release-verify.yml"), diff --git a/scripts/__tests__/run-vitest-stable-shard.test.mjs b/scripts/__tests__/run-vitest-stable-shard.test.mjs index d4fcb7a0d5..a044c67c15 100644 --- a/scripts/__tests__/run-vitest-stable-shard.test.mjs +++ b/scripts/__tests__/run-vitest-stable-shard.test.mjs @@ -319,9 +319,8 @@ test("12 PR without-chat shards plus the dedicated chat and native-runner lanes assert.ok(defaultRun.generalServerSuiteCount === full.generalServerSuiteCount); }); -// Mirrors release-verify.yml (10 shards, called by the Release and Cloud -// readiness workflows) and local runs: no Rust-cached vitest lane exists -// there, so the native-runner suite must stay in the server shards. +// The legacy group and local default must remain complete when a caller has +// not selected a separate native lane. for (const [caller, envOverrides] of [["Release", { GITHUB_WORKFLOW: "Release" }], ["no ambient workflow", {}]]) { test(`10 without-chat shards under ${caller} keep the native-runner suite and cover the server group with chat alone`, () => { const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], envOverrides); @@ -339,6 +338,23 @@ for (const [caller, envOverrides] of [["Release", { GITHUB_WORKFLOW: "Release" } }); } +// Release verification selects its dedicated lane explicitly. Do not infer it +// from the caller name: previews and future workflow callers need the same cover. +for (const caller of ["Release", "Cloud readiness", "another caller"]) { + test(`release server, chat, and native lanes cover every server suite once under ${caller}`, () => { + const env = { GITHUB_WORKFLOW: caller }; + const full = dryRunJson(["--mode", "general", "--group", "general-server", "--shard-index", "0", "--shard-count", "1"], env); + const files = Array.from({ length: 10 }, (_, index) => dryRunJson([ + "--mode", "general", "--group", "general-server-without-chat-or-native-runner", + "--shard-index", String(index), "--shard-count", "10", + ], env)).flatMap((shard) => shard.selectedGeneralServerSuites); + const native = dryRunJson(["--mode", "general", "--group", "general-server-native-runner"], env); + const combined = [...files, chatSuitePath, ...native.selectedGeneralServerSuites]; + assert.equal(new Set(combined).size, combined.length, "lanes must not overlap"); + assert.deepEqual(combined.sort(), full.selectedGeneralServerSuites.sort()); + }); +} + test("the native-runner lane runs exactly the cargo-dependent suites", () => { const lane = dryRunJson(["--mode", "general", "--group", "general-server-native-runner"]); assert.deepEqual(lane.selectedGeneralServerSuites, [ diff --git a/scripts/run-vitest-stable.mjs b/scripts/run-vitest-stable.mjs index 1d98b848af..dd00f145cd 100644 --- a/scripts/run-vitest-stable.mjs +++ b/scripts/run-vitest-stable.mjs @@ -76,6 +76,7 @@ const generalModeName = "general"; const allModeName = "all"; const generalServerGroupName = "general-server"; const generalServerWithoutChatGroupName = "general-server-without-chat"; +const generalServerWithoutChatOrNativeRunnerGroupName = "general-server-without-chat-or-native-runner"; const generalChatGroupName = "general-chat"; const generalServerNativeRunnerGroupName = "general-server-native-runner"; const chatSuite = "server/src/__tests__/chat-channels.integration.test.ts"; @@ -92,10 +93,9 @@ const nativeRunnerSuites = [ // In the PR workflow (pr.yml, the caller of pr-trusted.yml — reusable // workflows inherit the caller's GITHUB_WORKFLOW), the last Verify Paperclip // Runner vitest shard runs the native-runner group instead, because those -// lanes restore the shared release-runner-v1 Rust cache (see +// lanes restore the shared release-runner-v2 Rust cache (see // packages/paperclip-runner/scripts/run-pr-vitest-lane.mjs). Every other -// caller — local runs, release-verify.yml under the Release and Cloud -// readiness workflows — keeps the suite in the server shards, so a renamed or +// caller of the same group keeps the suite in the server shards, so a renamed or // unknown workflow degrades to today's slower-but-covered behavior rather // than dropping the suite. const prWorkflowName = "PR"; @@ -103,6 +103,18 @@ const nativeRunnerSuiteRunsInRustCachedLane = process.env.GITHUB_WORKFLOW === pr const withoutChatExcludedSuites = nativeRunnerSuiteRunsInRustCachedLane ? [chatSuite, ...nativeRunnerSuites] : [chatSuite]; +// Release verification selects this explicit group and runs the omitted suites +// in its cached Runner job. Local callers keep the complete default group. +const generalServerGroups = [ + generalServerGroupName, + generalServerWithoutChatGroupName, + generalServerWithoutChatOrNativeRunnerGroupName, +]; +function excludedServerSuites(groupName) { + if (groupName === generalServerWithoutChatOrNativeRunnerGroupName) return [chatSuite, ...nativeRunnerSuites]; + if (groupName === generalServerWithoutChatGroupName) return withoutChatExcludedSuites; + return []; +} const generalWorkspacesAGroupName = "general-workspaces-a"; const generalWorkspacesBGroupName = "general-workspaces-b"; const generalWorkspacesAProjects = ["@paperclipai/ui", "paperclipai"]; @@ -111,6 +123,7 @@ const generalGroupNames = [generalServerGroupName, generalWorkspacesAGroupName, const allowedGeneralGroupNames = [ ...generalGroupNames, generalServerWithoutChatGroupName, + generalServerWithoutChatOrNativeRunnerGroupName, generalChatGroupName, generalServerNativeRunnerGroupName, ]; @@ -290,7 +303,7 @@ function parseCliOptions(argv) { const shardAllowed = mode === serializedModeName || (mode === generalModeName && - ([generalServerGroupName, generalServerWithoutChatGroupName, generalChatGroupName, generalWorkspacesAGroupName].includes(group))); + ([...generalServerGroups, generalChatGroupName, generalWorkspacesAGroupName].includes(group))); if (!shardAllowed && shardIndex !== null) { fail( "--shard-index/--shard-count are only valid with serialized mode or a shardable general server/chat/workspaces-a group.", @@ -448,14 +461,9 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount = ); return; } - if (groupName === generalServerGroupName || groupName === generalServerWithoutChatGroupName) { - // In the PR workflow the without-chat group also leaves the native-runner - // suite to the Rust-cached vitest lane; the full general-server group - // (local runs) keeps both. - const withoutChat = groupName === generalServerWithoutChatGroupName; - const files = withoutChat - ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file)) - : generalServerTestFiles; + if (generalServerGroups.includes(groupName)) { + const excludedSuites = excludedServerSuites(groupName); + const files = generalServerTestFiles.filter((file) => !excludedSuites.includes(file)); if (shardCount !== null && shardCount > 1) { const shardFiles = selectGeneralServerShard( files, @@ -483,10 +491,8 @@ function runGeneralGroup(routeTests, groupName, shardIndex = null, shardCount = } const excludeRouteArgs = routeTests.flatMap((file) => ["--exclude", file.serverPath]); - if (withoutChat) { - for (const suite of withoutChatExcludedSuites) { - excludeRouteArgs.push("--exclude", suite.replace(/^server\//, "")); - } + for (const suite of excludedSuites) { + excludeRouteArgs.push("--exclude", suite.replace(/^server\//, "")); } runVitest( [ @@ -584,12 +590,10 @@ if (options.dryRun) { options.mode === generalModeName && options.group === generalServerNativeRunnerGroupName ? nativeRunnerSuites : options.mode === generalModeName && - [generalServerGroupName, generalServerWithoutChatGroupName].includes(options.group) && + generalServerGroups.includes(options.group) && options.shardCount !== null ? selectGeneralServerShard( - options.group === generalServerWithoutChatGroupName - ? generalServerTestFiles.filter((file) => !withoutChatExcludedSuites.includes(file)) - : generalServerTestFiles, + generalServerTestFiles.filter((file) => !excludedServerSuites(options.group).includes(file)), options.shardIndex, options.shardCount, generalServerShardDurations, diff --git a/server/src/__tests__/dot-runner.test.ts b/server/src/__tests__/dot-runner.test.ts index c6b586db09..fe7caa79c8 100644 --- a/server/src/__tests__/dot-runner.test.ts +++ b/server/src/__tests__/dot-runner.test.ts @@ -37,7 +37,7 @@ describe("durable Dot Runner integration", () => { let root: string; beforeAll(async () => { const runnerRoot = fileURLToPath(new URL("../../../packages/paperclip-runner/", import.meta.url)); - execFileSync("cargo", ["build", "--release", "--locked", "--manifest-path", join(runnerRoot, "runner/Cargo.toml"), "-p", "paperclip-runner-core", "--bin", "paperclip-runnerd"], { cwd: runnerRoot, stdio: "pipe", timeout: 300000 }); + execFileSync("cargo", ["build", "--release", "--locked", "--manifest-path", join(runnerRoot, "runner/Cargo.toml"), "-p", "paperclip-runner-core", "--bin", "paperclip-runnerd"], { cwd: runnerRoot, stdio: "inherit", timeout: 300000 }); temporary = await startEmbeddedPostgresTestDatabase("paperclip-dot-runner-"); db = createDb(temporary.connectionString); root = await mkdtemp(join(tmpdir(), "paperclip-dot-state-")); @@ -47,7 +47,17 @@ describe("durable Dot Runner integration", () => { vi.stubEnv("PAPERCLIP_SECRETS_MASTER_KEY", randomBytes(32).toString("base64")); await instanceSettingsService(db).updateExperimental({ enablePublicMcp: true, enableOpenAiDot: true, enableNativeRunner: false }); }, 360000); - afterAll(async () => { await temporary?.cleanup(); await rm(root, { recursive: true, force: true }); vi.unstubAllEnvs(); }); + afterAll(async () => { + try { + await temporary?.cleanup(); + } finally { + try { + if (root) await rm(root, { recursive: true, force: true }); + } finally { + vi.unstubAllEnvs(); + } + } + }); it("requires each persisted prerequisite for pairing and new work without relying on the retired environment flag", async () => { const settings = instanceSettingsService(db);