diff --git a/.github/ci-task-matrix.json b/.github/ci-task-matrix.json index 5e9cc021b39..53d1909b158 100644 --- a/.github/ci-task-matrix.json +++ b/.github/ci-task-matrix.json @@ -284,5 +284,10 @@ "shard_label": "8/8", "task": "ci:e2e" } + ], + "verify_runner_e2e_support": [ + { + "task": "ci:runner-e2e-support" + } ] } diff --git a/.github/scripts/tests/ci-plan.test.mjs b/.github/scripts/tests/ci-plan.test.mjs index e03b6d3a1e6..c2d561467d8 100644 --- a/.github/scripts/tests/ci-plan.test.mjs +++ b/.github/scripts/tests/ci-plan.test.mjs @@ -33,7 +33,9 @@ test("a UI component retains browser and build coverage without unrelated server assert.equal(selected(result, "build"), true); assert.equal(selected(result, "e2e_shards"), true); assert.equal(selected(result, "verify_serialized_server"), false); - assert.deepEqual([...selectedTasks(result)].filter((task) => /server|chat|runner/.test(task)), []); + // The Runner support suite imports UI modules and checks that catalog-listed + // UI files stay tracked, so UI source changes are its inputs too. + assert.deepEqual([...selectedTasks(result)].filter((task) => /server|chat|runner/.test(task)), ["ci:runner-e2e-support"]); assert.equal(result.jobs.general_tests.selected, 2); }); diff --git a/.github/scripts/tests/ci-profile.test.mjs b/.github/scripts/tests/ci-profile.test.mjs index 4f8242c2863..39f21452fa7 100644 --- a/.github/scripts/tests/ci-profile.test.mjs +++ b/.github/scripts/tests/ci-profile.test.mjs @@ -330,10 +330,10 @@ test("workflow exporter runs trusted code and keeps keys out of the PR jobs", () assertNoPrWorkflowCredentials(pr); assert.match(pr, /if: always\(\) && env.PAPERCLIP_CI_PROFILE == 'true'/); assert.doesNotMatch(pr, /PAPERCLIP_CI_PROFILE:.*labels/); - assert.equal(pr.match(/PAPERCLIP_CI_PROFILE: \$\{\{ needs.gate.outputs.profile_ci \}\}/g).length, 10); + assert.equal(pr.match(/PAPERCLIP_CI_PROFILE: \$\{\{ needs.gate.outputs.profile_ci \}\}/g).length, 11); // upload-artifact excludes files inside dot-directories by default. Keep // this opt-in scoped to timing NDJSON, never the workspace or other secrets. - assert.equal(pr.match(/path: \.ci-profile\/\*\.ndjson\n {10}include-hidden-files: true/g).length, 10); + assert.equal(pr.match(/path: \.ci-profile\/\*\.ndjson\n {10}include-hidden-files: true/g).length, 11); const routing = readFileSync(new URL("../../workflows/ci-routing.yml", import.meta.url), "utf8"); assert.match(routing, /PROFILE_CI: \$\{\{ steps.profile.outputs.profile_ci \}\}/); assert.match(routing, /\/git\/ref\/heads\/master/); diff --git a/.github/workflows/pr-trusted.yml b/.github/workflows/pr-trusted.yml index a8c452ceb96..b3b35cb0093 100644 --- a/.github/workflows/pr-trusted.yml +++ b/.github/workflows/pr-trusted.yml @@ -727,10 +727,71 @@ jobs: if-no-files-found: warn retention-days: 14 + verify_runner_e2e_support: + name: Runner support suite + needs: [gate] + if: ${{ needs.gate.outputs.full_ci == 'true' && (needs.gate.outputs.plan == '' || fromJSON(needs.gate.outputs.plan).jobs.verify_runner_e2e_support.run) }} + runs-on: ${{ needs.gate.outputs.runner }} + timeout-minutes: 20 + + env: + PAPERCLIP_CI_PROFILE: ${{ needs.gate.outputs.profile_ci }} + PAPERCLIP_CI_PROFILE_JOB: Runner support suite + + # The paid Runner workflows depend on this suite: workflow security + # policy, evidence oracles, cost receipts and controller recovery. It + # used to run only in a manual Docker Runner check, so master changes + # broke it unseen (#19527, #19529). It needs no provider credentials. + steps: + - name: Checkout repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Initialize CI profile + continue-on-error: true + if: env.PAPERCLIP_CI_PROFILE == 'true' + run: node scripts/ci-profile/initialize.mjs + + - name: Setup Node.js for pnpm bootstrap + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + package-manager-cache: false + + - name: Setup pnpm + uses: pnpm/action-setup@0977fd99725f1db4007ccb2928dbb4e90d06cc86 # v6 + env: + NPM_CONFIG_AUDIT: "false" + NPM_CONFIG_FUND: "false" + NPM_CONFIG_UPDATE_NOTIFIER: "false" + with: + version: 9.15.4 + + # Share the checked-in lockfile key with master. PR merge refs must not + # save full copies of the store or evict the post-merge build caches. + - name: Restore materialized dependencies and shared builds + uses: ./.github/actions/setup-ci-dependencies + + # Builds the plugin SDK first, then runs Vitest and the node:test files. + - name: Run the Runner support suite + run: pnpm test:e2e:runner:unit + + - name: Upload CI profile timings + continue-on-error: true + if: always() && env.PAPERCLIP_CI_PROFILE == 'true' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ci-profile-${{ github.run_attempt }}-${{ github.job }}-${{ strategy.job-index || 0 }} + path: .ci-profile/*.ndjson + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + verify: name: verify if: ${{ always() }} - needs: [gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, sandbox_provider_tests, verify_serialized_server, canary_dry_run] + needs: [gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, sandbox_provider_tests, verify_serialized_server, canary_dry_run, verify_runner_e2e_support] runs-on: ${{ needs.gate.outputs.runner }} timeout-minutes: 5 steps: @@ -749,7 +810,7 @@ jobs: const plan = process.env.CI_PLAN ? JSON.parse(process.env.CI_PLAN) : null; if (plan) assert.equal(plan.version, 1, 'Supported task plan required'); if (plan?.testSelection) assert.equal(results.test_selection.result, 'success', 'Published selection must succeed'); - for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","sandbox_provider_tests","verify_serialized_server","canary_dry_run"]) { + for (const job of ["typecheck_release_registry","general_tests","verify_paperclip_runner","build","docker_context_integrity","sandbox_provider_tests","verify_serialized_server","canary_dry_run","verify_runner_e2e_support"]) { if (plan) assert.equal(typeof plan.jobs[job]?.run, 'boolean', 'Missing suite decision: ' + job); const selected = process.env.FULL_CI === 'true' && (!plan || plan.jobs[job].run); assert.equal(results[job].result, selected ? 'success' : 'skipped', job); diff --git a/scripts/__tests__/e2e-shard.test.mjs b/scripts/__tests__/e2e-shard.test.mjs index b77536157b2..e1474715184 100644 --- a/scripts/__tests__/e2e-shard.test.mjs +++ b/scripts/__tests__/e2e-shard.test.mjs @@ -246,6 +246,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () "verify_serialized_server", "canary_dry_run", "e2e_shards", + "verify_runner_e2e_support", ]) { assert.match( jobs.get(jobId), @@ -263,7 +264,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () const verify = jobs.get("verify"); assert.match( verify, - /^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, sandbox_provider_tests, verify_serialized_server, canary_dry_run\]$/m, + /^ {4}needs: \[gate, test_selection, policy, typecheck_release_registry, general_tests, verify_paperclip_runner, build, docker_context_integrity, sandbox_provider_tests, verify_serialized_server, canary_dry_run, verify_runner_e2e_support\]$/m, ); assert.equal(runAggregate(verify, true).status, 0); assert.equal(runAggregate(verify, false).status, 0); @@ -272,7 +273,7 @@ test("the trusted PR workflow limits full CI to merge-relevant stack layers", () "an optional selector cannot block the successful original matrix"); } for (const job of ["gate", "policy", "typecheck_release_registry", "general_tests", "verify_paperclip_runner", - "build", "docker_context_integrity", "sandbox_provider_tests", "verify_serialized_server", "canary_dry_run"]) { + "build", "docker_context_integrity", "sandbox_provider_tests", "verify_serialized_server", "canary_dry_run", "verify_runner_e2e_support"]) { for (const result of ["failure", "cancelled", "skipped"]) { assert.notEqual(runAggregate(verify, true, { [job]: result }).status, 0, `${job}: ${result}`); } diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index dcd913f3133..cf4df5cbe18 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1184,6 +1184,13 @@ unavailable/terminated test process produce a `launcher/environment failure` diagnostic and exit 2. Assertion failures retain the test runner's exit status and output; assess installed-product failures from those assertions separately. +PR CI runs `pnpm test:e2e:runner:unit` as the required "Runner support suite" +job in `pr-trusted.yml`. It runs when server, CLI, package or UI source +changes, and on every full CI run. The suite pins workflow security policy, +evidence oracles, profile versions, suite hashes and dated fixtures. When a +change to one of those is deliberate, update the pin in the same pull request +and name the reason. Do not weaken a security check to make it pass. + The OpenRouter ranking snapshot is tracked in `openrouter-models.json`; nightly runs never mutate it. Refresh it deliberately, review the source/capture/hash diff, and rerun credential-free checks: diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index f3760eb3cca..7a452216c26 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -13,6 +13,8 @@ if (!existsSync(vitest)) { console.error(`${prefix} launcher/environment failure: Vitest is not installed. Run pnpm install before validation.`); process.exit(2); } +// Load after the check: a missing install must still report the launcher failure. +const { invocationEnv } = await import(new URL("../../scripts/ci-profile/record.mjs", import.meta.url).href); const env = { ...process.env }; delete env.NODE_PATH; @@ -21,14 +23,17 @@ delete env.NODE_OPTIONS; for (const [name, command] of [ // Server modules under test import the built plugin SDK (and shared) package. ["Plugin build dependencies", ["scripts/ensure-plugin-build-deps.mjs"]], - ["Vitest",[vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], - ["Node tests", ["--test", "tests/runner-e2e/product-regression-testops.test.mjs", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", + ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], + // The wrapper adds the CI profiling reporter when PAPERCLIP_CI_PROFILE is set. + ["Node tests", ["scripts/ci-profile/node-test.mjs", "tests/runner-e2e/product-regression-testops.test.mjs", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs", "tests/runner-e2e/chat-completion-testops-api.test.mjs", "tests/runner-e2e/chat-completion-testops-contract.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-reconcile.test.mjs", "tests/runner-e2e/chat-full-catalog-testops-adapter.test.mjs", "tests/runner-e2e/chat-catalog-testops-streaming.test.mjs"]], ]) { - const result = spawnSync(process.execPath, command, { cwd: root, env, stdio: "inherit" }); + const result = spawnSync(process.execPath, command, { + cwd: root, env: name === "Vitest" ? invocationEnv("vitest", env) : env, stdio: "inherit", + }); if (result.error || result.signal) { console.error(`${prefix} launcher/environment failure: ${name} ${result.error?.message ?? `terminated by ${result.signal}`}`); process.exit(2); diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index bece70afb21..9c554557990 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -1,9 +1,14 @@ +import { fileURLToPath } from "node:url"; import { configDefaults, defineConfig } from "vitest/config"; export default defineConfig({ root: import.meta.dirname, test: { name: "runner-e2e-support", + // CI profiling records per-test timings, like the root and Runner package configs. + ...(process.env.PAPERCLIP_CI_PROFILE === "true" + ? { reporters: [...configDefaults.reporters, fileURLToPath(new URL("../../scripts/ci-profile/vitest-reporter.mjs", import.meta.url))] } + : {}), globalSetup: ["./verified-provider-bundles.setup.mjs"], include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 3a6f8545f93..417b53bcaf0 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -83,10 +83,10 @@ describe("public repository paid workflow security", () => { }); // PR #18348 added the ninth pr-trusted bootstrap, in the Vitest selection job. - // #19533 added the tenth, for the sandbox provider tests. + // #19533 added the tenth (sandbox provider tests); the eleventh runs this suite. const trustedPnpmBootstraps = { "runner-full-stack-e2e.yml": 8, - "pr-trusted.yml": 10, + "pr-trusted.yml": 11, } as const; it("keeps pnpm bootstrap registry telemetry out of trusted workflow setup", async () => { diff --git a/turbo.json b/turbo.json index d4d1d1fd9d1..e8a3ccdc3f0 100644 --- a/turbo.json +++ b/turbo.json @@ -234,6 +234,16 @@ ], "outputs": [], "cache": false + }, + "//#ci:runner-e2e-support": { + "inputs": [ + "server/**", + "cli/**", + "packages/**", + "ui/**" + ], + "outputs": [], + "cache": false } } }