diff --git a/.github/workflows/runner-e2e-testops-registration.yml b/.github/workflows/runner-e2e-testops-registration.yml new file mode 100644 index 00000000000..61c8a0b10a8 --- /dev/null +++ b/.github/workflows/runner-e2e-testops-registration.yml @@ -0,0 +1,103 @@ +name: Runner E2E TestOps registration +run-name: Runner E2E TestOps registration / ${{ inputs.plan }} / ${{ inputs.operation }} + +# One-shot, idempotent registration of a sibling runner E2E TestOps plan. It +# never runs a provider cell. `inspect` only reads; `register` creates missing +# cases and the plan and assigns the plan to the standing plan's GitHub job. +on: + workflow_dispatch: + inputs: + plan: + description: "Sibling TestOps plan to register" + type: choice + options: [reliability] + default: reliability + operation: + description: "inspect reads TestOps only; register creates any missing case, the plan and its job assignment" + type: choice + options: [inspect, register] + default: inspect + +permissions: + contents: read + +concurrency: + group: runner-e2e-testops-registration + cancel-in-progress: false + +jobs: + authorize: + name: Authorize registration + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + steps: + - name: Require default branch and allowlisted numeric actor IDs + env: + GH_TOKEN: ${{ github.token }} + REF: ${{ github.ref }} + DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + ACTOR: ${{ github.actor }} + ACTOR_ID: ${{ github.actor_id }} + TRIGGERING_ACTOR: ${{ github.triggering_actor }} + ALLOWED_ACTOR_IDS: ${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }} + run: | + set -euo pipefail + if [ "$REF" != "refs/heads/$DEFAULT_BRANCH" ]; then + echo "TestOps registration may run only from the default branch." >&2 + exit 1 + fi + if ! jq -e 'type == "array" and length > 0 and all(.[]; type == "number" and . > 0 and floor == .)' <<< "${ALLOWED_ACTOR_IDS:-}" >/dev/null; then + echo "RUNNER_E2E_ALLOWED_ACTOR_IDS must be a non-empty JSON array of numeric GitHub user IDs." >&2 + exit 1 + fi + for attempt in 1 2 3; do + if triggering_actor_id="$(timeout 30s gh api "users/$TRIGGERING_ACTOR" --jq .id)"; then + break + fi + if [ "$attempt" = 3 ]; then exit 1; fi + sleep "$((attempt * 2))" + done + if [ "$triggering_actor_id" != "$ACTOR_ID" ] && [ "$TRIGGERING_ACTOR" = "$ACTOR" ]; then + echo "GitHub actor identity contexts disagree; refusing registration." >&2 + exit 1 + fi + for candidate in "$triggering_actor_id" "$ACTOR_ID"; do + if ! jq -e --argjson candidate "$candidate" 'index($candidate) != null' <<< "$ALLOWED_ACTOR_IDS" >/dev/null; then + echo "The initiating GitHub account is not authorized to register TestOps plans." >&2 + exit 1 + fi + done + + register: + name: Register the Reliability plan in TestOps + needs: authorize + runs-on: ubuntu-latest + timeout-minutes: 15 + environment: runner-e2e-testops-reporting + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.sha }} + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + - name: Inspect or register the plan, its cases and its job assignment + env: + ALLURE_TOKEN: ${{ secrets.ALLURE_TOKEN }} + ALLURE_ENDPOINT: https://paperclip.testops.cloud + ALLURE_PROJECT_ID: '1' + REGISTRATION_PLAN: ${{ inputs.plan }} + REGISTRATION_OPERATION: ${{ inputs.operation }} + run: node tests/runner-e2e/reliability-testops-registration.mjs testops-registration/receipt.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: reliability-testops-registration-${{ github.run_id }}-${{ github.run_attempt }} + path: testops-registration/receipt.json + if-no-files-found: ignore + retention-days: 90 diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index dd322fedca5..05610f75f75 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -49,9 +49,14 @@ on: type: string required: false testops_case: - description: "Product regression TestOps bootstrap: all, case name or case ID; leave empty for normal campaigns" + description: "TestOps plan bootstrap: all, case name or case ID; leave empty for normal campaigns" type: string required: false + testops_plan: + description: "TestOps plan for a testops_case bootstrap; a TestOps launch names its own plan" + type: choice + options: [product-regression, reliability] + default: product-regression ALLURE_JOB_RUN_ID: description: "Allure TestOps service input; its requested selection takes precedence" type: string @@ -307,7 +312,7 @@ jobs: if-no-files-found: error testops_selection: - name: Bind Product regression TestOps selection + name: Bind TestOps plan selection needs: authorize if: github.event_name == 'workflow_dispatch' && (inputs.ALLURE_JOB_RUN_ID != '' || inputs.testops_case != '') runs-on: ubuntu-latest @@ -348,10 +353,12 @@ jobs: SELECT_ENVIRONMENT: ${{ inputs.environment }} SELECT_CASE: ${{ inputs.case }} TESTOPS_CASE: ${{ inputs.testops_case }} + TESTOPS_PLAN: ${{ inputs.testops_plan || 'product-regression' }} ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }} ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }} - ALLURE_LAUNCH_NAME: Product regression · ${{ github.run_number }} - ALLURE_LAUNCH_TAGS: product-regression,github-actions + # Names only a bootstrap launch; a TestOps launch already has its plan's name. + ALLURE_LAUNCH_NAME: ${{ inputs.testops_plan == 'reliability' && 'Reliability' || 'Product regression' }} · ${{ github.run_number }} + ALLURE_LAUNCH_TAGS: ${{ inputs.testops_plan == 'reliability' && 'reliability' || 'product-regression' }},github-actions run: node tests/runner-e2e/product-regression-testops-cli.mjs select product-testops/selection.json - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: @@ -526,9 +533,9 @@ jobs: { echo "matrix=$(jq -c '{include: .include}' <<< "$catalog_json")" echo "needs_daytona=$(jq -r '.needsDaytona' <<< "$catalog_json")" - echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" + echo "needs_runner_typescript=$(jq -r '[.include[] | select((.profileId | startswith("runner-opencode")) or (.profileId | startswith("runner-acpx-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" echo "needs_native_binaries=$(jq -r '[.include[] | select((.profileId | startswith("runner-")) or (.suiteId == "openrouter-model-breadth"))] | length > 0' <<< "$catalog_json")" - echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId == "runner-opencode") or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")" + echo "needs_remote_provider_pack=$(jq -r '[.include[] | select((.environmentId == "daytona") and ((.profileId | startswith("runner-opencode")) or (.profileId | startswith("runner-acpx-"))))] | length > 0' <<< "$catalog_json")" echo "execution_ids=$(jq -c '.executionIds' <<< "$catalog_json")" } >> "$GITHUB_OUTPUT" if ! [[ "$MAX_PARALLEL_LIMIT" =~ ^[1-9][0-9]*$ ]] || [ "$MAX_PARALLEL_LIMIT" -gt 100 ]; then @@ -1173,7 +1180,7 @@ jobs: test -e "$daytona_root/node_modules/@daytonaio/sdk" - name: Materialize verified pinned OpenCode executable - if: matrix.environmentId == 'local' && (matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth') + if: matrix.environmentId == 'local' && (matrix.profileId == 'legacy-opencode' || startsWith(matrix.profileId, 'runner-opencode') || matrix.suiteId == 'openrouter-model-breadth') run: node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs - name: Install checksum-verified Grok executable @@ -1188,7 +1195,7 @@ jobs: path: runner-e2e-build - name: Download immutable remote provider pack - if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) + if: matrix.environmentId == 'daytona' && (startsWith(matrix.profileId, 'runner-opencode') || startsWith(matrix.profileId, 'runner-acpx-')) uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: ${{ needs.build_remote_provider_pack.outputs.provider_pack_artifact_name }} @@ -1197,7 +1204,7 @@ jobs: - name: Verify and restore campaign outputs if: startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' env: - NEEDS_RUNNER_TYPESCRIPT: ${{ matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }} + NEEDS_RUNNER_TYPESCRIPT: ${{ startsWith(matrix.profileId, 'runner-opencode') || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth' }} NEEDS_NATIVE_BINARY: ${{ startsWith(matrix.profileId, 'runner-') || matrix.suiteId == 'openrouter-model-breadth' }} run: | set -euo pipefail @@ -1217,7 +1224,7 @@ jobs: fi - name: Verify and restore remote provider pack - if: matrix.environmentId == 'daytona' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-')) + if: matrix.environmentId == 'daytona' && (startsWith(matrix.profileId, 'runner-opencode') || startsWith(matrix.profileId, 'runner-acpx-')) env: IMAGE_SOURCE_REVISION: ${{ needs.daytona_image.outputs.source_revision }} run: | @@ -1237,7 +1244,7 @@ jobs: packages/paperclip-runner/provider-pack/provider-pack.json >/dev/null - name: Qualify local provider Node interpreter - if: matrix.environmentId == 'local' && (matrix.profileId == 'runner-opencode' || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth') + if: matrix.environmentId == 'local' && (startsWith(matrix.profileId, 'runner-opencode') || startsWith(matrix.profileId, 'runner-acpx-') || matrix.suiteId == 'openrouter-model-breadth') run: | node <<'NODE' const fs = require("node:fs"); @@ -2047,7 +2054,7 @@ jobs: fi publish_testops: - name: Publish bound Product regression results to TestOps + name: Publish bound TestOps plan results needs: [authorize, testops_selection, catalog, report] if: always() && needs.testops_selection.outputs.job_run_started == 'true' && inputs.image_only != true runs-on: ubuntu-latest diff --git a/doc/evals.md b/doc/evals.md index ccbd4ec8d39..54572e71a5b 100644 --- a/doc/evals.md +++ b/doc/evals.md @@ -12,6 +12,18 @@ Subset runs do not qualify the full plan or the historical 355-row snapshot. Catalog and projection tests are not live qualification; retain the fresh TestOps launch and GitHub run when reporting acceptance. +The sibling **Reliability — staging soak** TestOps plan uses the same lane for +the seven staging-soak reliability cases. Six run once on native OpenCode with +DeepSeek 1017 through OpenRouter: on Daytona for `three-turn-revise`, +`long-silent-command` and `queued-across-restart`, and locally for the others. +`queue-steer` needs steering, which OpenCode does not support, so it runs on +native Codex on Daytona. +A launch selects cases from exactly one plan, so plan 2 keeps its 20-case +denominator. The plan is launched from TestOps before a release, or bootstrapped +with `testops_plan=reliability`. It has no schedule and does not run per PR. +Selection fails closed until TestOps-assigned identities are committed; see the +[registration steps](../tests/runner-e2e/EVERYDAY-WORKFLOWS.md#testops-reliability-plan). + The explicit deployment-targeted [saved agent instructions journey](../tests/runner-e2e/product-journeys/AGENT-SETTINGS.md) checks an owned agent’s ordinary UI save, persisted instruction bytes, one current-run output, and restoration. The task request omits its instruction-only reference. It reuses the existing Product E2E target and report path; local controls do not establish a staging pass. The explicit-only [verified API artifact transfer](../tests/runner-e2e/README.md#verified-api-artifact-transfer) @@ -381,7 +393,7 @@ image digest and `DAYTONA_API_KEY`; follow the README and fixture security guide explicitly; use a narrow selector while developing a fixture. Its staging-soak reliability cases need `--group reliability` as well; see [Everyday workflows](../tests/runner-e2e/EVERYDAY-WORKFLOWS.md#staging-soak-reliability-group) -for the command and the before-release cadence. +for the command, the TestOps Reliability plan and the before-release cadence. For Runner Evals, the narrowest useful local validation is the report program's help/validation path and the deterministic Runner checks documented in diff --git a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md index 2676d06357d..705164b8606 100644 --- a/tests/runner-e2e/EVERYDAY-WORKFLOWS.md +++ b/tests/runner-e2e/EVERYDAY-WORKFLOWS.md @@ -86,14 +86,16 @@ profile/environment selector, nor a scheduled campaign selects them. Only the pnpm test:e2e:runner -- --list --suite everyday-workflows --group reliability pnpm test:e2e:runner -- --suite everyday-workflows --group reliability --max-parallel 2 pnpm test:e2e:runner -- --suite everyday-workflows --case queue-steer +pnpm test:e2e:runner -- --id everyday-workflows.runner-opencode-deepseek-1017.local.interrupt-redirect pnpm test:e2e:runner -- --id everyday-workflows.runner-codex.daytona.interrupt-redirect ``` -**Cadence:** run the whole group before a release and after a risky runner, -recovery, queue, tree-control or restart change. Do not run it per PR. It is -paid: ten Daytona cells, single attempt each, with a 1,000-cent company and -lead-agent hard stop. Missing provider or Daytona credentials, or a mutable -Daytona image, stop the launch before any provider spend. +**Cadence:** launch the [TestOps Reliability plan](#testops-reliability-plan) +before a release and after a risky runner, recovery, queue, tree-control or +restart change. Do not run it per PR, and it has no schedule. It is paid: +single attempt per cell, with a 1,000-cent company and lead-agent hard stop. +Missing provider or Daytona credentials, or a mutable Daytona image, stop the +launch before any provider spend. **Product rule (every case):** work interrupted by a restart, Stop, reassignment or lost runner must resume automatically, or offer a Retry that @@ -122,15 +124,91 @@ message delivered by more than one successful run is duplicate execution. | `queue-steer` | H2 | During an active run, queue three follow-ups through the public comment API; edit one, discard one, and steer one (retrying only `steering_temporarily_unavailable`, for up to two minutes). | Edited text reaches the agent and the original never does; the discarded text never reaches the agent; the steered message is acknowledged by the active turn and never replayed; the remaining message is handled by exactly one run. | | `queued-across-restart` | S55/U9 (fix #19431) | Queue a follow-up through the composer while a run is active, then restart the controller with the existing restart machinery. | The follow-up dispatches only after the restart, automatically within six minutes, exactly once; every observed `queued` wait names a cause (`server_restarting`, `task_drain`, …) rather than a bare "Queued"; any offered Retry works on first use. | -**Matrix (10 cells).** Each case runs once on native Codex (`runner-codex`, +**Matrix (16 cells).** The `reliability` group selects the seven TestOps plan +cells and nine explicit-only extras. Six plan cells use native OpenCode with +DeepSeek 1017 via OpenRouter. `queue-steer` must steer an active turn, and the +OpenCode server driver declares `steering: false`, so the queued-comment route +answers `steering_unsupported`. Its plan cell is the existing steering-capable +native Codex Daytona cell. + +| Case | Plan cell | Why this profile and environment | +|---|---|---| +| `interrupt-redirect` | `runner-opencode-deepseek-1017.local` | S47 is control-plane interrupt and Stop-settlement dispatch. | +| `reassign-midrun` | `runner-opencode-deepseek-1017.local` | S48 is the reassignment request and native stop settlement. | +| `three-turn-revise` | `runner-opencode-deepseek-1017.daytona` (per-turn lease) | S49 hit turn 3 on an ephemeral Daytona lease. | +| `long-silent-command` | `runner-opencode-deepseek-1017.daytona` (warm lease) | S52/S52b hung on remote runners during a silent command. | +| `tree-pause-release` | `runner-opencode-deepseek-1017.local` | S50/S50b is tree-hold and parked-retry scheduling. | +| `queue-steer` | `runner-codex.daytona` (warm lease) | H2 needs a steering-capable driver; OpenCode has no steering. | +| `queued-across-restart` | `runner-opencode-deepseek-1017.daytona` (warm lease) | S55/U9 happened across a restart with a remote lease. | + +The local plan cells cannot reproduce Daytona-only signatures: the remote +`workspace_busy` save after Stop (S47's sandbox variant), remote stop +settlement during reassignment, and a lost or replaced remote runner during a +tree hold. The S51 lost-runner signature has no case yet. The explicit-only +native Codex Daytona extra for each case still covers the sandbox variants of +S47, S48 and S50. + +The extras are outside the TestOps plan. Every case except `queue-steer` (whose +Codex cell is the plan cell) also runs once on native Codex (`runner-codex`, `gpt-5.6-sol`) on Daytona: the warm environment, except `three-turn-revise` on -the per-turn lease. `long-silent-command` also runs on the released ACPX -facades Grok (`runner-acpx-grok`), Kimi (`runner-acpx-kimi`) and OpenRouter/Pi -(`runner-acpx-pi`), which hung 3/3 before the fix. They run as ordinary released -profiles, as in the rolling-deployment matrix; a profile whose ACPX -qualification returns to pending drops out of the group automatically. Their -credentials (`XAI_API_KEY`, `KIMI_API_KEY`, `OPENROUTER_API_KEY`) are only -required when those cells are selected. +the per-turn lease. `long-silent-command` also runs on +the released ACPX facades Grok (`runner-acpx-grok`), Kimi (`runner-acpx-kimi`) +and OpenRouter/Pi (`runner-acpx-pi`), which hung 3/3 before the fix. They run +as ordinary released profiles, as in the rolling-deployment matrix; a profile +whose ACPX qualification returns to pending drops out of the group +automatically. Their credentials (`OPENAI_API_KEY`, `XAI_API_KEY`, +`KIMI_API_KEY`, `OPENROUTER_API_KEY`) are only required when those cells are +selected. + +### TestOps Reliability plan + +The **Reliability — staging soak** TestOps plan is a sibling of the standing +Product regression plan (plan 2). It uses the same trusted +`runner-full-stack-e2e.yml` lane, the same TestOps GitHub job, selection, +paid admission, cells, retained report and dashboard, Allure projection, +upload verification and failure classes. Its mapping lives in +`reliability-testops.mjs`; `product-regression-testops.mjs` selects either +plan. A launch selects cases from exactly one plan; mixing them fails. Each +plan keeps its own denominator: only a complete seven-case selection is a full +Reliability verdict, and plan 2 still needs its 20 cases. Reports tag each +reliability result with its soak signature (`soak-S55/U9`, for example). A +launch needs `OPENROUTER_API_KEY`, `DAYTONA_API_KEY` and, for `queue-steer`, +`OPENAI_API_KEY`; the workflow passes each only to the cells that declare it. + +A launch from TestOps dispatches the workflow with `ALLURE_JOB_RUN_ID`; the +selected case IDs choose the plan and its cells. To bootstrap one manually: + +```sh +gh workflow run runner-full-stack-e2e.yml --ref master -f all=false \ + -f testops_plan=reliability -f testops_case=all # or a case name or ID +``` + +The plan has no schedule. A weekly run would add a paid 50-minute Daytona +cell to every week, but these signatures move with release-risky changes +rather than with time. Launch it from TestOps before releases instead. + +**Registration.** TestOps assigns case and plan IDs, and this repository never +invents them. Until `reliabilityRegistration` in `reliability-testops.mjs` +holds the plan ID and all seven case IDs, selection fails closed +(`product_regression_testops_reliability_unregistered` for a bootstrap, +`case_mismatch` for a TestOps launch) before any provider spend. A partial, +duplicate or plan-2 registration fails too. To register: + +1. Dispatch `runner-e2e-testops-registration.yml` from master with + `operation=inspect`. It only reads TestOps and reports existing cases, the + plan and the job assigned to plan 2. +2. Dispatch it again with `operation=register`. It reuses cases with the same + full name, creates any missing case and the plan, and assigns the plan to + plan 2's GitHub job with plan 2's job parameters. A rerun changes nothing. +3. Copy the `reliabilityRegistration` snippet from the run summary (or the + `reliability-testops-registration-*` receipt artifact) into + `reliability-testops.mjs`, and merge it in one reviewed PR. +4. Launch the plan from TestOps. Tag only a complete seven-case attempt + `full-run`. + +Only the registration job receives `ALLURE_TOKEN`, from the protected +`runner-e2e-testops-reporting` environment, and it runs only trusted default-branch +code for allowlisted actors. It never runs a provider cell. **Coverage boundary.** The graders and selection are calibrated by `everyday-reliability.test.ts` against passing and plausible failing diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 627535a0bc9..85f1e3a042a 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -1650,6 +1650,16 @@ case. All 20 imported case IDs and full names remain stable. The lane uses the existing native Codex/local cells and the existing native OpenCode/local task-workspaces cell. It does not qualify the historical 355-result snapshot. +The sibling **Reliability — staging soak** plan runs the seven staging-soak +reliability cases through this same lane: six on native OpenCode with DeepSeek +1017, and `queue-steer` on steering-capable native Codex. +Bootstrap it with `-f testops_plan=reliability -f testops_case=all` (or one case); +a TestOps launch names its plan through its case IDs. A launch never mixes the +two plans, and each keeps its own full-plan denominator. Its TestOps identities +come from the trusted `runner-e2e-testops-registration.yml` workflow; until they +are committed, selection fails closed. See +[the Reliability plan](EVERYDAY-WORKFLOWS.md#testops-reliability-plan). + The protected `runner-e2e-testops-reporting` environment supplies `ALLURE_TOKEN`. Only default-branch selection and reporting code receives this token. The ordinary paid provider jobs, their authorization and their evidence paths stay diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 5ea48690cb2..41f01fb2452 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -329,11 +329,11 @@ describe("runner E2E catalog", () => { expect(localIntegrityTasks).toHaveLength(2); expect(openRouterBreadthTasks).toHaveLength(3); expect(runnerSuites.map((suite) => suite.expectedMatrixSize)).toEqual([ - 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 62, 6, 6, 20, 26, 2, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, + 1, 9, 2, 9, 2, 4, 1, 1, 3, 2, 2, 4, 4, 4, 4, 4, 4, 8, 10, 6, 63, 12, 6, 4, 2, 4, 2, 30, 3, 16, 16, 20, 2, 1, 6, 8, 46, 2, 23, 15, 68, 6, 6, 20, 26, 2, 52, 5, 28, 18, 2, 6, 6, 13, 10, 4, 48, 16, 10, 2, 1, 1, 4, 116, 2, 1, 2, ]); - expect(validateRunnerCatalog()).toHaveLength(834); - expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(834); + expect(validateRunnerCatalog()).toHaveLength(840); + expect(new Set(runnerMatrix.map((entry) => entry.id)).size).toBe(840); expect( runnerMatrix.filter((entry) => entry.suite.id === "core-compatibility"), ).toHaveLength(48); diff --git a/tests/runner-e2e/catalog.ts b/tests/runner-e2e/catalog.ts index b1aa9050765..8b8a7749d9d 100644 --- a/tests/runner-e2e/catalog.ts +++ b/tests/runner-e2e/catalog.ts @@ -41,7 +41,7 @@ import { lifecycleLiveTasks, lifecycleLiveDefinitionDigest } from "./lifecycle-l import { publicMcpTasks, publicMcpExpandedDigest, publicMcpSetupDigest, publicMcpWorkflowDigest, publicMcpWorkerInstructions, publicMcpWorkerSkillDigest } from "./public-mcp-cases.js"; import { graderVersion as publicMcpGraderVersion } from "./public-mcp-grading.js"; import { everydayTasks, productionStoryProfile } from "./everyday-cases.js"; -import { EVERYDAY_RELIABILITY_GROUP, EVERYDAY_RELIABILITY_SLOW_GROUP, RELIABILITY_FACADE_PROFILE_IDS, everydayReliabilityExecutionIds, everydayReliabilitySignatures, everydayReliabilityTasks } from "./everyday-reliability-cases.js"; +import { EVERYDAY_RELIABILITY_GROUP, EVERYDAY_RELIABILITY_SLOW_GROUP, RELIABILITY_FACADE_PROFILE_IDS, RELIABILITY_PLAN_PROFILE_ID, everydayReliabilityExecutionIds, reliabilityPlanCells, everydayReliabilitySignatures, everydayReliabilityTasks } from "./everyday-reliability-cases.js"; import { REMOTE_WORKSPACE_STORAGE_SUITE, remoteWorkspaceStoragePrompt } from "./remote-workspace-storage.js"; import { CONNECTION_GUIDANCE_SUITE, CONNECTION_GUIDANCE_BUDGET_CENTS, connectionGuidanceTasks, connectionGuidanceDefinitionDigest } from "./connection-guidance-cases.js"; @@ -1270,8 +1270,18 @@ const everydayFacadeProfiles = RELIABILITY_FACADE_PROFILE_IDS }); /** Released facades currently in the reliability group; a pending one drops out. */ export const everydayReliabilityFacadeProfileIds = everydayFacadeProfiles.map(profile => profile.id); +/** + * Native OpenCode with DeepSeek 1017 through OpenRouter. It is the chat-feature + * candidate and the TestOps Reliability plan profile. The OpenCode server driver + * runs this model on Daytona exactly as it runs the qualified runner-opencode + * model there; only the configured model differs. + */ +const runnerOpenCodeDeepSeek1017Profile = nativeProfile({ id: RELIABILITY_PLAN_PROFILE_ID, label: "Native OpenCode DeepSeek 1017", + provider: "opencode", model: CHAT_FEATURE_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local", "daytona"], + modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-feature-testops:deepseek-v4.1-flash-1017" } }); const everydayWorkflowProfiles = [...everydayProfiles, productionStoryProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!), + productionStoryProfile(runnerOpenCodeDeepSeek1017Profile), ...everydayFacadeProfiles]; /** Every reliability cell outside the deliberately small explicit matrix. */ @@ -1739,8 +1749,9 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ environmentOverrides: { "three-turn-revise": { daytona: runnerEnvironments.find(environment => environment.id === "daytona")! } }, tasks: [...everydayTasks.map(task => ["hire-reuse", "delegate-feedback", "recover-controller"].includes(task.id) ? { ...task, automaticRetryPolicy: "single_attempt" as const } : task), ...everydayReliabilityTasks], - // 52 everyday cells, one native Daytona cell per reliability case, and released facades for the silent command. - expectedMatrixSize: 52 + everydayReliabilityTasks.length + everydayFacadeProfiles.length, + // 52 everyday cells; one native Codex Daytona cell per reliability case, six DeepSeek plan cells (queue-steer needs + // Codex steering) and released facades for the silent command. + expectedMatrixSize: 52 + 2 * everydayReliabilityTasks.length - 1 + everydayFacadeProfiles.length, excludedExecutionIds: [...everydayProfiles.flatMap(profile => everydayTasks .filter(task => !["build-revise", "delegate-feedback", "recover-controller", "create-skill-studio"].includes(task.id)) .map(task => `everyday-workflows.${profile.id}.daytona.${task.id}`)), @@ -1748,14 +1759,16 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ `everyday-workflows.runner-opencode.daytona.${task.id}`, ...(["hire-reuse", "delegate-feedback"].includes(task.id) ? [] : [`everyday-workflows.runner-opencode.local.${task.id}`]), ]), - // ACPX facades join only the reliability long-silent-command cells. - ...everydayFacadeProfiles.flatMap(profile => everydayTasks.flatMap(task => + // ACPX facades and the DeepSeek plan profile join only reliability cells. + ...[...everydayFacadeProfiles, runnerOpenCodeDeepSeek1017Profile].flatMap(profile => everydayTasks.flatMap(task => ["local", "daytona"].map(environmentId => `everyday-workflows.${profile.id}.${environmentId}.${task.id}`))), ...everydayReliabilityExcludedIds("everyday-workflows"), ], - definitionMetadata: { version: 11, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", + definitionMetadata: { version: 12, instructions: "production", grading: "outcome-and-invariants", scheduling: "explicit-only", reliability: { group: EVERYDAY_RELIABILITY_GROUP, selection: "explicit-group-only", signatures: everydayReliabilitySignatures, - productRule: "interrupted-work-resumes-automatically-or-offers-a-working-first-retry", companyAndLeadBudgetCents: 1_000 }, + productRule: "interrupted-work-resumes-automatically-or-offers-a-working-first-retry", companyAndLeadBudgetCents: 1_000, + testOpsPlan: { profile: RELIABILITY_PLAN_PROFILE_ID, cells: reliabilityPlanCells, + cadence: "testops-launch-or-manual-dispatch", extras: "explicit-only-codex-and-acpx-cells" } }, controllerDowntimeMs: 125_000, remoteCodexRestart: "bounded-main-server-outage-receipt-empty-cache-same-active-process-and-finalization", procedureCases: { ids: ["hire-reuse", "delegate-feedback"], maximumAttemptsPerCell: 1, companyAndLeadBudgetCents: 1_000 } }, @@ -1851,9 +1864,7 @@ export const runnerSuites: readonly RunnerSuiteFixture[] = [ profiles: [chatFeatureProfile(nativeProfile({ id: "runner-codex-subscription", label: "Native Codex subscription", provider: "codex", model: DEFAULT_CODEX_LOCAL_MODEL, credential: "CODEX_AUTH_JSON", supportedEnvironments: ["local"] })), chatFeatureProfile(runnerProfiles.find(profile => profile.id === "runner-opencode")!), - chatFeatureProfile(nativeProfile({ id: "runner-opencode-deepseek-1017", label: "Native OpenCode DeepSeek 1017", - provider: "opencode", model: CHAT_FEATURE_OPENCODE_MODEL, credential: "OPENROUTER_API_KEY", supportedEnvironments: ["local"], - modelQualification: { source: "candidate_runner_profile", qualificationId: "chat-feature-testops:deepseek-v4.1-flash-1017" } }))], + chatFeatureProfile(runnerOpenCodeDeepSeek1017Profile)], environments: [localEnvironment], tasks: chatFeatureTasks, expectedMatrixSize: 5, excludedExecutionIds: ["chat-features.runner-opencode-deepseek-1017.local.delegate-announcement"], definitionMetadata: { version: 2, definitionDigest: chatFeatureDefinitionDigest, scheduling: "explicit-only", diff --git a/tests/runner-e2e/everyday-reliability-cases.ts b/tests/runner-e2e/everyday-reliability-cases.ts index db349d61d06..0554a593f45 100644 --- a/tests/runner-e2e/everyday-reliability-cases.ts +++ b/tests/runner-e2e/everyday-reliability-cases.ts @@ -111,15 +111,55 @@ export const everydayReliabilityTasks: readonly RunnerTaskFixture[] = definition buildMatchers: () => [], // The workflow records independent reliability checks. })); -/** The native profile used for every reliability case, plus ACPX facades for the silent command. */ +/** The explicit-only native Codex extra for every reliability case, plus ACPX facades for the silent command. */ export const RELIABILITY_NATIVE_PROFILE_ID = "runner-codex"; export const RELIABILITY_FACADE_PROFILE_IDS = ["runner-acpx-grok", "runner-acpx-kimi", "runner-acpx-pi"] as const; -/** The deliberately small paid matrix: one native Daytona cell per case, plus facades for long-silent-command. */ +/** + * The TestOps Reliability plan profile: native OpenCode with DeepSeek 1017 + * through OpenRouter, the cheapest native profile with a Daytona-capable driver. + */ +export const RELIABILITY_PLAN_PROFILE_ID = "runner-opencode-deepseek-1017"; + +/** + * One plan cell per case. Daytona only where the soak failure was a sandbox + * lease failure: S49 (per-turn lease on turn 3), S52/S52b (a long silent command + * on a remote runner) and S55 (a restart with a remote lease). The other cases + * exercise control-plane queue, interrupt, reassignment and tree-hold paths that + * behave the same with a local runner. + * + * `queue-steer` is the exception to DeepSeek: it must steer an active turn, and + * the OpenCode server driver declares `steering: false`, so the queued-comment + * route answers `steering_unsupported`. The plan uses the existing native Codex + * Daytona cell for it, which supports steering. + */ +export const reliabilityPlanCells: Readonly> = { + "interrupt-redirect": { profile: RELIABILITY_PLAN_PROFILE_ID, environment: "local" }, + "reassign-midrun": { profile: RELIABILITY_PLAN_PROFILE_ID, environment: "local" }, + "three-turn-revise": { profile: RELIABILITY_PLAN_PROFILE_ID, environment: "daytona" }, + "long-silent-command": { profile: RELIABILITY_PLAN_PROFILE_ID, environment: "daytona" }, + "tree-pause-release": { profile: RELIABILITY_PLAN_PROFILE_ID, environment: "local" }, + "queue-steer": { profile: RELIABILITY_NATIVE_PROFILE_ID, environment: "daytona" }, + "queued-across-restart": { profile: RELIABILITY_PLAN_PROFILE_ID, environment: "daytona" }, +}; + +/** The seven TestOps Reliability plan cells, in case order. */ +export function reliabilityPlanExecutionIds(suiteId: string): string[] { + return everydayReliabilityCaseIds.map(caseId => + `${suiteId}.${reliabilityPlanCells[caseId].profile}.${reliabilityPlanCells[caseId].environment}.${caseId}`); +} + +/** + * The deliberately small paid matrix: the seven plan cells, plus the + * explicit-only extras (the native Codex Daytona cell for every other case and + * the ACPX facades for long-silent-command). Only the plan cells belong to TestOps. + */ export function everydayReliabilityExecutionIds(suiteId: string, facadeProfileIds: readonly string[] = RELIABILITY_FACADE_PROFILE_IDS): string[] { - return [ + return [...new Set([ + ...reliabilityPlanExecutionIds(suiteId), ...everydayReliabilityCaseIds.map(caseId => `${suiteId}.${RELIABILITY_NATIVE_PROFILE_ID}.daytona.${caseId}`), ...facadeProfileIds.map(profileId => `${suiteId}.${profileId}.daytona.long-silent-command`), - ]; + ])]; } diff --git a/tests/runner-e2e/everyday-reliability.test.ts b/tests/runner-e2e/everyday-reliability.test.ts index 584b5632ccc..55deb3dcc6e 100644 --- a/tests/runner-e2e/everyday-reliability.test.ts +++ b/tests/runner-e2e/everyday-reliability.test.ts @@ -3,8 +3,8 @@ import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from ". import { everydayReliabilityFacadeProfileIds, runnerMatrix } from "./catalog.js"; import { scheduledSelection } from "./scheduled-operations.js"; import { - EVERYDAY_RELIABILITY_GROUP, everydayReliabilityCaseIds, everydayReliabilityExecutionIds, everydayReliabilitySignatures, - everydayReliabilityTasks, isEverydayReliabilityCase, + EVERYDAY_RELIABILITY_GROUP, RELIABILITY_PLAN_PROFILE_ID, everydayReliabilityCaseIds, everydayReliabilityExecutionIds, + everydayReliabilitySignatures, everydayReliabilityTasks, isEverydayReliabilityCase, reliabilityPlanCells, } from "./everyday-reliability-cases.js"; import { completeStepLines, concurrentExecutors, recoveredRunIds, gradeInterruptRedirect, gradeLongSilentCommand, gradeQueueSteer, @@ -23,7 +23,8 @@ const run = (id: string, overrides: Partial = {}): StoryRun => ({ const carrying = (id: string, wake: string, overrides: Partial = {}, history = "") => run(id, { contextSnapshot: { issueId: "parent", wakeCommentIds: [wake] }, runnerProfileJson: { nativeExecutionInput: { task: { prompt: `${history} ${wake}` } } }, ...overrides }); -const reliabilityCellCount = everydayReliabilityCaseIds.length + everydayReliabilityFacadeProfileIds.length; +// One native Codex Daytona cell per case, a DeepSeek plan cell for every case except queue-steer, and facades for the silent command. +const reliabilityCellCount = 2 * everydayReliabilityCaseIds.length - 1 + everydayReliabilityFacadeProfileIds.length; const failed = (checks: StoryCheck[]) => checks.filter(check => !check.passed).map(check => check.id); describe("reliability selection stays explicit", () => { @@ -34,7 +35,8 @@ describe("reliability selection stays explicit", () => { expect(selected.map(execution => execution.id).sort()) .toEqual(everydayReliabilityExecutionIds("everyday-workflows", everydayReliabilityFacadeProfileIds).sort()); expect(selected).toHaveLength(reliabilityCellCount); - expect(selected.every(execution => execution.environment.id === "daytona")).toBe(true); + expect(selected.filter(execution => execution.environment.id !== "daytona").every(execution => + execution.profile.id === RELIABILITY_PLAN_PROFILE_ID && reliabilityPlanCells[execution.task.id as keyof typeof reliabilityPlanCells].environment === "local")).toBe(true); }); it("is excluded from --all, the plain suite, generic selectors and scheduled campaigns", () => { @@ -50,25 +52,27 @@ describe("reliability selection stays explicit", () => { it("allows a single case or exact ID and keeps the cells discoverable in --list", () => { expect(selectRunnerExecutions(parseRunnerSelectors(["--suite", "everyday-workflows", "--case", "long-silent-command"])) - .map(execution => execution.profile.id).sort()).toEqual(["runner-codex", ...everydayReliabilityFacadeProfileIds].sort()); + .map(execution => execution.profile.id).sort()).toEqual(["runner-codex", RELIABILITY_PLAN_PROFILE_ID, ...everydayReliabilityFacadeProfileIds].sort()); expect(selectRunnerExecutions(parseRunnerSelectors(["--id", "everyday-workflows.runner-codex.daytona.queue-steer"]))).toHaveLength(1); expect(reliable(selectRunnerExecutions(parseRunnerSelectors(["--list"])))).toHaveLength(reliabilityCellCount); expect(selectRunnerExecutions(parseRunnerSelectors(["--suite", "everyday-workflows", "--group", "slow"])) .every(execution => execution.task.id === "long-silent-command")).toBe(true); }); - it("uses one native Daytona profile per case and facades only for the silent command", () => { + it("uses native Codex on Daytona for every case, DeepSeek except for queue-steer, and facades only for the silent command", () => { const cells = runnerMatrix.filter(execution => isEverydayReliabilityCase(execution.task.id)); expect(cells).toHaveLength(reliabilityCellCount); for (const caseId of everydayReliabilityCaseIds) { const profiles = cells.filter(execution => execution.task.id === caseId).map(execution => execution.profile.id); expect(profiles).toContain("runner-codex"); - expect(profiles.length).toBe(caseId === "long-silent-command" ? 1 + everydayReliabilityFacadeProfileIds.length : 1); + const deepSeek = caseId === "queue-steer" ? 0 : 1; + expect(profiles.filter(profile => profile === RELIABILITY_PLAN_PROFILE_ID)).toHaveLength(deepSeek); + expect(profiles.length).toBe(1 + deepSeek + (caseId === "long-silent-command" ? everydayReliabilityFacadeProfileIds.length : 0)); } expect(runnerMatrix.filter(execution => execution.suite.id === "everyday-workflows" && ["runner-acpx-grok", "runner-acpx-kimi", "runner-acpx-pi"].includes(execution.profile.id) && execution.task.id !== "long-silent-command")).toEqual([]); - const turnThree = cells.find(execution => execution.task.id === "three-turn-revise")!; + const turnThree = cells.find(execution => execution.id === "everyday-workflows.runner-codex.daytona.three-turn-revise")!; expect(turnThree.environment.configurationKey).toBeUndefined(); // Per-turn ephemeral lease, as in S49. expect(cells.find(execution => execution.task.id === "long-silent-command")!.groups).toContain("slow"); }); diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs index 92da5762da8..a14e23e457f 100644 --- a/tests/runner-e2e/product-regression-testops-cli.mjs +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -5,7 +5,7 @@ import { appendFile, lstat, mkdir, readFile, writeFile } from 'node:fs/promises' import path from 'node:path'; import { pathToFileURL } from 'node:url'; import { testopsClient } from './chat-feature-testops-api.mjs'; -import { endpoint, workflowUrls, selectProductRegression, validateSelection, projectProductResults } from './product-regression-testops.mjs'; +import { endpoint, workflowUrls, selectProductRegression, validateSelection, projectProductResults, testOpsPlans, testOpsPlanDetails } from './product-regression-testops.mjs'; const execute = promisify(execFile); const hash = bytes => createHash('sha256').update(bytes).digest('hex'); @@ -69,14 +69,18 @@ export function validateUploadedProductResults(details, selection, summary) { } export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch, - wait = ms => new Promise(resolve => setTimeout(resolve, ms))) { + wait = ms => new Promise(resolve => setTimeout(resolve, ms)), plans = testOpsPlans()) { admitProductTestOps(env); if (command === 'select') { const requestedJob = env.ALLURE_JOB_RUN_ID || ''; if (requestedJob && !positive(requestedJob)) fail('job_run_invalid'); - // Reject a bad bootstrap selector before opening any run. Before adopting - // a TestOps dispatch, independently check its pending job and project. - const manualCases = requestedJob ? undefined : selectProductRegression(undefined, env.TESTOPS_CASE || 'all'); + // The bootstrap plan applies only to a manual bootstrap. A TestOps dispatch + // names its plan through the case identities in its own selection. + const planKey = env.TESTOPS_PLAN || 'product-regression'; + if (!Object.hasOwn(testOpsPlanDetails, planKey)) fail('plan_invalid'); + // Reject a bad or unregistered bootstrap selector before opening any run. + // Before adopting a TestOps dispatch, independently check its pending job and project. + const manualCases = requestedJob ? undefined : selectProductRegression(undefined, env.TESTOPS_CASE || 'all', false, planKey, plans); const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); if (requestedJob) { const pending = await api(`/api/jobrun/${requestedJob}`); @@ -102,11 +106,11 @@ export async function productTestOps(command, argument, env = process.env, exec { env: boundEnv, timeout: 30_000, maxBuffer: 32_768 }); plan = await readJson(planFile, 65_536); } - const cases = manualCases ?? selectProductRegression(plan, env.TESTOPS_CASE || 'all', true); + const cases = manualCases ?? selectProductRegression(plan, env.TESTOPS_CASE || 'all', true, planKey, plans); const selection = { schema: 'paperclip.product-regression-testops-selection/v1', runId: env.GITHUB_RUN_ID, attempt: env.GITHUB_RUN_ATTEMPT, workflowSha: env.GITHUB_SHA, targetSha: env.SOURCE_SHA, ...context, ...job, cases }; - validateSelection(selection, env); + validateSelection(selection, env, plans); const bytes = Buffer.from(JSON.stringify(selection, null, 2)); await mkdir(path.dirname(argument), { recursive: true }); await writeFile(argument, bytes, { flag: 'wx', mode: 0o600 }); @@ -116,7 +120,7 @@ export async function productTestOps(command, argument, env = process.env, exec if (!['project', 'verify'].includes(command)) fail('command_invalid'); const bytes = await readFile(argument); if (bytes.length > 65_536 || hash(bytes) !== env.SELECTION_SHA256) fail('selection_hash'); - const selection = validateSelection(JSON.parse(bytes), env); + const selection = validateSelection(JSON.parse(bytes), env, plans); if (env.ALLURE_JOB_RUN_ID !== selection.jobRunId) fail('job_run_mismatch'); if (command === 'project') { let campaign; @@ -126,7 +130,8 @@ export async function productTestOps(command, argument, env = process.env, exec await mkdir(env.ALLURE_RESULTS, { recursive: false, mode: 0o700 }); for (const row of records) await writeFile(path.join(env.ALLURE_RESULTS, row.file), JSON.stringify(row.value), { flag: 'wx', mode: 0o600 }); await writeFile(env.EXPORT_SUMMARY, JSON.stringify(summary), { flag: 'wx', mode: 0o600 }); - await appendFile(env.GITHUB_STEP_SUMMARY, `### [Product regression TestOps results](${endpoint}/launch/${selection.launchId})\n\n${summary.passed}/${summary.selected} passed; ${summary.failed} failed; ${summary.broken} broken. ${summary.fullPlan ? 'Complete 20-case plan selection.' : 'Targeted selection; not a full-plan verdict.'}\n`); + const details = testOpsPlanDetails[summary.plan]; + await appendFile(env.GITHUB_STEP_SUMMARY, `### [${details.label} TestOps results](${endpoint}/launch/${selection.launchId})\n\n${summary.passed}/${summary.selected} passed; ${summary.failed} failed; ${summary.broken} broken. ${summary.fullPlan ? `Complete ${details.size}-case plan selection.` : 'Targeted selection; not a full-plan verdict.'}\n`); return 0; } const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); diff --git a/tests/runner-e2e/product-regression-testops.d.mts b/tests/runner-e2e/product-regression-testops.d.mts index 4a1fd9eb3e4..a721e21038e 100644 --- a/tests/runner-e2e/product-regression-testops.d.mts +++ b/tests/runner-e2e/product-regression-testops.d.mts @@ -1,3 +1,5 @@ +export type TestOpsPlanKey = 'product-regression' | 'reliability'; + export interface ProductRegressionCase { id: number; suite: string; @@ -6,14 +8,26 @@ export interface ProductRegressionCase { model: string; fullName: string; executionId: string; + plan: TestOpsPlanKey; + environment: 'local' | 'daytona'; + signature?: string; +} + +export interface ReliabilityRegistration { + planId: number | null; + caseIds: Readonly>; } export const productRegressionCases: readonly ProductRegressionCase[]; export const productReportingModels: readonly string[]; export const productReportingMatcherKinds: readonly string[]; +export const testOpsPlanDetails: Readonly>; +export function testOpsPlans(reliabilityRegistration?: ReliabilityRegistration): + Readonly>; export function projectProductResults(campaign: unknown, selection: unknown, sourceSha: string): { - summary: { selected: number; passed: number; failed: number; broken: number }; + summary: { selected: number; passed: number; failed: number; broken: number; plan: TestOpsPlanKey; fullPlan: boolean }; records: Array<{ file: string; value: { model?: string | null; declaredModel?: string; reportedModelSha256?: string | null; modelSource?: 'first_task_configuration' | 'execution_profile' | 'not_recorded'; + environment?: string; labels?: Array<{ name: string; value: string }>; checks?: Array<{ name: string; ordinal: number; passed: boolean; pathSha256: string | null }> } }>; }; diff --git a/tests/runner-e2e/product-regression-testops.mjs b/tests/runner-e2e/product-regression-testops.mjs index dab6baa6d1b..fe16dfd15a7 100644 --- a/tests/runner-e2e/product-regression-testops.mjs +++ b/tests/runner-e2e/product-regression-testops.mjs @@ -1,4 +1,5 @@ import { createHash, randomUUID } from 'node:crypto'; +import { registeredReliabilityCases, reliabilityDefinitions, reliabilityModel, reliabilityPlanKey, reliabilityPlanName } from './reliability-testops.mjs'; // These are the standing, reviewed TestOps plan-2 identities. A case selects one // existing full-stack cell; it does not change the prompt, matcher or runtime. @@ -29,13 +30,32 @@ export const productRegressionCases = [ : suite === 'configured-repository' ? 'gpt-6.1-sol' : 'gpt-5.6-sol', fullName: `paperclip.Product E2E.${suite}.${name}`, executionId: `${suite}.${profile}.local.${name}`, + plan: 'product-regression', environment: 'local', })); +// Sibling TestOps plans share this trusted selection, projection and upload path. +// Each keeps its own denominator: a launch selects cases from exactly one plan, and +// only a complete selection of that plan is a full-plan verdict. Plan 2 stays at +// its 20 standing cases. +export const testOpsPlanDetails = Object.freeze({ + 'product-regression': Object.freeze({ label: 'Product regression', name: 'Product regression — core workflows', + size: productRegressionCases.length }), + [reliabilityPlanKey]: Object.freeze({ label: 'Reliability', name: reliabilityPlanName, + size: reliabilityDefinitions.length }), +}); + +/** Plan key to its bound cases; an unregistered plan maps to null and cannot be selected. */ +export function testOpsPlans(reliabilityRegistration) { + // An omitted registration uses the reviewed identities in reliability-testops.mjs. + const reliability = registeredReliabilityCases(reliabilityRegistration, productRegressionCases.map(row => row.id)); + return Object.freeze({ 'product-regression': productRegressionCases, [reliabilityPlanKey]: reliability }); +} + const fail = code => { throw new Error(`product_regression_testops_${code}`); }; const sha = value => createHash('sha256').update(value).digest('hex'); -// Closed transmission contracts, checked against all 20 catalog cells. Dynamic +// Closed transmission contracts, checked against every plan's catalog cells. Dynamic // matcher paths stay in the linked source artifact; only their digest leaves it. -export const productReportingModels = [...new Set(productRegressionCases.map(row => row.model))]; +export const productReportingModels = [...new Set([...productRegressionCases.map(row => row.model), reliabilityModel])]; export const productReportingMatcherKinds = ['message_exact', 'message_contains', 'message_occurrences', 'message_regex', 'message_ordered', 'issue_status', 'run_status', 'runtime_mode', 'environment', 'file_exists', 'file_exact', 'file_contains', 'artifact_exists', 'artifact_exact', 'json_path', 'json_schema']; @@ -50,38 +70,48 @@ export const workflowUrls = new Set([workflowUrl, 'https://github.com/paperclipai/paperclip/actions/workflows/343807492', ]); -export function selectProductRegression(plan, manual = 'all', fromTestOps = false) { +export function selectProductRegression(plan, manual = 'all', fromTestOps = false, planKey = 'product-regression', plans = testOpsPlans()) { if (fromTestOps && !plan) fail('plan_required'); + const known = Object.values(plans).flatMap(cases => cases ?? []); if (plan) { - if (plan.version !== '1.0' || !Array.isArray(plan.tests) || !plan.tests.length || plan.tests.length > productRegressionCases.length) fail('plan_invalid'); + // A TestOps selection names its own plan through its case identities. + if (plan.version !== '1.0' || !Array.isArray(plan.tests) || !plan.tests.length || plan.tests.length > known.length) fail('plan_invalid'); const selected = plan.tests.map(test => { - const row = productRegressionCases.find(value => String(value.id) === String(test.id)); + const row = known.find(value => String(value.id) === String(test.id)); if (!row || test.selector !== row.fullName) fail('case_mismatch'); return row; }); + if (new Set(selected.map(row => row.plan)).size !== 1) fail('plan_mixed'); + if (selected.length > plans[selected[0].plan].length) fail('plan_invalid'); if (new Set(selected.map(row => row.id)).size !== selected.length) fail('duplicate_case'); return selected.sort((a, b) => a.id - b.id); } - if (manual === 'all') return [...productRegressionCases]; - const selected = productRegressionCases.find(row => row.name === manual || String(row.id) === manual); + if (!Object.hasOwn(plans, planKey)) fail('plan_invalid'); + const cases = plans[planKey]; + if (!cases) fail(`${planKey}_unregistered`); + if (manual === 'all') return [...cases]; + const selected = cases.find(row => row.name === manual || String(row.id) === manual); if (!selected) fail('manual_selection_invalid'); return [selected]; } -export function validateSelection(value, env) { +export function validateSelection(value, env, plans = testOpsPlans()) { if (value?.schema !== 'paperclip.product-regression-testops-selection/v1' || value.runId !== env.GITHUB_RUN_ID || value.attempt !== env.GITHUB_RUN_ATTEMPT || value.workflowSha !== env.GITHUB_SHA || !/^[a-f0-9]{40}$/.test(value.workflowSha) || value.targetSha !== env.SOURCE_SHA || !/^[a-f0-9]{40}$/.test(value.targetSha) || !/^[1-9][0-9]{0,15}$/.test(value.jobRunId) || !Array.isArray(value.cases) - || !value.cases.length || value.cases.length > productRegressionCases.length) fail('selection_binding'); - const selected = selectProductRegression({ version: '1.0', tests: value.cases.map(row => ({ id: row.id, selector: row.fullName })) }, 'all', true); + || !value.cases.length || value.cases.length > Object.values(plans).flatMap(cases => cases ?? []).length) fail('selection_binding'); + const selected = selectProductRegression({ version: '1.0', tests: value.cases.map(row => ({ id: row.id, selector: row.fullName })) }, 'all', true, undefined, plans); if (JSON.stringify(selected) !== JSON.stringify(value.cases)) fail('selection_binding'); return value; } export function projectProductResults(campaign, selection, sourceSha) { if (!/^[a-f0-9]{40}$/.test(sourceSha)) fail('source_missing'); + const planKey = selection.cases?.[0]?.plan; + if (!Object.hasOwn(testOpsPlanDetails, planKey) || selection.cases.some(row => row.plan !== planKey + || !['local', 'daytona'].includes(row.environment))) fail('selection_binding'); if (campaign && (campaign.schema !== 'paperclip.runner-e2e.campaign/v2' || campaign.campaignId !== `gha-${selection.runId}-${selection.attempt}` || campaign.source?.sha !== sourceSha || !Array.isArray(campaign.results) || !Array.isArray(campaign.expected) @@ -109,7 +139,7 @@ export function projectProductResults(campaign, selection, sourceSha) { if (result && ((!missingArtifact && result.source?.sha !== sourceSha) || (missingArtifact && result.source?.sha != null && result.source.sha !== sourceSha) || result.suiteId !== selected.suite || result.caseId !== selected.name - || result.profileId !== selected.profile || result.environmentId !== 'local' + || result.profileId !== selected.profile || result.environmentId !== selected.environment || !Number.isSafeInteger(result.attempt) || result.attempt < (missingArtifact ? 0 : 1) || result.attempt > 10 || !['passed', 'failed'].includes(result.status) || (!missingArtifact && !Array.isArray(result.matcherResults)))) fail('result_binding'); const checks = result?.matcherResults ?? []; @@ -130,7 +160,7 @@ export function projectProductResults(campaign, selection, sourceSha) { executionId: selected.executionId, workflowRunId: selection.runId, workflowAttempt: selection.attempt, workflowSha: selection.workflowSha, sourceSha, status, originalStatus: result?.status ?? null, attempt: result?.attempt ?? null, cleanup: result?.cleanup ?? 'not_started', - profile: selected.profile, environment: 'local', + profile: selected.profile, environment: selected.environment, declaredModel: selected.model, model: configuredModel, modelSource: configuredModel == null ? 'not_recorded' : firstTask ? 'first_task_configuration' : 'execution_profile', reportedModelSha256: firstTask && typeof result?.model === 'string' ? sha(result.model) : null, @@ -156,11 +186,12 @@ export function projectProductResults(campaign, selection, sourceSha) { const uuid = randomUUID(), attachment = `${randomUUID()}-attachment.json`; const value = { uuid, name: selected.name, fullName: selected.fullName, testCaseId: sha(selected.fullName), - historyId: sha(JSON.stringify([selected.fullName, selected.profile, provenance.model, 'local'])), + historyId: sha(JSON.stringify([selected.fullName, selected.profile, provenance.model, selected.environment])), status, stage: 'finished', statusDetails: { message: provenance.note }, labels: [{ name: 'AS_ID', value: String(selected.id) }, { name: 'parentSuite', value: 'Product E2E' }, - { name: 'suite', value: selected.suite }, { name: 'framework', value: 'Paperclip Evals' }], - parameters: [{ name: 'profile', value: selected.profile }, { name: 'environment', value: 'local / GitHub Actions' }, + { name: 'suite', value: selected.suite }, { name: 'framework', value: 'Paperclip Evals' }, + ...(selected.signature ? [{ name: 'tag', value: `soak-${selected.signature}` }] : [])], + parameters: [{ name: 'profile', value: selected.profile }, { name: 'environment', value: `${selected.environment} / GitHub Actions` }, { name: 'declaredModel', value: selected.model }, { name: 'configuredModel', value: provenance.model ?? 'Not recorded' }, { name: 'sourceSha', value: sourceSha }], links: [{ name: 'Full-stack execution and retained diagnostics', type: 'custom', @@ -179,5 +210,5 @@ export function projectProductResults(campaign, selection, sourceSha) { } return { records, summary: { selected: selection.cases.length, passed: summaries.filter(row => row.status === 'passed').length, failed: summaries.filter(row => row.status === 'failed').length, broken: summaries.filter(row => row.status === 'broken').length, - fullPlan: selection.cases.length === productRegressionCases.length, cases: summaries } }; + plan: planKey, fullPlan: selection.cases.length === testOpsPlanDetails[planKey].size, cases: summaries } }; } diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index 9ad90133089..03368ba39b1 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -81,7 +81,7 @@ test('reported pass requires real assertions, successful cleanup and exact curre test('missing results remain explicit broken cases and cannot improve the denominator', () => { const projected = projectProductResults(undefined, selection(productRegressionCases), targetSha); const { cases, ...counts } = projected.summary; - assert.deepEqual(counts, { selected: 20, passed: 0, failed: 0, broken: 20, fullPlan: true }); + assert.deepEqual(counts, { selected: 20, passed: 0, failed: 0, broken: 20, plan: 'product-regression', fullPlan: true }); assert.equal(cases.length, 20); assert.equal(cases.every(row => row.status === 'broken'), true); assert.equal(projected.records.filter(row => row.file.endsWith('-result.json')).length, 20); diff --git a/tests/runner-e2e/reliability-testops-catalog.test.ts b/tests/runner-e2e/reliability-testops-catalog.test.ts new file mode 100644 index 00000000000..7eec6f1883b --- /dev/null +++ b/tests/runner-e2e/reliability-testops-catalog.test.ts @@ -0,0 +1,101 @@ +import { expect, it } from "vitest"; +import { buildMatrixJobs, parseRunnerSelectors, selectRunnerExecutions } from "./selectors.js"; +import { runnerMatrix } from "./catalog.js"; +import { scheduledSelection } from "./scheduled-operations.js"; +import { RELIABILITY_PLAN_PROFILE_ID, everydayReliabilityCaseIds, reliabilityPlanCells, reliabilityPlanExecutionIds, + everydayReliabilitySignatures } from "./everyday-reliability-cases.js"; +import { productReportingMatcherKinds, productReportingModels, projectProductResults, testOpsPlans } from "./product-regression-testops.mjs"; +import { reliabilityDefinitions, reliabilityModel, reliabilityProfile } from "./reliability-testops.mjs"; +import { gradeLongSilentCommand, gradeQueuedAcrossRestart, gradeResumptionRule } from "./everyday-reliability.js"; +import type { RunnerE2EResult } from "./types.js"; + +it("binds every TestOps Reliability case to its existing explicit-only cell", () => { + expect(reliabilityProfile).toBe(RELIABILITY_PLAN_PROFILE_ID); + expect(reliabilityDefinitions.map(row => row.name)).toEqual(everydayReliabilityCaseIds); + expect(reliabilityDefinitions.map(row => row.executionId)).toEqual(reliabilityPlanExecutionIds("everyday-workflows")); + const executions = selectRunnerExecutions(parseRunnerSelectors(reliabilityDefinitions.flatMap(row => ["--id", row.executionId]))); + expect(executions).toHaveLength(7); + for (const row of reliabilityDefinitions) { + const execution = executions.find(value => value.id === row.executionId)!; + expect(execution.suite.id).toBe(row.suite); + expect(execution.task.id).toBe(row.name); + const cell = reliabilityPlanCells[row.name as keyof typeof reliabilityPlanCells]; + expect(execution.profile.id).toBe(row.profile); + expect(row.profile).toBe(cell.profile); + expect(execution.profile.model).toBe(row.model); + expect(productReportingModels).toContain(execution.profile.model); + expect(execution.environment.id).toBe(row.environment); + expect(row.environment).toBe(cell.environment); + // Six cells use DeepSeek 1017 on OpenCode; queue-steer needs a steering-capable driver (Codex app server). + if (row.name === "queue-steer") expect([execution.profile.provider, execution.profile.id]).toEqual(["codex", "runner-codex"]); + else expect([execution.profile.provider, execution.profile.model]).toEqual(["opencode", reliabilityModel]); + expect(row.signature).toBe(everydayReliabilitySignatures[row.name as keyof typeof everydayReliabilitySignatures].signature); + // OpenRouter (or the OpenAI key for the Codex steering cell), plus Daytona only for sandbox cells. + const model = row.profile === reliabilityProfile ? "OPENROUTER_API_KEY" : "OPENAI_API_KEY"; + expect(execution.requiredCredentials).toEqual(row.environment === "daytona" ? [model, "DAYTONA_API_KEY"] : [model]); + expect(execution.task.automaticRetryPolicy).toBe("single_attempt"); + } + // S49 ran on a per-turn lease; the other Daytona cells keep the suite's warm lease. + expect(executions.find(value => value.task.id === "three-turn-revise")!.environment.configurationKey).toBeUndefined(); + expect(executions.find(value => value.task.id === "long-silent-command")!.environment.configurationKey).toBe("warm-reuse-v1"); + for (const job of buildMatrixJobs(executions)) expect(job.timeoutMinutes).toBeGreaterThan(20); +}); + +it("keeps a case that steers an active turn on a steering-capable driver", async () => { + // queue-steer requires a 2xx same-turn steer. The OpenCode server driver + // declares steering false, so the route answers steering_unsupported there. + const { readFile } = await import("node:fs/promises"); + const openCode = await readFile(new URL("../../packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts", import.meta.url), "utf8"); + const codex = await readFile(new URL("../../packages/paperclip-runner/src/drivers/codex/codex-app-server-driver-impl.ts", import.meta.url), "utf8"); + expect(openCode).toMatch(/\n {2}steering: false,\n/); + expect(codex).toMatch(/\n {6}steering: true,\n/); + const steering = runnerMatrix.find(execution => execution.id === reliabilityDefinitions.find(row => row.name === "queue-steer")!.executionId)!; + expect(steering.profile.provider).toBe("codex"); + expect(runnerMatrix.some(execution => execution.task.id === "queue-steer" && execution.profile.provider === "opencode")).toBe(false); +}); + +it("keeps the plan out of generic, scheduled and per-PR selections and adds no DeepSeek everyday cells", () => { + const plan = new Set(reliabilityDefinitions.map(row => row.executionId)); + const select = (args: string[]) => { + try { return selectRunnerExecutions(parseRunnerSelectors(args)); } + catch (error) { + if (error instanceof Error && error.message === "Runner E2E selectors matched zero executions") return []; + throw error; + } + }; + for (const args of [["--all"], ["--suite", "everyday-workflows"], ["--profile", RELIABILITY_PLAN_PROFILE_ID], ["--environment", "daytona"]]) { + expect(select(args).filter(execution => plan.has(execution.id)), args.join(" ")).toEqual([]); + } + for (const cadence of ["daily", "weekly"] as const) expect(scheduledSelection(cadence).filter(id => plan.has(id))).toEqual([]); + expect(select(["--suite", "everyday-workflows", "--profile", "runner-codex"]).filter(execution => plan.has(execution.id))).toEqual([]); + expect(runnerMatrix.filter(execution => execution.profile.id === RELIABILITY_PLAN_PROFILE_ID && execution.suite.id === "everyday-workflows") + .map(execution => execution.id).sort()).toEqual([...plan].filter(id => id.includes(RELIABILITY_PLAN_PROFILE_ID)).sort()); + expect([...plan].filter(id => !id.includes(RELIABILITY_PLAN_PROFILE_ID))).toEqual(["everyday-workflows.runner-codex.daytona.queue-steer"]); + // The chat-feature candidate cell is unchanged and stays local. + expect(runnerMatrix.filter(execution => execution.profile.id === RELIABILITY_PLAN_PROFILE_ID && execution.suite.id !== "everyday-workflows") + .map(execution => execution.id)).toEqual(["chat-features.runner-opencode-deepseek-1017.local.create-backlog"]); +}); + +it("reports the real reliability grader output through the standing projection without dropping failures", () => { + const cases = testOpsPlans({ planId: 9001, caseIds: Object.fromEntries(reliabilityDefinitions.map((row, index) => [row.name, 9101 + index])) }).reliability!; + const checks = [ + ...gradeResumptionRule({ observations: [], retryClicks: [] }), + ...gradeLongSilentCommand({ iterations: 13, iterationSeconds: 60, stopSamples: [], nonce: "control" }), + ...gradeQueuedAcrossRestart({ runs: [], commentId: "comment", restartStartedAt: 0, dispatchedBeforeRestart: false, waitObservations: [] }), + ]; + expect(checks.some(check => !check.passed)).toBe(true); + for (const row of cases) { + const matcherResults = checks.map(check => ({ matcher: { kind: "json_path" as const, path: check.id, expected: true }, passed: check.passed, detail: "contract control" })); + for (const result of matcherResults) expect(productReportingMatcherKinds).toContain(result.matcher.kind); + const sourceSha = "b".repeat(40); + const raw: RunnerE2EResult = { schema: "paperclip.runner-e2e.result/v2", executionId: row.executionId, suiteId: row.suite, caseId: row.name, + profileId: row.profile, environmentId: row.environment, source: { sha: sourceSha, ref: null, workflowRunUrl: null }, + attempt: 1, status: "failed", failureClass: "candidate_failure", cleanup: "passed", model: row.model, provider: "opencode", + runtimeMode: "native", startedAt: "2026-11-06T00:00:00Z", finishedAt: "2026-11-06T00:00:01Z", durationMs: 1000, matcherResults }; + const projected = projectProductResults({ schema: "paperclip.runner-e2e.campaign/v2", campaignId: "gha-123-1", source: { sha: sourceSha }, + expected: [row.executionId], results: [raw] }, { runId: "123", attempt: "1", workflowSha: "a".repeat(40), cases: [row] }, sourceSha); + expect(projected.summary).toMatchObject({ selected: 1, passed: 0, failed: 1, plan: "reliability", fullPlan: false }); + expect(projected.records[1]!.value.checks?.map(check => check.passed)).toEqual(matcherResults.map(check => check.passed)); + expect(projected.records[1]!.value.environment).toBe(row.environment); + } +}); diff --git a/tests/runner-e2e/reliability-testops-registration.mjs b/tests/runner-e2e/reliability-testops-registration.mjs new file mode 100644 index 00000000000..c9da1096cec --- /dev/null +++ b/tests/runner-e2e/reliability-testops-registration.mjs @@ -0,0 +1,197 @@ +import { appendFile, mkdir, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { testopsClient } from './chat-feature-testops-api.mjs'; +import { endpoint, productRegressionCases, workflowUrls } from './product-regression-testops.mjs'; +import { reliabilityDefinitions, reliabilityPlanName, reliabilityRegistration } from './reliability-testops.mjs'; + +// One-shot, idempotent registration of the Reliability plan in TestOps. It runs +// only in the trusted registration workflow, which alone receives ALLURE_TOKEN. +// `inspect` only reads. `register` reuses existing cases by full name, creates +// any missing case and the plan, and assigns the plan to the same GitHub job +// that launches the standing Product regression plan. TestOps assigns every ID; +// the receipt carries them into one reviewed change to reliability-testops.mjs. + +const fail = code => { throw new Error(`reliability_testops_registration_${code}`); }; +const positive = value => Number.isSafeInteger(value) && value > 0; +const same = (left, right) => JSON.stringify(left) === JSON.stringify(right); +const sorted = values => [...values].sort((a, b) => a - b); +export const registrationWorkflowRef = 'paperclipai/paperclip/.github/workflows/runner-e2e-testops-registration.yml@refs/heads/master'; +export const standingPlanId = 2; +const maximumCatalogPages = 30; + +export function admitRegistration(env) { + if (env.GITHUB_REPOSITORY !== 'paperclipai/paperclip' || env.GITHUB_REF !== 'refs/heads/master' + || env.GITHUB_EVENT_NAME !== 'workflow_dispatch' || env.GITHUB_WORKFLOW_REF !== registrationWorkflowRef + || !/^[a-f0-9]{40}$/.test(env.GITHUB_SHA ?? '') || !/^[1-9][0-9]{0,15}$/.test(env.GITHUB_RUN_ID ?? '') + || env.ALLURE_ENDPOINT !== endpoint || env.ALLURE_PROJECT_ID !== '1' || env.REGISTRATION_PLAN !== 'reliability' + || !['inspect', 'register'].includes(env.REGISTRATION_OPERATION)) fail('untrusted_context'); + return env.REGISTRATION_OPERATION; +} + +export function desiredCase(row) { + return { + name: `Reliability · ${row.name} (${row.signature})`, + automated: true, + description: [ + `Staging-soak reliability case ${row.name} (soak signature ${row.signature}).`, + `Runs the existing full-stack cell ${row.executionId}: profile ${row.profile} with ${row.model} on ${row.environment}.`, + 'Launched by the Reliability TestOps plan through runner-full-stack-e2e.yml. Registration is not a result or a qualification claim.', + ].join('\n\n'), + }; +} + +const page = (value, number) => { + if (!Array.isArray(value?.content) || value.content.length > 100 || value.number !== number + || !Number.isSafeInteger(value.totalPages) || value.totalPages < 0 || value.totalPages > maximumCatalogPages + || !Number.isSafeInteger(value.totalElements) || value.totalElements < 0) fail('catalog_page'); + return value; +}; + +/** Read-only state: matching cases by full name, the plan by name, and the standing plan's job. */ +export async function inspectReliability(api) { + const wanted = new Map(reliabilityDefinitions.map(row => [row.fullName, row])); + const found = new Map(), seen = new Set(); + let total; + for (let number = 0; number < maximumCatalogPages; number++) { + const listed = page(await api(`/api/testcase?${new URLSearchParams({ projectId: '1', page: String(number), size: '100', sort: 'id,asc' })}`), number); + if (total !== undefined && total !== listed.totalElements) fail('catalog_changed'); + total = listed.totalElements; + const ids = listed.content.map(row => { + if (!positive(row?.id) || seen.has(row.id)) fail('catalog_identity'); + seen.add(row.id); return row.id; + }); + for (let offset = 0; offset < ids.length; offset += 8) { + const batch = ids.slice(offset, offset + 8); + const details = await Promise.all(batch.map(id => api(`/api/testcase/${id}`))); + for (const [index, actual] of details.entries()) { + if (actual?.id !== batch[index] || actual.projectId !== 1) fail('catalog_case'); + const row = wanted.get(actual.fullName); + if (!row || actual.deleted === true) continue; + if (found.has(row.name)) fail('case_conflict'); + found.set(row.name, actual.id); + } + } + if (number + 1 >= listed.totalPages) break; + if (number + 1 === maximumCatalogPages) fail('catalog_too_large'); + } + if (seen.size !== total) fail('catalog_incomplete'); + if ([...found.values()].some(id => productRegressionCases.some(row => row.id === id))) fail('case_conflict'); + + const plans = await api(`/api/testplan?${new URLSearchParams({ projectId: '1', name: reliabilityPlanName, page: '0', size: '100' })}`); + if (!Array.isArray(plans?.content) || plans.totalPages > 1) fail('plan_page'); + const named = plans.content.filter(row => row.name === reliabilityPlanName); + if (named.length > 1) fail('plan_conflict'); + let plan = null; + if (named.length) { + plan = await api(`/api/testplan/${named[0].id}`); + if (!positive(plan?.id) || plan.id !== named[0].id || plan.id === standingPlanId || plan.projectId !== 1 + || plan.name !== reliabilityPlanName) fail('plan_identity'); + } + const standing = await api(`/api/testplan/${standingPlanId}/job`); + if (!Array.isArray(standing)) fail('standing_job'); + const candidates = standing.filter(row => row?.job?.type === 'github' && workflowUrls.has(row.job.url)); + if (candidates.length !== 1 || !positive(candidates[0].job.id)) fail('standing_job'); + const job = await api(`/api/job/${candidates[0].job.id}`); + if (job?.id !== candidates[0].job.id || job.type !== 'github' || !workflowUrls.has(job.url)) fail('standing_job'); + const assignments = plan ? await api(`/api/testplan/${plan.id}/job`) : []; + if (!Array.isArray(assignments)) fail('plan_job'); + return { caseIds: Object.fromEntries(found), plan, jobId: job.id, + standingParameters: Array.isArray(candidates[0].job.parameters) ? candidates[0].job.parameters : [], assignments }; +} + +function assertPlan(plan, ids) { + const selection = plan?.treeSelection; + if (!positive(plan?.id) || plan.projectId !== 1 || plan.name !== reliabilityPlanName || !selection || selection.inverted === true + || !Array.isArray(selection.leafsInclude) || !same(sorted(selection.leafsInclude), sorted(ids)) + || ['groupsInclude', 'leafsExclude'].some(key => (selection[key]?.length ?? 0) > 0)) fail('plan_scope'); +} + +export async function registerReliability(api) { + const before = await inspectReliability(api); + const created = { cases: [], plan: false, assignment: false, parameters: false }; + const caseIds = {}; + for (const row of reliabilityDefinitions) { + if (before.caseIds[row.name]) { caseIds[row.name] = before.caseIds[row.name]; continue; } + const response = await api('/api/testcase', 'POST', { projectId: 1, fullName: row.fullName, ...desiredCase(row) }); + if (!positive(response?.id)) fail('created_case'); + const actual = await api(`/api/testcase/${response.id}`); + if (actual?.id !== response.id || actual.projectId !== 1 || actual.fullName !== row.fullName || actual.automated !== true) fail('created_case'); + caseIds[row.name] = actual.id; + created.cases.push(row.name); + } + const ids = reliabilityDefinitions.map(row => caseIds[row.name]); + if (new Set(ids).size !== ids.length) fail('case_conflict'); + const treeSelection = { inverted: false, leafsInclude: ids, groupsInclude: [], groupsExclude: [], leafsExclude: [] }; + let plan = before.plan; + if (!plan) { + const response = await api('/api/testplan', 'POST', { projectId: 1, name: reliabilityPlanName, treeSelection }); + if (!positive(response?.id) || response.id === standingPlanId) fail('created_plan'); + plan = await api(`/api/testplan/${response.id}`); + created.plan = true; + } else if (!same(sorted(plan.treeSelection?.leafsInclude ?? []), sorted(ids))) { + await api(`/api/testplan/${plan.id}`, 'PATCH', { treeSelection }); + plan = await api(`/api/testplan/${plan.id}`); + } + assertPlan(plan, ids); + let assignments = await api(`/api/testplan/${plan.id}/job`); + if (!Array.isArray(assignments) || assignments.length > 1 || assignments.some(row => row?.job?.id !== before.jobId)) fail('plan_job_conflict'); + if (!assignments.length) { + await api(`/api/testplan/${plan.id}/assign`, 'POST', { jobId: before.jobId, selection: { inverted: false, manual: false, leafsInclude: ids } }); + created.assignment = true; + } + // A launch from this plan must dispatch exactly as the standing plan does. + if (before.standingParameters.length && !same(assignments[0]?.job?.parameters ?? [], before.standingParameters)) { + await api(`/api/testplan/${plan.id}/jobparameter`, 'POST', { jobId: before.jobId, parameters: before.standingParameters }); + created.parameters = true; + } + assignments = await api(`/api/testplan/${plan.id}/job`); + if (!Array.isArray(assignments) || assignments.length !== 1 || assignments[0]?.job?.id !== before.jobId + || assignments[0].testCasesCount !== ids.length + || (before.standingParameters.length && !same(assignments[0].job.parameters ?? [], before.standingParameters))) fail('plan_job'); + return { caseIds, plan, jobId: before.jobId, created }; +} + +export function registrationReceipt(operation, state, env) { + const caseIds = Object.fromEntries(reliabilityDefinitions.filter(row => state.caseIds[row.name]).map(row => [row.name, state.caseIds[row.name]])); + const complete = Object.keys(caseIds).length === reliabilityDefinitions.length && positive(state.plan?.id) + && (state.assignments ?? [{ job: { id: state.jobId } }]).some(row => row?.job?.id === state.jobId); + const registration = { planId: state.plan?.id ?? null, caseIds }; + return { + schema: 'paperclip.reliability-testops-registration/v1', operation, + workflowRunId: env.GITHUB_RUN_ID, workflowSha: env.GITHUB_SHA, + planName: reliabilityPlanName, planUrl: state.plan ? `${endpoint}/testplan/${state.plan.id}` : null, + jobId: state.jobId, registration, complete, + missing: reliabilityDefinitions.filter(row => !caseIds[row.name]).map(row => row.name), + committed: same(registration, { planId: reliabilityRegistration.planId, caseIds: { ...reliabilityRegistration.caseIds } }), + created: state.created ?? null, + qualification: 'registration_only', actualResult: null, + }; +} + +export function registrationSnippet(receipt) { + const lines = reliabilityDefinitions.map(row => ` '${row.name}': ${receipt.registration.caseIds[row.name]},`); + return ['export const reliabilityRegistration = Object.freeze({', ` planId: ${receipt.registration.planId},`, + ' caseIds: Object.freeze({', ...lines, ' }),', '});'].join('\n'); +} + +export async function reliabilityRegistrationCli(output, env = process.env, fetchImpl = fetch) { + const operation = admitRegistration(env); + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + const state = operation === 'register' ? await registerReliability(api) : await inspectReliability(api); + const receipt = registrationReceipt(operation, state, env); + await mkdir(path.dirname(output), { recursive: true }); + await writeFile(output, JSON.stringify(receipt, null, 2), { flag: 'wx', mode: 0o600 }); + const body = receipt.complete + ? `${receipt.committed ? 'The committed registration already matches TestOps.' : 'Commit these TestOps identities to `tests/runner-e2e/reliability-testops.mjs` in one reviewed change:'}\n\n\`\`\`js\n${registrationSnippet(receipt)}\n\`\`\`\n` + : `Not registered yet. Missing cases: ${receipt.missing.join(', ') || 'none'}; plan: ${receipt.planUrl ?? 'absent'}. Dispatch again with operation \`register\`.\n`; + if (env.GITHUB_STEP_SUMMARY) await appendFile(env.GITHUB_STEP_SUMMARY, `### Reliability TestOps registration (${operation})\n\n${body}`); + return receipt; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + reliabilityRegistrationCli(process.argv[2]).then(() => { process.exitCode = 0; }).catch(error => { + const safe = /^(?:reliability_testops_registration|testops)_[a-z_]+$/.test(error.message) ? error.message : 'reliability_testops_registration_failed'; + process.stderr.write(`${safe}\n`); process.exitCode = 1; + }); +} diff --git a/tests/runner-e2e/reliability-testops.d.mts b/tests/runner-e2e/reliability-testops.d.mts new file mode 100644 index 00000000000..4da814d1baf --- /dev/null +++ b/tests/runner-e2e/reliability-testops.d.mts @@ -0,0 +1,29 @@ +export interface ReliabilityDefinition { + plan: 'reliability'; + suite: 'everyday-workflows'; + name: string; + environment: 'local' | 'daytona'; + signature: string; + profile: string; + model: string; + fullName: string; + executionId: string; +} + +export interface ReliabilityRegistration { + planId: number | null; + caseIds: Readonly>; +} + +export const reliabilityPlanKey: 'reliability'; +export const reliabilityPlanName: string; +export const reliabilitySuite: 'everyday-workflows'; +export const reliabilityProfile: string; +export const reliabilityModel: string; +export const reliabilityDefinitions: readonly ReliabilityDefinition[]; +export const reliabilityRegistration: Readonly; +export function registeredReliabilityCases( + registration?: ReliabilityRegistration, + reservedCaseIds?: readonly number[], + reservedPlanIds?: readonly number[], +): ReadonlyArray | null; diff --git a/tests/runner-e2e/reliability-testops.mjs b/tests/runner-e2e/reliability-testops.mjs new file mode 100644 index 00000000000..0262cb18fda --- /dev/null +++ b/tests/runner-e2e/reliability-testops.mjs @@ -0,0 +1,61 @@ +// The TestOps "Reliability — staging soak" plan: a sibling of the standing +// Product regression plan (plan 2) on the same trusted workflow, TestOps job, +// selection, projection, upload and verification path. Each case selects one +// existing everyday-workflows reliability cell; it does not change the prompt, +// grader, budget or runtime. Six plan cells run native OpenCode with DeepSeek +// 1017 through OpenRouter. queue-steer must steer an active turn, which the +// OpenCode driver does not support (`steering: false`), so the plan uses the +// existing native Codex Daytona cell for it. The other native Codex and ACPX +// reliability cells stay explicit-only extras outside the plan. + +export const reliabilityPlanKey = 'reliability'; +export const reliabilityPlanName = 'Reliability — staging soak'; +export const reliabilitySuite = 'everyday-workflows'; +export const reliabilityProfile = 'runner-opencode-deepseek-1017'; +export const reliabilityModel = 'openrouter/deepseek/deepseek-v4.1-flash-1017'; +const steeringProfile = ['runner-codex', 'gpt-5.6-sol']; + +// Daytona only where the soak failure was a sandbox lease failure (S49, S52/S52b, +// S55). The catalog test binds this table to everyday-reliability-cases.ts. +export const reliabilityDefinitions = Object.freeze([ + ['interrupt-redirect', 'local', 'S47'], + ['reassign-midrun', 'local', 'S48'], + ['three-turn-revise', 'daytona', 'S49'], + ['long-silent-command', 'daytona', 'S52/S52b'], + ['tree-pause-release', 'local', 'S50/S50b'], + ['queue-steer', 'daytona', 'H2', ...steeringProfile], + ['queued-across-restart', 'daytona', 'S55/U9'], +].map(([name, environment, signature, profile = reliabilityProfile, model = reliabilityModel]) => Object.freeze({ + plan: reliabilityPlanKey, suite: reliabilitySuite, name, environment, signature, profile, model, + fullName: `paperclip.Product E2E.${reliabilitySuite}.${name}`, + executionId: `${reliabilitySuite}.${profile}.${environment}.${name}`, +}))); + +// TestOps assigns these identities; they are never invented here. Fill them, +// in one reviewed change, from the receipt of the trusted +// `runner-e2e-testops-registration.yml` workflow (operation `register`). +// Until every case and the plan have an identity, selecting this plan fails +// closed with `product_regression_testops_reliability_unregistered`. +export const reliabilityRegistration = Object.freeze({ + planId: null, + caseIds: Object.freeze({}), +}); + +const fail = code => { throw new Error(`product_regression_testops_${code}`); }; +const positive = value => Number.isSafeInteger(value) && value > 0; + +/** + * Bind the reviewed TestOps identities to the reliability cells. Returns null + * while nothing is registered; any partial, duplicate, foreign or malformed + * registration fails instead of selecting a subset. + */ +export function registeredReliabilityCases(registration = reliabilityRegistration, reservedCaseIds = [], reservedPlanIds = [2]) { + const ids = registration?.caseIds ?? {}; + if (registration?.planId == null && Object.keys(ids).length === 0) return null; + const names = reliabilityDefinitions.map(row => row.name); + if (!positive(registration.planId) || reservedPlanIds.includes(registration.planId) + || Object.keys(ids).length !== names.length || names.some(name => !positive(ids[name])) + || new Set(names.map(name => ids[name])).size !== names.length + || names.some(name => reservedCaseIds.includes(ids[name]))) fail('reliability_registration_invalid'); + return Object.freeze(reliabilityDefinitions.map(row => Object.freeze({ id: ids[row.name], ...row }))); +} diff --git a/tests/runner-e2e/reliability-testops.test.mjs b/tests/runner-e2e/reliability-testops.test.mjs new file mode 100644 index 00000000000..c10727bd54c --- /dev/null +++ b/tests/runner-e2e/reliability-testops.test.mjs @@ -0,0 +1,326 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { parse } from 'yaml'; +import { productRegressionCases, projectProductResults, selectProductRegression, testOpsPlanDetails, testOpsPlans, + validateSelection, workflowUrl } from './product-regression-testops.mjs'; +import { productTestOps } from './product-regression-testops-cli.mjs'; +import { registeredReliabilityCases, reliabilityDefinitions, reliabilityModel, reliabilityPlanName, reliabilityProfile, + reliabilityRegistration } from './reliability-testops.mjs'; +import { admitRegistration, inspectReliability, registerReliability, registrationReceipt, registrationSnippet, + registrationWorkflowRef } from './reliability-testops-registration.mjs'; + +// Synthetic identities exercise the bound path only. Real IDs come from TestOps. +const synthetic = { planId: 9001, caseIds: Object.fromEntries(reliabilityDefinitions.map((row, index) => [row.name, 9101 + index])) }; +const plans = testOpsPlans(synthetic); +const cases = plans.reliability; +const workflowSha = 'a'.repeat(40), targetSha = 'b'.repeat(40); +const env = { + GITHUB_REPOSITORY: 'paperclipai/paperclip', GITHUB_REF: 'refs/heads/master', GITHUB_EVENT_NAME: 'workflow_dispatch', + GITHUB_WORKFLOW_REF: 'paperclipai/paperclip/.github/workflows/runner-full-stack-e2e.yml@refs/heads/master', + GITHUB_RUN_ID: '123', GITHUB_RUN_ATTEMPT: '1', GITHUB_SHA: workflowSha, SOURCE_SHA: targetSha, + ALLURE_ENDPOINT: 'https://paperclip.testops.cloud', ALLURE_PROJECT_ID: '1', +}; +const selection = selected => ({ schema: 'paperclip.product-regression-testops-selection/v1', runId: '123', attempt: '1', + workflowSha, targetSha, jobRunId: '456', launchId: 789, cases: selected }); +const result = (row, overrides = {}) => ({ + executionId: row.executionId, suiteId: row.suite, caseId: row.name, profileId: row.profile, environmentId: row.environment, + source: { sha: targetSha }, attempt: 1, status: 'passed', cleanup: 'passed', model: row.model, + matcherResults: [{ matcher: { kind: 'json_path', path: 'resume.retry-works-first-use', expected: true }, passed: true }], ...overrides, +}); +const campaign = (selected, results) => ({ schema: 'paperclip.runner-e2e.campaign/v2', campaignId: 'gha-123-1', + source: { sha: targetSha }, expected: selected.map(row => row.executionId), results }); + +test('the plan is six DeepSeek 1017 cells plus native Codex for steering, Daytona only for sandbox signatures', () => { + assert.equal(reliabilityDefinitions.length, 7); + assert.equal(new Set(reliabilityDefinitions.map(row => row.executionId)).size, 7); + assert.ok(reliabilityDefinitions.every(row => row.fullName === `paperclip.Product E2E.everyday-workflows.${row.name}`)); + // OpenCode declares steering: false, so only queue-steer uses the steering-capable native Codex cell. + assert.deepEqual(reliabilityDefinitions.filter(row => row.profile !== reliabilityProfile || row.model !== reliabilityModel) + .map(row => [row.name, row.profile, row.model, row.environment]), [['queue-steer', 'runner-codex', 'gpt-5.6-sol', 'daytona']]); + assert.deepEqual(reliabilityDefinitions.filter(row => row.profile === reliabilityProfile && row.environment === 'daytona').map(row => row.name), + ['three-turn-revise', 'long-silent-command', 'queued-across-restart']); + assert.equal(testOpsPlanDetails.reliability.size, 7); + assert.equal(testOpsPlanDetails['product-regression'].size, 20); +}); + +test('the committed registration is either absent or complete, and selection fails closed until it exists', () => { + const committed = registeredReliabilityCases(reliabilityRegistration, productRegressionCases.map(row => row.id)); + if (committed === null) { + assert.equal(testOpsPlans().reliability, null); + assert.throws(() => selectProductRegression(undefined, 'all', false, 'reliability'), /product_regression_testops_reliability_unregistered/); + assert.throws(() => selectProductRegression(undefined, 'queue-steer', false, 'reliability'), /reliability_unregistered/); + } else { + assert.equal(committed.length, 7); + assert.equal(selectProductRegression(undefined, 'all', false, 'reliability').length, 7); + } + // Plan 2 keeps its exact standing bootstrap behavior. + assert.equal(selectProductRegression(undefined, 'all').length, 20); + assert.equal(selectProductRegression(undefined, '66')[0].name, 'message-marker'); +}); + +test('a partial, duplicate, foreign or plan-2 registration is rejected rather than selecting a subset', () => { + const reserved = productRegressionCases.map(row => row.id); + const { 'queue-steer': _omitted, ...partial } = synthetic.caseIds; + for (const invalid of [{ ...synthetic, planId: null }, { ...synthetic, planId: 2 }, { ...synthetic, caseIds: partial }, + { ...synthetic, caseIds: { ...synthetic.caseIds, 'queue-steer': synthetic.caseIds['reassign-midrun'] } }, + { ...synthetic, caseIds: { ...synthetic.caseIds, 'queue-steer': 66 } }, + { ...synthetic, caseIds: { ...synthetic.caseIds, 'queue-steer': '9105' } }, + { ...synthetic, caseIds: { ...synthetic.caseIds, unknown: 1 } }]) { + assert.throws(() => registeredReliabilityCases(invalid, reserved), /reliability_registration_invalid/); + } + assert.equal(registeredReliabilityCases({ planId: null, caseIds: {} }, reserved), null); +}); + +test('a TestOps launch selects one plan by its case identities and never mixes the plans', () => { + const all = { version: '1.0', tests: cases.map(row => ({ id: String(row.id), selector: row.fullName })) }; + assert.deepEqual(selectProductRegression(all, 'all', true, 'product-regression', plans).map(row => row.name), + reliabilityDefinitions.map(row => row.name)); + const subset = { version: '1.0', tests: [{ id: String(cases[5].id), selector: cases[5].fullName }] }; + assert.deepEqual(selectProductRegression(subset, 'all', true, 'product-regression', plans), [cases[5]]); + const plan2 = productRegressionCases[0]; + assert.throws(() => selectProductRegression({ version: '1.0', tests: [...subset.tests, { id: String(plan2.id), selector: plan2.fullName }] }, + 'all', true, 'product-regression', plans), /plan_mixed/); + assert.throws(() => selectProductRegression({ version: '1.0', tests: [{ id: String(cases[0].id), selector: cases[1].fullName }] }, + 'all', true, 'product-regression', plans), /case_mismatch/); + // Without the reviewed identities, a TestOps reliability launch cannot bind any case. + assert.throws(() => selectProductRegression(subset, 'all', true, 'product-regression', testOpsPlans({ planId: null, caseIds: {} })), /case_mismatch/); + assert.throws(() => selectProductRegression(undefined, 'all', false, 'unknown', plans), /plan_invalid/); + assert.equal(selectProductRegression(undefined, 'long-silent-command', false, 'reliability', plans)[0].environment, 'daytona'); + validateSelection(selection(cases), env, plans); + assert.throws(() => validateSelection(selection(cases), env, testOpsPlans({ planId: null, caseIds: {} }))); +}); + +test('projection keeps each cell environment, a seven-case denominator and the original verdicts', () => { + const projected = projectProductResults(campaign(cases, cases.map(row => result(row))), selection(cases), targetSha); + assert.deepEqual({ ...projected.summary, cases: undefined }, + { selected: 7, passed: 7, failed: 0, broken: 0, plan: 'reliability', fullPlan: true, cases: undefined }); + const daytona = cases.find(row => row.name === 'queued-across-restart'); + const records = projected.records.filter(row => row.file.endsWith('-result.json')).map(row => row.value); + const restart = records.find(row => row.name === 'queued-across-restart'); + assert.equal(restart.parameters.find(row => row.name === 'environment').value, 'daytona / GitHub Actions'); + assert.ok(restart.labels.some(row => row.name === 'tag' && row.value === 'soak-S55/U9')); + assert.equal(restart.labels.find(row => row.name === 'AS_ID').value, String(daytona.id)); + const attachments = projected.records.filter(row => !row.file.endsWith('-result.json')).map(row => row.value); + assert.equal(attachments.find(row => row.executionId === daytona.executionId).environment, 'daytona'); + // The environment is part of the binding: a local result cannot stand in for a Daytona cell. + assert.throws(() => projectProductResults(campaign([daytona], [result(daytona, { environmentId: 'local' })]), selection([daytona]), targetSha), + /result_binding/); + for (const [failureClass, status] of [['candidate_failure', 'failed'], ['provider_variance', 'failed'], ['transient_infrastructure', 'broken']]) { + const row = projectProductResults(campaign([daytona], [result(daytona, { status: 'failed', failureClass })]), selection([daytona]), targetSha); + assert.equal(row.summary.cases[0].status, status); + assert.equal(row.summary.fullPlan, false); + } + const missing = projectProductResults(undefined, selection(cases), targetSha); + assert.equal(missing.summary.broken, 7); + assert.throws(() => projectProductResults(campaign([daytona], [result(daytona, { model: 'gpt-5.6-sol' })]), selection([daytona]), targetSha), + /metadata_invalid/); + assert.throws(() => projectProductResults(campaign(cases, []), selection([...cases, productRegressionCases[0]]), targetSha), + /selection_binding/); +}); + +test('a manual reliability bootstrap fails before opening any TestOps run until registration exists', async t => { + const root = await mkdtemp(path.join(tmpdir(), 'reliability-testops-')); + t.after(() => rm(root, { recursive: true, force: true })); + const runtime = { ...env, RUNNER_TEMP: root, GITHUB_OUTPUT: path.join(root, 'output'), ALLURE_TOKEN: 'synthetic-test-token', + TESTOPS_PLAN: 'reliability', TESTOPS_CASE: 'all', ALLURE_JOB_RUN_ID: '' }; + let opened = false; + const exec = async () => { opened = true; return { stdout: '' }; }; + const fetchImpl = async () => { opened = true; throw new Error('unexpected network'); }; + await assert.rejects(productTestOps('select', path.join(root, 'selection.json'), runtime, exec, fetchImpl, undefined, + testOpsPlans({ planId: null, caseIds: {} })), /reliability_unregistered/); + await assert.rejects(productTestOps('select', path.join(root, 'selection.json'), { ...runtime, TESTOPS_PLAN: 'unknown' }, exec, fetchImpl), + /plan_invalid/); + assert.equal(opened, false); +}); + +test('a TestOps reliability launch binds, projects and verifies through the standing plan path', async t => { + const root = await mkdtemp(path.join(tmpdir(), 'reliability-testops-launch-')); + t.after(() => rm(root, { recursive: true, force: true })); + const selectionFile = path.join(root, 'selection.json'); + const runtime = { ...env, RUNNER_TEMP: root, GITHUB_OUTPUT: path.join(root, 'output'), GITHUB_STEP_SUMMARY: path.join(root, 'summary'), + ALLURE_JOB_RUN_ID: '456', ALLURE_TOKEN: 'synthetic-test-token', TESTOPS_CASE: '', TESTOPS_PLAN: 'product-regression', + REPORT_DIR: path.join(root, 'report'), ALLURE_RESULTS: path.join(root, 'allure'), EXPORT_SUMMARY: path.join(root, 'export.json') }; + const uploaded = cases.map((row, index) => ({ id: 600 + index, testCaseId: row.id, fullName: row.fullName, projectId: 1, + launchId: 789, jobRun: { id: 456 }, status: 'passed' })); + const fetchImpl = async url => { + const route = new URL(url).pathname; + const resources = { + '/api/uaa/oauth/token': { access_token: 'synthetic-access-token' }, + '/api/jobrun/456': { id: 456, launchId: 789, job: { id: 50, type: 'github' }, url: 'https://github.com/paperclipai/paperclip/actions/runs/123' }, + '/api/job/50': { id: 50, type: 'github', url: workflowUrl }, '/api/launch/789': { id: 789, projectId: 1 }, + '/api/testresult': { content: uploaded.map(row => ({ id: row.id, testCaseId: row.testCaseId })), totalPages: 1, totalElements: uploaded.length }, + ...Object.fromEntries(uploaded.map(row => [`/api/testresult/${row.id}`, row])), + }; + assert.ok(resources[route], route); + return new Response(JSON.stringify(resources[route]), { status: 200, headers: { 'Content-Type': 'application/json' } }); + }; + const exec = async (command, args) => { + if (args[1] === 'env') return { stdout: 'ALLURE_JOB_RUN_ID=456\nALLURE_LAUNCH_ID=789\n' }; + if (args[1] === 'plan') await writeFile(args.at(-1), JSON.stringify({ version: '1.0', tests: cases.map(row => ({ id: String(row.id), selector: row.fullName })) })); + return { stdout: '' }; + }; + assert.equal(await productTestOps('select', selectionFile, runtime, exec, fetchImpl, undefined, plans), 0); + const output = await readFile(runtime.GITHUB_OUTPUT, 'utf8'); + assert.match(output, new RegExp(`ids=${cases.map(row => row.executionId.replaceAll('.', '\\.')).join(',')}\\n`)); + runtime.SELECTION_SHA256 = output.match(/selection_sha256=([a-f0-9]{64})/)[1]; + await mkdir(path.join(runtime.REPORT_DIR, 'normalized'), { recursive: true }); + await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign(cases, cases.map(row => result(row))))); + assert.equal(await productTestOps('project', selectionFile, runtime, exec, fetchImpl, undefined, plans), 0); + assert.match(await readFile(runtime.GITHUB_STEP_SUMMARY, 'utf8'), /Reliability TestOps results.*\n\n7\/7 passed.*Complete 7-case plan selection/); + assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => {}, plans), 0); + // The committed (unregistered) mapping cannot validate a selection bound with other identities. + if (testOpsPlans().reliability === null) { + await assert.rejects(productTestOps('project', selectionFile, { ...runtime, ALLURE_RESULTS: path.join(root, 'other'), + EXPORT_SUMMARY: path.join(root, 'other.json') }, exec, fetchImpl), /case_mismatch/); + } +}); + +function fakeTestOps({ existing = [], plan = null, assignments = [], standing = [{ job: { id: 50, type: 'github', url: workflowUrl, parameters: [] }, testCasesCount: 20 }] } = {}) { + const state = { cases: new Map([[66, { id: 66, projectId: 1, fullName: productRegressionCases.find(row => row.id === 66).fullName }], + ...existing.map(row => [row.id, { projectId: 1, automated: true, ...row }])]), plan, assignments, calls: [], nextId: 9500 }; + const api = async (route, method = 'GET', body) => { + state.calls.push({ route, method, body }); + const url = new URL(`https://paperclip.testops.cloud${route}`); + if (method === 'GET' && url.pathname === '/api/testcase') { + const rows = [...state.cases.keys()].sort((a, b) => a - b).map(id => ({ id })); + return { content: rows, number: 0, totalPages: 1, totalElements: rows.length }; + } + const caseMatch = /^\/api\/testcase\/(\d+)$/.exec(url.pathname); + if (method === 'GET' && caseMatch) return state.cases.get(Number(caseMatch[1])); + if (method === 'POST' && url.pathname === '/api/testcase') { + const id = state.nextId++; state.cases.set(id, { id, ...body }); return { id }; + } + if (method === 'GET' && url.pathname === '/api/testplan') return { content: state.plan ? [{ id: state.plan.id, name: state.plan.name }] : [], totalPages: 1 }; + if (method === 'POST' && url.pathname === '/api/testplan') { + state.plan = { id: 77, projectId: 1, name: body.name, treeSelection: body.treeSelection, testCasesCount: body.treeSelection.leafsInclude.length }; + return { id: 77 }; + } + if (method === 'GET' && url.pathname === '/api/testplan/77') return state.plan; + if (method === 'PATCH' && url.pathname === '/api/testplan/77') { + state.plan = { ...state.plan, treeSelection: body.treeSelection, testCasesCount: body.treeSelection.leafsInclude.length }; return null; + } + if (method === 'POST' && url.pathname === '/api/testplan/77/jobparameter') { + state.assignments = state.assignments.map(row => row.job.id === body.jobId ? { ...row, job: { ...row.job, parameters: body.parameters } } : row); + return null; + } + if (method === 'GET' && url.pathname === '/api/testplan/2/job') return standing; + if (method === 'GET' && url.pathname === '/api/job/50') return { id: 50, type: 'github', url: workflowUrl }; + if (method === 'GET' && url.pathname === '/api/testplan/77/job') return state.assignments; + if (method === 'POST' && url.pathname === '/api/testplan/77/assign') { + state.assignments = [{ job: { id: body.jobId, parameters: [] }, testCasesCount: body.selection.leafsInclude.length }]; return null; + } + throw new Error(`unexpected ${method} ${route}`); + }; + return { api, state }; +} + +test('registration is admitted only from the trusted dispatch and reads before it writes', async () => { + const admitted = { ...env, GITHUB_WORKFLOW_REF: registrationWorkflowRef, REGISTRATION_PLAN: 'reliability', REGISTRATION_OPERATION: 'inspect' }; + assert.equal(admitRegistration(admitted), 'inspect'); + for (const mutation of [{ GITHUB_REF: 'refs/heads/candidate' }, { GITHUB_EVENT_NAME: 'pull_request_target' }, + { GITHUB_WORKFLOW_REF: env.GITHUB_WORKFLOW_REF }, { ALLURE_ENDPOINT: 'https://attacker.invalid' }, + { REGISTRATION_OPERATION: 'delete' }, { REGISTRATION_PLAN: 'product-regression' }]) { + assert.throws(() => admitRegistration({ ...admitted, ...mutation }), /untrusted_context/); + } + const { api, state } = fakeTestOps(); + const inspected = await inspectReliability(api); + assert.deepEqual(inspected.caseIds, {}); + assert.equal(inspected.jobId, 50); + assert.equal(state.calls.some(row => row.method !== 'GET'), false); + const receipt = registrationReceipt('inspect', inspected, admitted); + assert.equal(receipt.complete, false); + assert.equal(receipt.missing.length, 7); +}); + +test('registration creates only missing cases, one plan on the standing job, and is idempotent', async () => { + const reused = reliabilityDefinitions[0]; + const { api, state } = fakeTestOps({ existing: [{ id: 300, fullName: reused.fullName }] }); + const first = await registerReliability(api); + assert.equal(first.caseIds[reused.name], 300); + assert.deepEqual(first.created.cases, reliabilityDefinitions.slice(1).map(row => row.name)); + assert.equal(first.created.plan, true); + assert.equal(first.created.assignment, true); + assert.equal(state.plan.name, reliabilityPlanName); + assert.deepEqual(state.plan.treeSelection.leafsInclude, reliabilityDefinitions.map(row => first.caseIds[row.name])); + for (const row of reliabilityDefinitions.slice(1)) { + const created = state.cases.get(first.caseIds[row.name]); + assert.equal(created.fullName, row.fullName); + assert.equal(created.automated, true); + assert.match(created.description, new RegExp(row.executionId.replaceAll('.', '\\.'))); + } + const receipt = registrationReceipt('register', first, { GITHUB_RUN_ID: '1', GITHUB_SHA: workflowSha }); + assert.equal(receipt.complete, true); + assert.equal(registeredReliabilityCases(receipt.registration, productRegressionCases.map(row => row.id)).length, 7); + assert.match(registrationSnippet(receipt), /planId: 77,\n {2}caseIds: Object\.freeze\(\{\n {4}'interrupt-redirect': 300,/); + const writes = state.calls.filter(row => row.method !== 'GET').length; + const second = await registerReliability(api); + assert.deepEqual(second.caseIds, first.caseIds); + assert.equal(state.calls.filter(row => row.method !== 'GET').length, writes); +}); + +test('registration copies the standing plan job parameters and repairs a drifted plan selection once', async () => { + const parameters = [{ name: 'all', value: 'false' }]; + const { api, state } = fakeTestOps({ standing: [{ job: { id: 50, type: 'github', url: workflowUrl, parameters }, testCasesCount: 20 }] }); + const first = await registerReliability(api); + assert.equal(first.created.parameters, true); + assert.deepEqual(state.calls.filter(row => row.route === '/api/testplan/77/jobparameter').map(row => row.body), + [{ jobId: 50, parameters }]); + assert.deepEqual(state.assignments[0].job.parameters, parameters); + const writes = state.calls.filter(row => row.method !== 'GET').length; + assert.equal((await registerReliability(api)).created.parameters, false); + assert.equal(state.calls.filter(row => row.method !== 'GET').length, writes); + // A plan whose selection drifted is patched back to exactly the seven cases. + state.plan = { ...state.plan, treeSelection: { ...state.plan.treeSelection, leafsInclude: [first.caseIds['queue-steer']] }, testCasesCount: 1 }; + await registerReliability(api); + assert.deepEqual(state.calls.filter(row => row.method === 'PATCH').map(row => row.route), ['/api/testplan/77']); + assert.deepEqual(state.plan.treeSelection.leafsInclude, reliabilityDefinitions.map(row => first.caseIds[row.name])); + // A parameter write that does not read back fails instead of leaving a plan that dispatches differently. + const stale = fakeTestOps({ standing: [{ job: { id: 50, type: 'github', url: workflowUrl, parameters }, testCasesCount: 20 }] }); + const original = stale.api; + const ignoringParameters = async (route, method, body) => route.endsWith('/jobparameter') ? null : original(route, method, body); + await assert.rejects(registerReliability(ignoringParameters), /plan_job/); +}); + +test('registration refuses a standing plan without exactly one trusted job or a plan bound to another job', async () => { + await assert.rejects(registerReliability(fakeTestOps({ standing: [] }).api), /standing_job/); + await assert.rejects(registerReliability(fakeTestOps({ standing: [{ job: { id: 51, type: 'github', url: 'https://github.com/other/repo' } }] }).api), /standing_job/); + const { api, state } = fakeTestOps(); + await registerReliability(api); + state.assignments = [{ job: { id: 99 }, testCasesCount: 7 }]; + await assert.rejects(registerReliability(api), /plan_job_conflict/); +}); + +test('the workflows expose the reliability plan without new provider or reporting authority', async () => { + const full = parse(await readFile(new URL('../../.github/workflows/runner-full-stack-e2e.yml', import.meta.url), 'utf8')); + assert.deepEqual(full.on.workflow_dispatch.inputs.testops_plan.options, ['product-regression', 'reliability']); + assert.equal(full.on.workflow_dispatch.inputs.testops_plan.default, 'product-regression'); + // Not run per PR or on the bounded cron: only a TestOps launch or manual dispatch selects the plan. + assert.deepEqual(Object.keys(full.on).sort(), ['schedule', 'workflow_dispatch']); + const selectionStep = full.jobs.testops_selection.steps.find(step => step.id === 'selection'); + assert.equal(selectionStep.env.TESTOPS_PLAN, "${{ inputs.testops_plan || 'product-regression' }}"); + const testEnv = full.jobs.test.steps.flatMap(step => Object.entries(step.env ?? {})); + assert.ok(testEnv.some(([name, value]) => name === 'OPENROUTER_API_KEY' && value === "${{ matrix.credentialName == 'OPENROUTER_API_KEY' && secrets.OPENROUTER_API_KEY || '' }}")); + assert.ok(testEnv.some(([name, value]) => name === 'DAYTONA_API_KEY' && value === "${{ matrix.environmentId == 'daytona' && secrets.DAYTONA_API_KEY || '' }}")); + const opencodeSteps = full.jobs.test.steps.filter(step => /OpenCode executable|remote provider pack/.test(step.name ?? '')); + assert.ok(opencodeSteps.length >= 3); + assert.ok(opencodeSteps.every(step => String(step.if).includes("startsWith(matrix.profileId, 'runner-opencode')"))); + const registration = parse(await readFile(new URL('../../.github/workflows/runner-e2e-testops-registration.yml', import.meta.url), 'utf8')); + assert.deepEqual(Object.keys(registration.on), ['workflow_dispatch']); + assert.equal(registration.jobs.register.environment, 'runner-e2e-testops-reporting'); + assert.deepEqual(registration.jobs.register.needs, 'authorize'); + // The only gate in front of ALLURE_TOKEN: default branch plus the numeric actor allowlist. + const gate = registration.jobs.authorize.steps[0]; + assert.equal(gate.env.ALLOWED_ACTOR_IDS, '${{ vars.RUNNER_E2E_ALLOWED_ACTOR_IDS }}'); + assert.equal(gate.env.REF, '${{ github.ref }}'); + assert.match(gate.run, /if \[ "\$REF" != "refs\/heads\/\$DEFAULT_BRANCH" \]; then/); + assert.match(gate.run, /type == "array" and length > 0 and all\(\.\[\]; type == "number"/); + assert.match(gate.run, /for candidate in "\$triggering_actor_id" "\$ACTOR_ID"; do/); + assert.equal(registration.jobs.authorize.environment, undefined); + assert.equal(JSON.stringify(registration.jobs.authorize).includes('secrets.'), false); + const text = JSON.stringify(registration); + assert.doesNotMatch(text, /secrets\.(OPENAI|OPENROUTER|ANTHROPIC|DAYTONA)/); + assert.equal(registration.jobs.register.steps[0].with.ref, '${{ github.sha }}'); +}); diff --git a/tests/runner-e2e/run-unit.mjs b/tests/runner-e2e/run-unit.mjs index 7a452216c26..b2e14598d60 100644 --- a/tests/runner-e2e/run-unit.mjs +++ b/tests/runner-e2e/run-unit.mjs @@ -25,7 +25,7 @@ for (const [name, command] of [ ["Plugin build dependencies", ["scripts/ensure-plugin-build-deps.mjs"]], ["Vitest", [vitest, "run", "--config", "tests/runner-e2e/vitest.config.ts", ...args]], // The wrapper adds the CI profiling reporter when PAPERCLIP_CI_PROFILE is set. - ["Node tests", ["scripts/ci-profile/node-test.mjs", "tests/runner-e2e/product-regression-testops.test.mjs", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", + ["Node tests", ["scripts/ci-profile/node-test.mjs", "tests/runner-e2e/product-regression-testops.test.mjs", "tests/runner-e2e/reliability-testops.test.mjs", "tests/runner-e2e/artifact-budget.test.mjs", "tests/runner-e2e/native-completion-checks.test.mjs", "tests/runner-e2e/native-completion-git-source.test.mjs", "tests/runner-e2e/native-completion-source-contract.test.mjs", "tests/runner-e2e/chat-feature-testops-api.test.mjs", "tests/runner-e2e/chat-feature-testops-cli.test.mjs", "tests/runner-e2e/chat-feature-testops-results.test.mjs", "tests/runner-e2e/testops-chat-reporter.test.mjs", "tests/runner-e2e/chat-feature-testops-retained-verification.node-test.mjs", "tests/runner-e2e/chat-completion-testops-api.test.mjs", "tests/runner-e2e/chat-completion-testops-contract.test.mjs", diff --git a/tests/runner-e2e/vitest.config.ts b/tests/runner-e2e/vitest.config.ts index 9c554557990..45178ceb703 100644 --- a/tests/runner-e2e/vitest.config.ts +++ b/tests/runner-e2e/vitest.config.ts @@ -12,7 +12,7 @@ export default defineConfig({ globalSetup: ["./verified-provider-bundles.setup.mjs"], include: ["**/*.test.ts", "**/*.test.mjs"], // These use node:test and run separately in test:e2e:runner:unit. - exclude: [...configDefaults.exclude, "product-regression-testops.test.mjs", "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"], + exclude: [...configDefaults.exclude, "product-regression-testops.test.mjs", "reliability-testops.test.mjs", "native-completion-{checks,git-source,source-contract}.test.mjs", "artifact-budget.test.mjs", "chat-feature-testops-api.test.mjs", "chat-feature-testops-cli.test.mjs", "chat-feature-testops-results.test.mjs", "testops-chat-reporter.test.mjs", "chat-completion-testops-{api,contract}.test.mjs", "chat-full-catalog-testops-*.test.mjs", "chat-catalog-testops-streaming.test.mjs"], testTimeout: 30_000, }, }); diff --git a/tests/runner-e2e/workflow-security.test.ts b/tests/runner-e2e/workflow-security.test.ts index 417b53bcaf0..1ede9fed2dc 100644 --- a/tests/runner-e2e/workflow-security.test.ts +++ b/tests/runner-e2e/workflow-security.test.ts @@ -751,7 +751,7 @@ describe("public repository paid workflow security", () => { "Materialize verified pinned OpenCode executable", ); expect(testJob).toContain( - "matrix.profileId == 'legacy-opencode' || matrix.profileId == 'runner-opencode' || matrix.suiteId == 'openrouter-model-breadth'", + "matrix.profileId == 'legacy-opencode' || startsWith(matrix.profileId, 'runner-opencode') || matrix.suiteId == 'openrouter-model-breadth'", ); expect(testJob).toContain( "node packages/paperclip-runner/scripts/materialize-opencode-binary.mjs",