From ac57b183f0d6f3f08922ebe5c74cb384c2c5a197 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 4 Nov 2026 17:41:18 -0600 Subject: [PATCH] fix(evals): run campaign cells when TestOps selection is skipped (#19053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The Runner Full-Stack E2E workflow runs the paid Product E2E cells, including the Product regression TestOps plan > - #18983 added an optional `testops_selection` job and made `catalog` depend on it > - That job is skipped in every ordinary manual or scheduled campaign > - GitHub's implicit `success()` also fails when an ancestor job was skipped, so every job below `catalog` was skipped > - Since #18983 merged, ordinary campaigns run no cells: the merge reports every selected cell as "No result artifact was uploaded" > - This pull request gives each job below `catalog` an explicit status condition > - The benefit is that ordinary, nightly and branch-targeted campaigns run their cells again, and TestOps campaigns are unchanged ## Linked Issues or Issue Description Refs #18983, #19000 **What happened?** Run https://github.com/paperclipai/paperclip/actions/runs/41914430034 selected 20 Product regression cells with the `id` input. `catalog` passed. `daytona_image`, `build_runner_artifacts`, `build_remote_provider_pack` and every matrix job were skipped. The merge failed with 20 "No result artifact was uploaded" rows. **Expected behavior** An ordinary manual campaign runs its selected cells when `testops_selection` is skipped, as it did before #18983. **Steps to reproduce** Dispatch `runner-full-stack-e2e.yml` on `master` without `testops_case` or `ALLURE_JOB_RUN_ID`. The scheduled daily campaign has the same path. ## What Changed - `daytona_image`, `build_runner_artifacts`, `build_remote_provider_pack`, `test`, `test_recurring` and `test_manual_private` now start with `!cancelled()`. - Each of those jobs now requires `needs..result == 'success'` for each of its real dependencies. A failed or skipped build still skips the cells, as before. The existing schedule, cadence and evidence-route conditions are unchanged. - A new regression test parses the workflow. It requires every job below `testops_selection` to use a status function and to require each real dependency result. ## Verification - `node --test tests/runner-e2e/product-regression-testops.test.mjs`: 15/15 pass. - Negative control: the new test fails against the current `master` workflow. - `daytona-image.test.ts`, `workflow-target.test.ts` and `workflow-timeout.test.ts` pass. `workflow-security.test.ts` has 5 failures. The same 5 fail on unchanged `master` (they concern `pr-trusted.yml` and `chat-catalog-testops-case.yml`). - A workflow change takes effect only from `master`, because paid campaigns run only from the default branch. After merge, dispatch an ordinary `id=` campaign and confirm the cells run. ## Risks - Low risk. The conditions keep the previous failure propagation: a job runs only when each direct dependency succeeded and the run is not cancelled. - `catalog` success already implies `authorize` and `target_lock` success, so those are not repeated. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), via Claude Code with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [x] All Paperclip CI gates are green - [x] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .github/workflows/runner-full-stack-e2e.yml | 28 +++++++++++++++---- .../product-regression-testops.test.mjs | 20 +++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index 9fe2a49e05e..f1d0b44a454 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -541,6 +541,10 @@ jobs: daytona_image: name: Publish verified Daytona image needs: [authorize, target_lock, catalog] + # testops_selection is skipped outside TestOps campaigns. GitHub's implicit + # success() also checks skipped ancestors, so jobs below catalog state + # their real dependency results explicitly. + if: ${{ !cancelled() && needs.catalog.result == 'success' }} # Keep image builds on the same authorized fleet as campaign execution. runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: 45 @@ -687,7 +691,9 @@ jobs: build_runner_artifacts: name: Build reusable runner campaign artifacts needs: [authorize, target_lock, catalog] - if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') + if: >- + !cancelled() && needs.catalog.result == 'success' && + (github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')) # Compile native binaries on the same reviewed image used to execute them, # avoiding libc/architecture drift between GitHub-hosted and AWS lanes. runs-on: ${{ needs.authorize.outputs.test_runner }} @@ -815,7 +821,10 @@ jobs: name: Build reusable remote provider pack needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts] - if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true') + if: >- + !cancelled() && needs.catalog.result == 'success' && needs.daytona_image.result == 'success' && + needs.build_runner_artifacts.result == 'success' && + (github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')) runs-on: ubuntu-latest timeout-minutes: 15 permissions: @@ -951,7 +960,10 @@ jobs: test: name: ${{ matrix.executionId }} - if: needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' + if: >- + !cancelled() && needs.catalog.result == 'success' && needs.daytona_image.result == 'success' && + needs.build_runner_artifacts.result == 'success' && needs.build_remote_provider_pack.result == 'success' && + needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'actions' needs: [ authorize, @@ -1443,7 +1455,10 @@ jobs: test_recurring: name: ${{ matrix.executionId }} - if: needs.catalog.outputs.operations_cadence != '' && needs.authorize.outputs.target_sha == github.sha && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + if: >- + !cancelled() && needs.catalog.result == 'success' && needs.daytona_image.result == 'success' && + needs.build_runner_artifacts.result == 'success' && needs.build_remote_provider_pack.result == 'success' && + needs.catalog.outputs.operations_cadence != '' && needs.authorize.outputs.target_sha == github.sha && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack] runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: ${{ matrix.timeoutMinutes }} @@ -1460,7 +1475,10 @@ jobs: test_manual_private: name: ${{ matrix.executionId }} - if: needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'private' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + if: >- + !cancelled() && needs.catalog.result == 'success' && needs.daytona_image.result == 'success' && + needs.build_runner_artifacts.result == 'success' && needs.build_remote_provider_pack.result == 'success' && + needs.catalog.outputs.operations_cadence == '' && needs.authorize.outputs.evidence_retention == 'private' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) needs: [authorize, target_lock, catalog, daytona_image, build_runner_artifacts, build_remote_provider_pack] runs-on: ${{ needs.authorize.outputs.test_runner }} timeout-minutes: ${{ matrix.timeoutMinutes }} diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index 386ffba8092..9ad90133089 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -270,3 +270,23 @@ test('workflow keeps reporting credentials and trusted code outside candidate/pr assert.match(reportingJob, /Finish the original TestOps job run\n\s+if: always\(\)/); assert.doesNotMatch(workflow.slice(0, workflow.indexOf('\njobs:')), /workflow_call:/); }); + +test('ordinary campaigns still run cells when the TestOps selection job is skipped', async () => { + const { parse } = await import('yaml'); + const { jobs } = parse(await readFile(new URL('../../.github/workflows/runner-full-stack-e2e.yml', import.meta.url), 'utf8')); + const needs = name => [jobs[name].needs ?? []].flat(); + const descends = name => needs(name).some(parent => parent === 'testops_selection' || descends(parent)); + // GitHub's implicit success() also fails on a skipped ancestor, so any job + // below the optional selection must use a status function and then require + // each real dependency result itself. + const below = Object.keys(jobs).filter(descends); + assert.ok(below.includes('test') && below.includes('daytona_image')); + for (const name of below) { + const condition = String(jobs[name].if ?? ''); + assert.match(condition, /always\(\)|!cancelled\(\)/, `${name} needs an explicit status function`); + if (condition.includes('always()') || name === 'catalog') continue; + for (const parent of needs(name).filter(parent => !['authorize', 'target_lock'].includes(parent))) { + assert.match(condition, new RegExp(`needs\\.${parent}\\.result == 'success'`), `${name} must still require ${parent}`); + } + } +});