fix(evals): close failed TestOps runs and verify each verdict

Keep official run-start context available to always-on cleanup after selection failure or cancellation. Validate manual selectors before opening a run and bind pending TestOps jobs before adoption. Compare every uploaded case verdict against its sealed projection, including a swapped-status negative control.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-04 10:02:28 -06:00
1 parent 737111400f
commit b8a8d005ab
4 files changed
+62 -11

No files matched your search

+8 -2
View File
@@ -308,6 +308,7 @@ jobs:
actions: read
outputs:
ids: ${{ steps.selection.outputs.ids }}
job_run_started: ${{ steps.selection.outputs.job_run_started }}
job_run_id: ${{ steps.selection.outputs.job_run_id }}
selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
launch_url: ${{ steps.selection.outputs.launch_url }}
@@ -2017,7 +2018,7 @@ jobs:
publish_testops:
name: Publish bound Product regression results to TestOps
needs: [authorize, testops_selection, catalog, report]
if: always() && !cancelled() && needs.testops_selection.result == 'success'
if: always() && needs.testops_selection.outputs.job_run_started == 'true'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
@@ -2028,7 +2029,7 @@ jobs:
SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }}
EVAL_CADENCE: manual
ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }}
ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id || inputs.ALLURE_JOB_RUN_ID }}
ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure
REPORT_DIR: ${{ github.workspace }}/product-report
@@ -2044,22 +2045,27 @@ jobs:
with:
node-version: 24
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
if: needs.testops_selection.result == 'success' && !cancelled()
id: download_selection
with:
name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
path: product-testops
- name: Download retained full-stack report
if: needs.testops_selection.result == 'success' && !cancelled()
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
path: product-report
- uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
if: always()
with:
allurectl-version: '2.23.1'
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Project original verdicts and explicitly report missing results
if: steps.download_selection.outcome == 'success' && !cancelled()
id: project
run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json
- name: Upload only the bound result projection