mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-08 12:33:05 +01:00
fix(evals): close failed TestOps runs and verify each verdict
Keep official run-start context available to always-on cleanup after selection failure or cancellation. Validate manual selectors before opening a run and bind pending TestOps jobs before adoption. Compare every uploaded case verdict against its sealed projection, including a swapped-status negative control. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
737111400f
commit
b8a8d005ab
4 files changed
+62
-11
No files matched your search
@@ -308,6 +308,7 @@ jobs:
|
||||
actions: read
|
||||
outputs:
|
||||
ids: ${{ steps.selection.outputs.ids }}
|
||||
job_run_started: ${{ steps.selection.outputs.job_run_started }}
|
||||
job_run_id: ${{ steps.selection.outputs.job_run_id }}
|
||||
selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
|
||||
launch_url: ${{ steps.selection.outputs.launch_url }}
|
||||
@@ -2017,7 +2018,7 @@ jobs:
|
||||
publish_testops:
|
||||
name: Publish bound Product regression results to TestOps
|
||||
needs: [authorize, testops_selection, catalog, report]
|
||||
if: always() && !cancelled() && needs.testops_selection.result == 'success'
|
||||
if: always() && needs.testops_selection.outputs.job_run_started == 'true'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
environment: runner-e2e-testops-reporting
|
||||
@@ -2028,7 +2029,7 @@ jobs:
|
||||
SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
|
||||
SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }}
|
||||
EVAL_CADENCE: manual
|
||||
ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }}
|
||||
ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id || inputs.ALLURE_JOB_RUN_ID }}
|
||||
ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
|
||||
ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure
|
||||
REPORT_DIR: ${{ github.workspace }}/product-report
|
||||
@@ -2044,22 +2045,27 @@ jobs:
|
||||
with:
|
||||
node-version: 24
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
if: needs.testops_selection.result == 'success' && !cancelled()
|
||||
id: download_selection
|
||||
with:
|
||||
name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: product-testops
|
||||
- name: Download retained full-stack report
|
||||
if: needs.testops_selection.result == 'success' && !cancelled()
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: product-report
|
||||
- uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
|
||||
if: always()
|
||||
with:
|
||||
allurectl-version: '2.23.1'
|
||||
allure-endpoint: https://paperclip.testops.cloud
|
||||
allure-project-id: '1'
|
||||
allure-token: ${{ secrets.ALLURE_TOKEN }}
|
||||
- name: Project original verdicts and explicitly report missing results
|
||||
if: steps.download_selection.outcome == 'success' && !cancelled()
|
||||
id: project
|
||||
run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json
|
||||
- name: Upload only the bound result projection
|
||||
|
||||
Reference in new issue
Block a user