test(evals): run Product regression selections from TestOps

Reuse the paid full-stack workflow for the standing 20-case Product regression plan. Bind TestOps case identities, original job runs, source revisions and retained verdicts in separate trusted reporting jobs. Preserve existing paid admission and make missing evidence explicitly broken.

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-04 09:49:07 -06:00
1 parent 86814c6550
commit d4b11b2024
10 files changed
+596 -5

No files matched your search

+129 -3
View File
@@ -48,6 +48,18 @@ on:
description: "Comma-separated full suite.profile.environment.case IDs; exclusive with other selectors"
type: string
required: false
testops_case:
description: "Product regression TestOps bootstrap: all, case name or case ID; leave empty for normal campaigns"
type: string
required: false
ALLURE_JOB_RUN_ID:
description: "Allure TestOps service input; its requested selection takes precedence"
type: string
required: false
ALLURE_USERNAME:
description: "Allure TestOps service input"
type: string
required: false
permissions:
contents: read
@@ -284,10 +296,66 @@ jobs:
retention-days: 30
if-no-files-found: error
testops_selection:
name: Bind Product regression TestOps selection
needs: authorize
if: github.event_name == 'workflow_dispatch' && (inputs.ALLURE_JOB_RUN_ID != '' || inputs.testops_case != '')
runs-on: ubuntu-latest
timeout-minutes: 5
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
outputs:
ids: ${{ steps.selection.outputs.ids }}
job_run_id: ${{ steps.selection.outputs.job_run_id }}
selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
launch_url: ${{ steps.selection.outputs.launch_url }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: '2.23.1'
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Resolve only admitted plan cases before provider jobs
id: selection
env:
SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}
SELECT_ID: ${{ inputs.id }}
SELECT_SUITE: ${{ inputs.suite }}
SELECT_GROUP: ${{ inputs.group }}
SELECT_PROFILE: ${{ inputs.profile }}
SELECT_ENVIRONMENT: ${{ inputs.environment }}
SELECT_CASE: ${{ inputs.case }}
TESTOPS_CASE: ${{ inputs.testops_case }}
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
ALLURE_LAUNCH_NAME: Product regression · ${{ github.run_number }}
ALLURE_LAUNCH_TAGS: product-regression,github-actions
run: node tests/runner-e2e/product-regression-testops-cli.mjs select product-testops/selection.json
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
with:
name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
path: product-testops/selection.json
if-no-files-found: error
retention-days: 30
catalog:
name: Validate catalog and select cells
needs: [authorize, target_lock]
if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')
needs: [authorize, target_lock, testops_selection]
if: >-
always() && needs.authorize.result == 'success' && needs.target_lock.result == 'success' &&
(needs.testops_selection.result == 'success' || needs.testops_selection.result == 'skipped') &&
(github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'))
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
@@ -371,7 +439,7 @@ jobs:
SELECT_PROFILE: ${{ inputs.profile }}
SELECT_ENVIRONMENT: ${{ inputs.environment }}
SELECT_CASE: ${{ inputs.case }}
SELECT_ID: ${{ inputs.id }}
SELECT_ID: ${{ needs.testops_selection.outputs.ids || inputs.id }}
MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }}
MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }}
REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}
@@ -1946,6 +2014,64 @@ jobs:
echo "ready=false" >> "$GITHUB_OUTPUT"
fi
publish_testops:
name: Publish bound Product regression results to TestOps
needs: [authorize, testops_selection, catalog, report]
if: always() && !cancelled() && needs.testops_selection.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
environment: runner-e2e-testops-reporting
permissions:
contents: read
actions: read
env:
SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }}
EVAL_CADENCE: manual
ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }}
ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure
REPORT_DIR: ${{ github.workspace }}/product-report
EXPORT_SUMMARY: ${{ github.workspace }}/product-regression-export.json
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# Only trusted workflow code sees the reporting token. Candidate
# reports are parsed as data and never loaded as executable modules.
ref: ${{ github.sha }}
persist-credentials: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: 24
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
path: product-testops
- name: Download retained full-stack report
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
with:
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
path: product-report
- uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
with:
allurectl-version: '2.23.1'
allure-endpoint: https://paperclip.testops.cloud
allure-project-id: '1'
allure-token: ${{ secrets.ALLURE_TOKEN }}
- name: Project original verdicts and explicitly report missing results
id: project
run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json
- name: Upload only the bound result projection
if: steps.project.outcome == 'success'
run: allurectl upload --validate-json "$ALLURE_RESULTS"
- name: Verify server-side case identities, completeness and verdicts
if: steps.project.outcome == 'success'
run: node tests/runner-e2e/product-regression-testops-cli.mjs verify product-testops/selection.json
- name: Finish the original TestOps job run
if: always()
run: allurectl job-run stop
publish_history:
name: Publish S3 history and Pages bundle with declared screenshots
needs: [authorize, catalog, report]