mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-08 21:35:36 +01:00
test(evals): run Product regression selections from TestOps
Reuse the paid full-stack workflow for the standing 20-case Product regression plan. Bind TestOps case identities, original job runs, source revisions and retained verdicts in separate trusted reporting jobs. Preserve existing paid admission and make missing evidence explicitly broken. Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
86814c6550
commit
d4b11b2024
10 files changed
+596
-5
No files matched your search
@@ -48,6 +48,18 @@ on:
|
||||
description: "Comma-separated full suite.profile.environment.case IDs; exclusive with other selectors"
|
||||
type: string
|
||||
required: false
|
||||
testops_case:
|
||||
description: "Product regression TestOps bootstrap: all, case name or case ID; leave empty for normal campaigns"
|
||||
type: string
|
||||
required: false
|
||||
ALLURE_JOB_RUN_ID:
|
||||
description: "Allure TestOps service input; its requested selection takes precedence"
|
||||
type: string
|
||||
required: false
|
||||
ALLURE_USERNAME:
|
||||
description: "Allure TestOps service input"
|
||||
type: string
|
||||
required: false
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
@@ -284,10 +296,66 @@ jobs:
|
||||
retention-days: 30
|
||||
if-no-files-found: error
|
||||
|
||||
testops_selection:
|
||||
name: Bind Product regression TestOps selection
|
||||
needs: authorize
|
||||
if: github.event_name == 'workflow_dispatch' && (inputs.ALLURE_JOB_RUN_ID != '' || inputs.testops_case != '')
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
environment: runner-e2e-testops-reporting
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
ids: ${{ steps.selection.outputs.ids }}
|
||||
job_run_id: ${{ steps.selection.outputs.job_run_id }}
|
||||
selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
|
||||
launch_url: ${{ steps.selection.outputs.launch_url }}
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
- uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
|
||||
with:
|
||||
allurectl-version: '2.23.1'
|
||||
allure-endpoint: https://paperclip.testops.cloud
|
||||
allure-project-id: '1'
|
||||
allure-token: ${{ secrets.ALLURE_TOKEN }}
|
||||
- name: Resolve only admitted plan cases before provider jobs
|
||||
id: selection
|
||||
env:
|
||||
SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
|
||||
EVAL_CADENCE: ${{ inputs.cadence || 'manual' }}
|
||||
SELECT_ID: ${{ inputs.id }}
|
||||
SELECT_SUITE: ${{ inputs.suite }}
|
||||
SELECT_GROUP: ${{ inputs.group }}
|
||||
SELECT_PROFILE: ${{ inputs.profile }}
|
||||
SELECT_ENVIRONMENT: ${{ inputs.environment }}
|
||||
SELECT_CASE: ${{ inputs.case }}
|
||||
TESTOPS_CASE: ${{ inputs.testops_case }}
|
||||
ALLURE_JOB_RUN_ID: ${{ inputs.ALLURE_JOB_RUN_ID }}
|
||||
ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
|
||||
ALLURE_LAUNCH_NAME: Product regression · ${{ github.run_number }}
|
||||
ALLURE_LAUNCH_TAGS: product-regression,github-actions
|
||||
run: node tests/runner-e2e/product-regression-testops-cli.mjs select product-testops/selection.json
|
||||
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
||||
with:
|
||||
name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: product-testops/selection.json
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
catalog:
|
||||
name: Validate catalog and select cells
|
||||
needs: [authorize, target_lock]
|
||||
if: github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true')
|
||||
needs: [authorize, target_lock, testops_selection]
|
||||
if: >-
|
||||
always() && needs.authorize.result == 'success' && needs.target_lock.result == 'success' &&
|
||||
(needs.testops_selection.result == 'success' || needs.testops_selection.result == 'skipped') &&
|
||||
(github.event_name != 'schedule' || (vars.RUNNER_FULL_STACK_E2E_NIGHTLY_ENABLED == 'true' && vars.RUNNER_FULL_STACK_E2E_OPERATIONS_ENABLED == 'true'))
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
permissions:
|
||||
@@ -371,7 +439,7 @@ jobs:
|
||||
SELECT_PROFILE: ${{ inputs.profile }}
|
||||
SELECT_ENVIRONMENT: ${{ inputs.environment }}
|
||||
SELECT_CASE: ${{ inputs.case }}
|
||||
SELECT_ID: ${{ inputs.id }}
|
||||
SELECT_ID: ${{ needs.testops_selection.outputs.ids || inputs.id }}
|
||||
MAX_PARALLEL: ${{ vars.RUNNER_E2E_MAX_PARALLEL || needs.authorize.outputs.max_parallel_default }}
|
||||
MAX_PARALLEL_LIMIT: ${{ needs.authorize.outputs.max_parallel_limit }}
|
||||
REQUESTED_MAX_PARALLEL: ${{ inputs.max_parallel }}
|
||||
@@ -1946,6 +2014,64 @@ jobs:
|
||||
echo "ready=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
publish_testops:
|
||||
name: Publish bound Product regression results to TestOps
|
||||
needs: [authorize, testops_selection, catalog, report]
|
||||
if: always() && !cancelled() && needs.testops_selection.result == 'success'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
environment: runner-e2e-testops-reporting
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
env:
|
||||
SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
|
||||
SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }}
|
||||
EVAL_CADENCE: manual
|
||||
ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }}
|
||||
ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
|
||||
ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure
|
||||
REPORT_DIR: ${{ github.workspace }}/product-report
|
||||
EXPORT_SUMMARY: ${{ github.workspace }}/product-regression-export.json
|
||||
steps:
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
# Only trusted workflow code sees the reporting token. Candidate
|
||||
# reports are parsed as data and never loaded as executable modules.
|
||||
ref: ${{ github.sha }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
|
||||
with:
|
||||
node-version: 24
|
||||
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: product-testops
|
||||
- name: Download retained full-stack report
|
||||
continue-on-error: true
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
|
||||
with:
|
||||
name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: product-report
|
||||
- uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
|
||||
with:
|
||||
allurectl-version: '2.23.1'
|
||||
allure-endpoint: https://paperclip.testops.cloud
|
||||
allure-project-id: '1'
|
||||
allure-token: ${{ secrets.ALLURE_TOKEN }}
|
||||
- name: Project original verdicts and explicitly report missing results
|
||||
id: project
|
||||
run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json
|
||||
- name: Upload only the bound result projection
|
||||
if: steps.project.outcome == 'success'
|
||||
run: allurectl upload --validate-json "$ALLURE_RESULTS"
|
||||
- name: Verify server-side case identities, completeness and verdicts
|
||||
if: steps.project.outcome == 'success'
|
||||
run: node tests/runner-e2e/product-regression-testops-cli.mjs verify product-testops/selection.json
|
||||
- name: Finish the original TestOps job run
|
||||
if: always()
|
||||
run: allurectl job-run stop
|
||||
|
||||
publish_history:
|
||||
name: Publish S3 history and Pages bundle with declared screenshots
|
||||
needs: [authorize, catalog, report]
|
||||
|
||||
Reference in new issue
Block a user