diff --git a/.github/workflows/runner-full-stack-e2e.yml b/.github/workflows/runner-full-stack-e2e.yml index df8358cf991..9fe2a49e05e 100644 --- a/.github/workflows/runner-full-stack-e2e.yml +++ b/.github/workflows/runner-full-stack-e2e.yml @@ -308,6 +308,7 @@ jobs: actions: read outputs: ids: ${{ steps.selection.outputs.ids }} + job_run_started: ${{ steps.selection.outputs.job_run_started }} job_run_id: ${{ steps.selection.outputs.job_run_id }} selection_sha256: ${{ steps.selection.outputs.selection_sha256 }} launch_url: ${{ steps.selection.outputs.launch_url }} @@ -2017,7 +2018,7 @@ jobs: publish_testops: name: Publish bound Product regression results to TestOps needs: [authorize, testops_selection, catalog, report] - if: always() && !cancelled() && needs.testops_selection.result == 'success' + if: always() && needs.testops_selection.outputs.job_run_started == 'true' runs-on: ubuntu-latest timeout-minutes: 10 environment: runner-e2e-testops-reporting @@ -2028,7 +2029,7 @@ jobs: SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }} SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }} EVAL_CADENCE: manual - ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }} + ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id || inputs.ALLURE_JOB_RUN_ID }} ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }} ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure REPORT_DIR: ${{ github.workspace }}/product-report @@ -2044,22 +2045,27 @@ jobs: with: node-version: 24 - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: needs.testops_selection.result == 'success' && !cancelled() + id: download_selection with: name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }} path: product-testops - name: Download retained full-stack report + if: needs.testops_selection.result == 'success' && !cancelled() continue-on-error: true uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }} path: product-report - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1 + if: always() with: allurectl-version: '2.23.1' allure-endpoint: https://paperclip.testops.cloud allure-project-id: '1' allure-token: ${{ secrets.ALLURE_TOKEN }} - name: Project original verdicts and explicitly report missing results + if: steps.download_selection.outcome == 'success' && !cancelled() id: project run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json - name: Upload only the bound result projection diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs index 664b0c5d1f0..6ebd33d5d88 100644 --- a/tests/runner-e2e/product-regression-testops-cli.mjs +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -42,10 +42,11 @@ export function officialProductContext(stdout) { return { jobRunId: found.ALLURE_JOB_RUN_ID, launchId: Number(found.ALLURE_LAUNCH_ID) }; } -export async function verifyProductJob(api, context, env) { +export async function verifyProductJob(api, context, env, allowUnstarted = false) { const run = await api(`/api/jobrun/${context.jobRunId}`); if (run.id !== Number(context.jobRunId) || run.launchId !== context.launchId || !positive(run.job?.id) - || run.job.type !== 'github' || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`) fail('job_run_mismatch'); + || run.job.type !== 'github' || (run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}` + && !(allowUnstarted && (run.url == null || run.url === '')))) fail('job_run_mismatch'); const [job, launch] = await Promise.all([api(`/api/job/${run.job.id}`), api(`/api/launch/${context.launchId}`)]); if (job.id !== run.job.id || job.type !== 'github' || !workflowUrls.has(job.url) || launch.id !== context.launchId || launch.projectId !== 1) fail('job_mismatch'); @@ -53,13 +54,18 @@ export async function verifyProductJob(api, context, env) { } export function validateUploadedProductResults(details, selection, summary) { + if (!Array.isArray(summary.cases) || summary.cases.length !== selection.cases.length + || new Set(summary.cases.map(row => row.caseId)).size !== selection.cases.length + || summary.cases.some(row => !selection.cases.some(value => value.id === row.caseId) + || !['passed', 'failed', 'broken'].includes(row.status))) fail('expected_status_invalid'); if (details.length !== selection.cases.length || new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => { const selected = selection.cases.find(value => value.id === row.testCaseId); return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status); })) fail('uploaded_identity_mismatch'); const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status === status).length])); - if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch'); + if (Object.entries(counts).some(([status, count]) => summary[status] !== count) + || details.some(row => summary.cases.find(value => value.caseId === row.testCaseId).status !== row.status)) fail('uploaded_status_mismatch'); } export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) { @@ -67,14 +73,26 @@ export async function productTestOps(command, argument, env = process.env, exec if (command === 'select') { const requestedJob = env.ALLURE_JOB_RUN_ID || ''; if (requestedJob && !positive(requestedJob)) fail('job_run_invalid'); + // Reject a bad bootstrap selector before opening any run. Before adopting + // a TestOps dispatch, independently check its pending job and project. + const manualCases = requestedJob ? undefined : selectProductRegression(undefined, env.TESTOPS_CASE || 'all'); + const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); + if (requestedJob) { + const pending = await api(`/api/jobrun/${requestedJob}`); + if (!Number.isSafeInteger(pending.launchId) || pending.launchId < 1) fail('job_run_invalid'); + await verifyProductJob(api, { jobRunId: requestedJob, launchId: pending.launchId }, env, true); + } // Start and stop are separated so the official job run remains open while // the existing isolated GitHub matrix runs in other jobs. + // A final always() job uses the same official CI identity even if start + // times out ambiguously or cancellation interrupts the following reads. + await appendFile(env.GITHUB_OUTPUT, 'job_run_started=true\n'); await exec('allurectl', ['job-run', 'start'], { env, timeout: 60_000, maxBuffer: 32_768 }); const { stdout } = await exec('allurectl', ['job-run', 'env'], { env, timeout: 30_000, maxBuffer: 16_384 }); const context = officialProductContext(stdout); if (requestedJob && requestedJob !== context.jobRunId) fail('job_run_mismatch'); + await appendFile(env.GITHUB_OUTPUT, `job_run_id=${context.jobRunId}\nlaunch_url=${endpoint}/launch/${context.launchId}\n`); const boundEnv = { ...env, ALLURE_JOB_RUN_ID: context.jobRunId }; - const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl); const job = await verifyProductJob(api, context, env); let plan; if (requestedJob) { @@ -83,7 +101,7 @@ export async function productTestOps(command, argument, env = process.env, exec { env: boundEnv, timeout: 30_000, maxBuffer: 32_768 }); plan = await readJson(planFile, 65_536); } - const cases = selectProductRegression(plan, env.TESTOPS_CASE || 'all', Boolean(requestedJob)); + const cases = manualCases ?? selectProductRegression(plan, env.TESTOPS_CASE || 'all', true); const selection = { schema: 'paperclip.product-regression-testops-selection/v1', runId: env.GITHUB_RUN_ID, attempt: env.GITHUB_RUN_ATTEMPT, workflowSha: env.GITHUB_SHA, targetSha: env.SOURCE_SHA, ...context, ...job, cases }; @@ -91,7 +109,7 @@ export async function productTestOps(command, argument, env = process.env, exec const bytes = Buffer.from(JSON.stringify(selection, null, 2)); await mkdir(path.dirname(argument), { recursive: true }); await writeFile(argument, bytes, { flag: 'wx', mode: 0o600 }); - await appendFile(env.GITHUB_OUTPUT, `ids=${cases.map(row => row.executionId).join(',')}\njob_run_id=${context.jobRunId}\nselection_sha256=${hash(bytes)}\nlaunch_url=${endpoint}/launch/${context.launchId}\n`); + await appendFile(env.GITHUB_OUTPUT, `ids=${cases.map(row => row.executionId).join(',')}\nselection_sha256=${hash(bytes)}\n`); return 0; } if (!['project', 'verify'].includes(command)) fail('command_invalid'); diff --git a/tests/runner-e2e/product-regression-testops.mjs b/tests/runner-e2e/product-regression-testops.mjs index c4500dbd656..6c672f0aafa 100644 --- a/tests/runner-e2e/product-regression-testops.mjs +++ b/tests/runner-e2e/product-regression-testops.mjs @@ -159,5 +159,5 @@ export function projectProductResults(campaign, selection, sourceSha) { } return { records, summary: { selected: selection.cases.length, passed: summaries.filter(row => row.status === 'passed').length, failed: summaries.filter(row => row.status === 'failed').length, broken: summaries.filter(row => row.status === 'broken').length, - fullPlan: selection.cases.length === productRegressionCases.length } }; + fullPlan: selection.cases.length === productRegressionCases.length, cases: summaries } }; } diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index 0762d650d0d..bdba00d15b3 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -79,7 +79,10 @@ test('reported pass requires real assertions, successful cleanup and exact curre test('missing results remain explicit broken cases and cannot improve the denominator', () => { const projected = projectProductResults(undefined, selection(productRegressionCases), targetSha); - assert.deepEqual(projected.summary, { selected: 20, passed: 0, failed: 0, broken: 20, fullPlan: true }); + const { cases, ...counts } = projected.summary; + assert.deepEqual(counts, { selected: 20, passed: 0, failed: 0, broken: 20, fullPlan: true }); + assert.equal(cases.length, 20); + assert.equal(cases.every(row => row.status === 'broken'), true); assert.equal(projected.records.filter(row => row.file.endsWith('-result.json')).length, 20); const partial = projectProductResults(campaign([]), selection([selected]), targetSha); assert.equal(partial.summary.broken, 1); @@ -135,6 +138,9 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e assert.deepEqual(await verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url } : values[route], { jobRunId: '456', launchId: 789 }, env), { jobId: 50, launchId: 789 }); } + const pending = route => route === '/api/jobrun/456' ? { ...values[route], url: null } : values[route]; + await verifyProductJob(pending, { jobRunId: '456', launchId: 789 }, env, true); + await assert.rejects(verifyProductJob(pending, { jobRunId: '456', launchId: 789 }, env)); await assert.rejects(verifyProductJob(route => route === '/api/launch/789' ? { id: 789, projectId: 2 } : values[route], { jobRunId: '456', launchId: 789 }, env)); await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: 'https://github.com/other/repo' } : values[route], { jobRunId: '456', launchId: 789 }, env)); await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: workflowUrl.replace('runner-full-stack-e2e', 'runner-protocol-live-evals') } : values[route], { jobRunId: '456', launchId: 789 }, env)); @@ -142,10 +148,16 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e test('server verification accepts real Allure string statuses and rejects wrong verdicts or identities', () => { const rows = [{ testCaseId: selected.id, fullName: selected.fullName, projectId: 1, launchId: 789, jobRun: { id: 456 }, status: 'passed' }]; - const summary = { passed: 1, failed: 0, broken: 0 }; + const summary = { passed: 1, failed: 0, broken: 0, cases: [{ caseId: selected.id, status: 'passed' }] }; validateUploadedProductResults(rows, selection([selected]), summary); assert.throws(() => validateUploadedProductResults([{ ...rows[0], status: 'failed' }], selection([selected]), summary)); assert.throws(() => validateUploadedProductResults([{ ...rows[0], jobRun: { id: 999 } }], selection([selected]), summary)); + const other = productRegressionCases[0]; + const mixed = [rows[0], { ...rows[0], testCaseId: other.id, fullName: other.fullName, status: 'failed' }]; + const expected = { passed: 1, failed: 1, broken: 0, cases: [{ caseId: selected.id, status: 'passed' }, { caseId: other.id, status: 'failed' }] }; + validateUploadedProductResults(mixed, selection([selected, other]), expected); + assert.throws(() => validateUploadedProductResults([{ ...mixed[0], status: 'failed' }, { ...mixed[1], status: 'passed' }], + selection([selected, other]), expected), /uploaded_status_mismatch/); }); test('the separated selection, retained-report projection and server verification preserve one TestOps subset', async t => { @@ -182,6 +194,18 @@ test('the separated selection, retained-report projection and server verificatio await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign())); assert.equal(await productTestOps('project', selectionFile, runtime, exec, fetchImpl), 0); assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl), 0); + const failureOutput = path.join(root, 'failed-selection-output'); + const badPlan = async (command, args) => args[1] === 'plan' + ? (await writeFile(args.at(-1), JSON.stringify({ version: '1.0', tests: [{ id: '999', selector: selected.fullName }] })), { stdout: '' }) + : exec(command, args); + await assert.rejects(productTestOps('select', path.join(root, 'failed-selection.json'), { ...runtime, GITHUB_OUTPUT: failureOutput }, badPlan, fetchImpl), /case_mismatch/); + const cleanupContext = await readFile(failureOutput, 'utf8'); + assert.match(cleanupContext, /job_run_started=true/); + assert.match(cleanupContext, /job_run_id=456/); + let opened = false; + await assert.rejects(productTestOps('select', path.join(root, 'invalid-manual.json'), + { ...runtime, ALLURE_JOB_RUN_ID: '', TESTOPS_CASE: 'unknown' }, async () => { opened = true; }, fetchImpl), /manual_selection_invalid/); + assert.equal(opened, false); uploadedStatus = 'failed'; await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl), /uploaded_status_mismatch/); await writeFile(selectionFile, JSON.stringify({ ...receipt, targetSha: workflowSha })); @@ -198,5 +222,8 @@ test('workflow keeps reporting credentials and trusted code outside candidate/pr assert.doesNotMatch(block, /secrets\.(OPENAI|OPENROUTER|ANTHROPIC)|ref: \$\{\{ needs.authorize.outputs.target_sha/); } assert.match(workflow, /SELECT_ID: \$\{\{ needs.testops_selection.outputs.ids \|\| inputs.id \}\}/); + assert.match(reportingJob.split(' steps:\n')[0], /if: always\(\) && needs.testops_selection.outputs.job_run_started == 'true'/); + assert.doesNotMatch(reportingJob.split(' steps:\n')[0], /!cancelled\(\)/); + assert.match(reportingJob, /Finish the original TestOps job run\n\s+if: always\(\)/); assert.doesNotMatch(workflow.slice(0, workflow.indexOf('\njobs:')), /workflow_call:/); });