fix(evals): close failed TestOps runs and verify each verdict

Keep official run-start context available to always-on cleanup after selection failure or cancellation. Validate manual selectors before opening a run and bind pending TestOps jobs before adoption. Compare every uploaded case verdict against its sealed projection, including a swapped-status negative control.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

.github/workflows/runner-full-stack-e2e.yml

@@ -308,6 +308,7 @@ jobs:
       actions: read
     outputs:
       ids: ${{ steps.selection.outputs.ids }}
+      job_run_started: ${{ steps.selection.outputs.job_run_started }}
       job_run_id: ${{ steps.selection.outputs.job_run_id }}
       selection_sha256: ${{ steps.selection.outputs.selection_sha256 }}
       launch_url: ${{ steps.selection.outputs.launch_url }}
@@ -2017,7 +2018,7 @@ jobs:
   publish_testops:
     name: Publish bound Product regression results to TestOps
     needs: [authorize, testops_selection, catalog, report]
-    if: always() && !cancelled() && needs.testops_selection.result == 'success'
+    if: always() && needs.testops_selection.outputs.job_run_started == 'true'
     runs-on: ubuntu-latest
     timeout-minutes: 10
     environment: runner-e2e-testops-reporting
@@ -2028,7 +2029,7 @@ jobs:
       SOURCE_SHA: ${{ needs.authorize.outputs.target_sha }}
       SELECTION_SHA256: ${{ needs.testops_selection.outputs.selection_sha256 }}
       EVAL_CADENCE: manual
-      ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id }}
+      ALLURE_JOB_RUN_ID: ${{ needs.testops_selection.outputs.job_run_id || inputs.ALLURE_JOB_RUN_ID }}
       ALLURE_USERNAME: ${{ inputs.ALLURE_USERNAME }}
       ALLURE_RESULTS: ${{ github.workspace }}/product-regression-allure
       REPORT_DIR: ${{ github.workspace }}/product-report
@@ -2044,22 +2045,27 @@ jobs:
         with:
           node-version: 24
       - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        if: needs.testops_selection.result == 'success' && !cancelled()
+        id: download_selection
         with:
           name: product-testops-selection-${{ github.run_id }}-${{ github.run_attempt }}
           path: product-testops
       - name: Download retained full-stack report
+        if: needs.testops_selection.result == 'success' && !cancelled()
         continue-on-error: true
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
         with:
           name: runner-e2e-report-${{ github.run_id }}-${{ github.run_attempt }}
           path: product-report
       - uses: allure-framework/setup-allurectl@2d89ee0b7ba2f59dbe9849b6df5763cab78e14db # v1
+        if: always()
         with:
           allurectl-version: '2.23.1'
           allure-endpoint: https://paperclip.testops.cloud
           allure-project-id: '1'
           allure-token: ${{ secrets.ALLURE_TOKEN }}
       - name: Project original verdicts and explicitly report missing results
+        if: steps.download_selection.outcome == 'success' && !cancelled()
         id: project
         run: node tests/runner-e2e/product-regression-testops-cli.mjs project product-testops/selection.json
       - name: Upload only the bound result projection

tests/runner-e2e/product-regression-testops-cli.mjs

@@ -42,10 +42,11 @@ export function officialProductContext(stdout) {
   return { jobRunId: found.ALLURE_JOB_RUN_ID, launchId: Number(found.ALLURE_LAUNCH_ID) };
 }
 
-export async function verifyProductJob(api, context, env) {
+export async function verifyProductJob(api, context, env, allowUnstarted = false) {
   const run = await api(`/api/jobrun/${context.jobRunId}`);
   if (run.id !== Number(context.jobRunId) || run.launchId !== context.launchId || !positive(run.job?.id)
-      || run.job.type !== 'github' || run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`) fail('job_run_mismatch');
+      || run.job.type !== 'github' || (run.url !== `https://github.com/paperclipai/paperclip/actions/runs/${env.GITHUB_RUN_ID}`
+        && !(allowUnstarted && (run.url == null || run.url === '')))) fail('job_run_mismatch');
   const [job, launch] = await Promise.all([api(`/api/job/${run.job.id}`), api(`/api/launch/${context.launchId}`)]);
   if (job.id !== run.job.id || job.type !== 'github' || !workflowUrls.has(job.url)
       || launch.id !== context.launchId || launch.projectId !== 1) fail('job_mismatch');
@@ -53,13 +54,18 @@ export async function verifyProductJob(api, context, env) {
 }
 
 export function validateUploadedProductResults(details, selection, summary) {
+  if (!Array.isArray(summary.cases) || summary.cases.length !== selection.cases.length
+      || new Set(summary.cases.map(row => row.caseId)).size !== selection.cases.length
+      || summary.cases.some(row => !selection.cases.some(value => value.id === row.caseId)
+        || !['passed', 'failed', 'broken'].includes(row.status))) fail('expected_status_invalid');
   if (details.length !== selection.cases.length || new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => {
     const selected = selection.cases.find(value => value.id === row.testCaseId);
     return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId
       || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status);
   })) fail('uploaded_identity_mismatch');
   const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status === status).length]));
-  if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch');
+  if (Object.entries(counts).some(([status, count]) => summary[status] !== count)
+      || details.some(row => summary.cases.find(value => value.caseId === row.testCaseId).status !== row.status)) fail('uploaded_status_mismatch');
 }
 
 export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) {
@@ -67,14 +73,26 @@ export async function productTestOps(command, argument, env = process.env, exec
   if (command === 'select') {
     const requestedJob = env.ALLURE_JOB_RUN_ID || '';
     if (requestedJob && !positive(requestedJob)) fail('job_run_invalid');
+    // Reject a bad bootstrap selector before opening any run. Before adopting
+    // a TestOps dispatch, independently check its pending job and project.
+    const manualCases = requestedJob ? undefined : selectProductRegression(undefined, env.TESTOPS_CASE || 'all');
+    const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl);
+    if (requestedJob) {
+      const pending = await api(`/api/jobrun/${requestedJob}`);
+      if (!Number.isSafeInteger(pending.launchId) || pending.launchId < 1) fail('job_run_invalid');
+      await verifyProductJob(api, { jobRunId: requestedJob, launchId: pending.launchId }, env, true);
+    }
     // Start and stop are separated so the official job run remains open while
     // the existing isolated GitHub matrix runs in other jobs.
+    // A final always() job uses the same official CI identity even if start
+    // times out ambiguously or cancellation interrupts the following reads.
+    await appendFile(env.GITHUB_OUTPUT, 'job_run_started=true\n');
     await exec('allurectl', ['job-run', 'start'], { env, timeout: 60_000, maxBuffer: 32_768 });
     const { stdout } = await exec('allurectl', ['job-run', 'env'], { env, timeout: 30_000, maxBuffer: 16_384 });
     const context = officialProductContext(stdout);
     if (requestedJob && requestedJob !== context.jobRunId) fail('job_run_mismatch');
+    await appendFile(env.GITHUB_OUTPUT, `job_run_id=${context.jobRunId}\nlaunch_url=${endpoint}/launch/${context.launchId}\n`);
     const boundEnv = { ...env, ALLURE_JOB_RUN_ID: context.jobRunId };
-    const api = await testopsClient(env.ALLURE_TOKEN, fetchImpl);
     const job = await verifyProductJob(api, context, env);
     let plan;
     if (requestedJob) {
@@ -83,7 +101,7 @@ export async function productTestOps(command, argument, env = process.env, exec
         { env: boundEnv, timeout: 30_000, maxBuffer: 32_768 });
       plan = await readJson(planFile, 65_536);
     }
-    const cases = selectProductRegression(plan, env.TESTOPS_CASE || 'all', Boolean(requestedJob));
+    const cases = manualCases ?? selectProductRegression(plan, env.TESTOPS_CASE || 'all', true);
     const selection = { schema: 'paperclip.product-regression-testops-selection/v1',
       runId: env.GITHUB_RUN_ID, attempt: env.GITHUB_RUN_ATTEMPT, workflowSha: env.GITHUB_SHA,
       targetSha: env.SOURCE_SHA, ...context, ...job, cases };
@@ -91,7 +109,7 @@ export async function productTestOps(command, argument, env = process.env, exec
     const bytes = Buffer.from(JSON.stringify(selection, null, 2));
     await mkdir(path.dirname(argument), { recursive: true });
     await writeFile(argument, bytes, { flag: 'wx', mode: 0o600 });
-    await appendFile(env.GITHUB_OUTPUT, `ids=${cases.map(row => row.executionId).join(',')}\njob_run_id=${context.jobRunId}\nselection_sha256=${hash(bytes)}\nlaunch_url=${endpoint}/launch/${context.launchId}\n`);
+    await appendFile(env.GITHUB_OUTPUT, `ids=${cases.map(row => row.executionId).join(',')}\nselection_sha256=${hash(bytes)}\n`);
     return 0;
   }
   if (!['project', 'verify'].includes(command)) fail('command_invalid');

tests/runner-e2e/product-regression-testops.mjs

@@ -159,5 +159,5 @@ export function projectProductResults(campaign, selection, sourceSha) {
   }
   return { records, summary: { selected: selection.cases.length, passed: summaries.filter(row => row.status === 'passed').length,
     failed: summaries.filter(row => row.status === 'failed').length, broken: summaries.filter(row => row.status === 'broken').length,
-    fullPlan: selection.cases.length === productRegressionCases.length } };
+    fullPlan: selection.cases.length === productRegressionCases.length, cases: summaries } };
 }

tests/runner-e2e/product-regression-testops.test.mjs

@@ -79,7 +79,10 @@ test('reported pass requires real assertions, successful cleanup and exact curre
 
 test('missing results remain explicit broken cases and cannot improve the denominator', () => {
   const projected = projectProductResults(undefined, selection(productRegressionCases), targetSha);
-  assert.deepEqual(projected.summary, { selected: 20, passed: 0, failed: 0, broken: 20, fullPlan: true });
+  const { cases, ...counts } = projected.summary;
+  assert.deepEqual(counts, { selected: 20, passed: 0, failed: 0, broken: 20, fullPlan: true });
+  assert.equal(cases.length, 20);
+  assert.equal(cases.every(row => row.status === 'broken'), true);
   assert.equal(projected.records.filter(row => row.file.endsWith('-result.json')).length, 20);
   const partial = projectProductResults(campaign([]), selection([selected]), targetSha);
   assert.equal(partial.summary.broken, 1);
@@ -135,6 +138,9 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e
     assert.deepEqual(await verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url } : values[route],
       { jobRunId: '456', launchId: 789 }, env), { jobId: 50, launchId: 789 });
   }
+  const pending = route => route === '/api/jobrun/456' ? { ...values[route], url: null } : values[route];
+  await verifyProductJob(pending, { jobRunId: '456', launchId: 789 }, env, true);
+  await assert.rejects(verifyProductJob(pending, { jobRunId: '456', launchId: 789 }, env));
   await assert.rejects(verifyProductJob(route => route === '/api/launch/789' ? { id: 789, projectId: 2 } : values[route], { jobRunId: '456', launchId: 789 }, env));
   await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: 'https://github.com/other/repo' } : values[route], { jobRunId: '456', launchId: 789 }, env));
   await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: workflowUrl.replace('runner-full-stack-e2e', 'runner-protocol-live-evals') } : values[route], { jobRunId: '456', launchId: 789 }, env));
@@ -142,10 +148,16 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e
 
 test('server verification accepts real Allure string statuses and rejects wrong verdicts or identities', () => {
   const rows = [{ testCaseId: selected.id, fullName: selected.fullName, projectId: 1, launchId: 789, jobRun: { id: 456 }, status: 'passed' }];
-  const summary = { passed: 1, failed: 0, broken: 0 };
+  const summary = { passed: 1, failed: 0, broken: 0, cases: [{ caseId: selected.id, status: 'passed' }] };
   validateUploadedProductResults(rows, selection([selected]), summary);
   assert.throws(() => validateUploadedProductResults([{ ...rows[0], status: 'failed' }], selection([selected]), summary));
   assert.throws(() => validateUploadedProductResults([{ ...rows[0], jobRun: { id: 999 } }], selection([selected]), summary));
+  const other = productRegressionCases[0];
+  const mixed = [rows[0], { ...rows[0], testCaseId: other.id, fullName: other.fullName, status: 'failed' }];
+  const expected = { passed: 1, failed: 1, broken: 0, cases: [{ caseId: selected.id, status: 'passed' }, { caseId: other.id, status: 'failed' }] };
+  validateUploadedProductResults(mixed, selection([selected, other]), expected);
+  assert.throws(() => validateUploadedProductResults([{ ...mixed[0], status: 'failed' }, { ...mixed[1], status: 'passed' }],
+    selection([selected, other]), expected), /uploaded_status_mismatch/);
 });
 
 test('the separated selection, retained-report projection and server verification preserve one TestOps subset', async t => {
@@ -182,6 +194,18 @@ test('the separated selection, retained-report projection and server verificatio
   await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign()));
   assert.equal(await productTestOps('project', selectionFile, runtime, exec, fetchImpl), 0);
   assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl), 0);
+  const failureOutput = path.join(root, 'failed-selection-output');
+  const badPlan = async (command, args) => args[1] === 'plan'
+    ? (await writeFile(args.at(-1), JSON.stringify({ version: '1.0', tests: [{ id: '999', selector: selected.fullName }] })), { stdout: '' })
+    : exec(command, args);
+  await assert.rejects(productTestOps('select', path.join(root, 'failed-selection.json'), { ...runtime, GITHUB_OUTPUT: failureOutput }, badPlan, fetchImpl), /case_mismatch/);
+  const cleanupContext = await readFile(failureOutput, 'utf8');
+  assert.match(cleanupContext, /job_run_started=true/);
+  assert.match(cleanupContext, /job_run_id=456/);
+  let opened = false;
+  await assert.rejects(productTestOps('select', path.join(root, 'invalid-manual.json'),
+    { ...runtime, ALLURE_JOB_RUN_ID: '', TESTOPS_CASE: 'unknown' }, async () => { opened = true; }, fetchImpl), /manual_selection_invalid/);
+  assert.equal(opened, false);
   uploadedStatus = 'failed';
   await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl), /uploaded_status_mismatch/);
   await writeFile(selectionFile, JSON.stringify({ ...receipt, targetSha: workflowSha }));
@@ -198,5 +222,8 @@ test('workflow keeps reporting credentials and trusted code outside candidate/pr
     assert.doesNotMatch(block, /secrets\.(OPENAI|OPENROUTER|ANTHROPIC)|ref: \$\{\{ needs.authorize.outputs.target_sha/);
   }
   assert.match(workflow, /SELECT_ID: \$\{\{ needs.testops_selection.outputs.ids \|\| inputs.id \}\}/);
+  assert.match(reportingJob.split('    steps:\n')[0], /if: always\(\) && needs.testops_selection.outputs.job_run_started == 'true'/);
+  assert.doesNotMatch(reportingJob.split('    steps:\n')[0], /!cancelled\(\)/);
+  assert.match(reportingJob, /Finish the original TestOps job run\n\s+if: always\(\)/);
   assert.doesNotMatch(workflow.slice(0, workflow.indexOf('\njobs:')), /workflow_call:/);
 });