fix(evals): wait for the uploaded case verdicts to settle

Existing planned rows can precede upload processing. Keep the bounded observer active until exact case identities and verdicts match the retained report. Unknown, partial and swapped verdicts still fail without another product attempt.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/product-regression-testops-cli.mjs

@@ -68,7 +68,8 @@ export function validateUploadedProductResults(details, selection, summary) {
       || details.some(row => summary.cases.find(value => value.caseId === row.testCaseId).status !== row.status)) fail('uploaded_status_mismatch');
 }
 
-export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) {
+export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch,
+  wait = ms => new Promise(resolve => setTimeout(resolve, ms))) {
   admitProductTestOps(env);
   if (command === 'select') {
     const requestedJob = env.ALLURE_JOB_RUN_ID || '';
@@ -132,18 +133,25 @@ export async function productTestOps(command, argument, env = process.env, exec
   await verifyProductJob(api, selection, env);
   const summary = await readJson(env.EXPORT_SUMMARY, 16_384);
   // Check server-side identity and count. A successful upload is not a pass.
-  let observed;
+  let observed, lastVerification;
   for (let attempt = 0; attempt < 12; attempt++) {
     observed = await api(`/api/testresult?${new URLSearchParams({ launchId: String(selection.launchId), page: '0', size: '100' })}`);
     if (!Array.isArray(observed?.content) || observed.totalPages > 1 || observed.totalElements !== observed.content.length) fail('result_page_invalid');
     const actual = observed.content;
-    if (actual.length === selection.cases.length && actual.every(row => selection.cases.some(value => value.id === row.testCaseId))) break;
-    if (attempt === 11) fail('upload_incomplete');
-    await new Promise(resolve => setTimeout(resolve, 2_000));
+    if (actual.length === selection.cases.length && actual.every(row => selection.cases.some(value => value.id === row.testCaseId))) {
+      const details = await Promise.all(actual.map(row => api(`/api/testresult/${row.id}`)));
+      try { validateUploadedProductResults(details, selection, summary); return 0; }
+      catch (error) {
+        if (!['product_regression_testops_uploaded_identity_mismatch', 'product_regression_testops_uploaded_status_mismatch'].includes(error.message)) throw error;
+        lastVerification = error;
+      }
+    }
+    // Planned rows can exist before the asynchronous upload processor settles
+    // their run binding and verdict. Observe this one upload, without rerunning
+    // the product or accepting unknown/partial results.
+    if (attempt === 11) { if (lastVerification) throw lastVerification; fail('upload_incomplete'); }
+    await wait(2_000);
   }
-  const details = await Promise.all(observed.content.map(row => api(`/api/testresult/${row.id}`)));
-  validateUploadedProductResults(details, selection, summary);
-  return 0;
 }
 
 if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {

tests/runner-e2e/product-regression-testops.test.mjs

@@ -194,6 +194,10 @@ test('the separated selection, retained-report projection and server verificatio
   await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign()));
   assert.equal(await productTestOps('project', selectionFile, runtime, exec, fetchImpl), 0);
   assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl), 0);
+  uploadedStatus = 'unknown';
+  let readinessWaits = 0;
+  assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => { readinessWaits++; uploadedStatus = 'passed'; }), 0);
+  assert.equal(readinessWaits, 1);
   const failureOutput = path.join(root, 'failed-selection-output');
   const badPlan = async (command, args) => args[1] === 'plan'
     ? (await writeFile(args.at(-1), JSON.stringify({ version: '1.0', tests: [{ id: '999', selector: selected.fullName }] })), { stdout: '' })
@@ -207,7 +211,7 @@ test('the separated selection, retained-report projection and server verificatio
     { ...runtime, ALLURE_JOB_RUN_ID: '', TESTOPS_CASE: 'unknown' }, async () => { opened = true; }, fetchImpl), /manual_selection_invalid/);
   assert.equal(opened, false);
   uploadedStatus = 'failed';
-  await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl), /uploaded_status_mismatch/);
+  await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => {}), /uploaded_status_mismatch/);
   await writeFile(selectionFile, JSON.stringify({ ...receipt, targetSha: workflowSha }));
   await assert.rejects(productTestOps('project', selectionFile, runtime, exec, fetchImpl), /selection_hash/);
 });