From a72c07681888ebaf08d1528fccb39a7f083e5029 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 10:05:36 -0600 Subject: [PATCH] fix(evals): wait for the uploaded case verdicts to settle Existing planned rows can precede upload processing. Keep the bounded observer active until exact case identities and verdicts match the retained report. Unknown, partial and swapped verdicts still fail without another product attempt. Co-Authored-By: Paperclip --- .../product-regression-testops-cli.mjs | 24 ++++++++++++------- .../product-regression-testops.test.mjs | 6 ++++- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs index 6ebd33d5d88..a9d506ff3f5 100644 --- a/tests/runner-e2e/product-regression-testops-cli.mjs +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -68,7 +68,8 @@ export function validateUploadedProductResults(details, selection, summary) { || details.some(row => summary.cases.find(value => value.caseId === row.testCaseId).status !== row.status)) fail('uploaded_status_mismatch'); } -export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) { +export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch, + wait = ms => new Promise(resolve => setTimeout(resolve, ms))) { admitProductTestOps(env); if (command === 'select') { const requestedJob = env.ALLURE_JOB_RUN_ID || ''; @@ -132,18 +133,25 @@ export async function productTestOps(command, argument, env = process.env, exec await verifyProductJob(api, selection, env); const summary = await readJson(env.EXPORT_SUMMARY, 16_384); // Check server-side identity and count. A successful upload is not a pass. - let observed; + let observed, lastVerification; for (let attempt = 0; attempt < 12; attempt++) { observed = await api(`/api/testresult?${new URLSearchParams({ launchId: String(selection.launchId), page: '0', size: '100' })}`); if (!Array.isArray(observed?.content) || observed.totalPages > 1 || observed.totalElements !== observed.content.length) fail('result_page_invalid'); const actual = observed.content; - if (actual.length === selection.cases.length && actual.every(row => selection.cases.some(value => value.id === row.testCaseId))) break; - if (attempt === 11) fail('upload_incomplete'); - await new Promise(resolve => setTimeout(resolve, 2_000)); + if (actual.length === selection.cases.length && actual.every(row => selection.cases.some(value => value.id === row.testCaseId))) { + const details = await Promise.all(actual.map(row => api(`/api/testresult/${row.id}`))); + try { validateUploadedProductResults(details, selection, summary); return 0; } + catch (error) { + if (!['product_regression_testops_uploaded_identity_mismatch', 'product_regression_testops_uploaded_status_mismatch'].includes(error.message)) throw error; + lastVerification = error; + } + } + // Planned rows can exist before the asynchronous upload processor settles + // their run binding and verdict. Observe this one upload, without rerunning + // the product or accepting unknown/partial results. + if (attempt === 11) { if (lastVerification) throw lastVerification; fail('upload_incomplete'); } + await wait(2_000); } - const details = await Promise.all(observed.content.map(row => api(`/api/testresult/${row.id}`))); - validateUploadedProductResults(details, selection, summary); - return 0; } if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index bdba00d15b3..fcf46039387 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -194,6 +194,10 @@ test('the separated selection, retained-report projection and server verificatio await writeFile(path.join(runtime.REPORT_DIR, 'normalized', 'normalized-results.json'), JSON.stringify(campaign())); assert.equal(await productTestOps('project', selectionFile, runtime, exec, fetchImpl), 0); assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl), 0); + uploadedStatus = 'unknown'; + let readinessWaits = 0; + assert.equal(await productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => { readinessWaits++; uploadedStatus = 'passed'; }), 0); + assert.equal(readinessWaits, 1); const failureOutput = path.join(root, 'failed-selection-output'); const badPlan = async (command, args) => args[1] === 'plan' ? (await writeFile(args.at(-1), JSON.stringify({ version: '1.0', tests: [{ id: '999', selector: selected.fullName }] })), { stdout: '' }) @@ -207,7 +211,7 @@ test('the separated selection, retained-report projection and server verificatio { ...runtime, ALLURE_JOB_RUN_ID: '', TESTOPS_CASE: 'unknown' }, async () => { opened = true; }, fetchImpl), /manual_selection_invalid/); assert.equal(opened, false); uploadedStatus = 'failed'; - await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl), /uploaded_status_mismatch/); + await assert.rejects(productTestOps('verify', selectionFile, runtime, exec, fetchImpl, async () => {}), /uploaded_status_mismatch/); await writeFile(selectionFile, JSON.stringify({ ...receipt, targetSha: workflowSha })); await assert.rejects(productTestOps('project', selectionFile, runtime, exec, fetchImpl), /selection_hash/); });