From 0da58a64d78d34b848020d91e1ddffcd155989e6 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 09:51:15 -0600 Subject: [PATCH] test(evals): retain authoritative attempt verdict receipts Use normalized campaign accounting for earlier failed attempts. Preserve provider model identifiers with slashes and validate the actual string status returned by TestOps. Cover missing-artifact placeholder rows and server-side status/identity controls. Co-Authored-By: Paperclip --- .../product-regression-testops-cli.mjs | 18 ++++++---- .../runner-e2e/product-regression-testops.mjs | 19 ++++++++++- .../product-regression-testops.test.mjs | 34 ++++++++++++++++--- 3 files changed, 58 insertions(+), 13 deletions(-) diff --git a/tests/runner-e2e/product-regression-testops-cli.mjs b/tests/runner-e2e/product-regression-testops-cli.mjs index ac5154a9690..f0cc35937f5 100644 --- a/tests/runner-e2e/product-regression-testops-cli.mjs +++ b/tests/runner-e2e/product-regression-testops-cli.mjs @@ -52,6 +52,16 @@ export async function verifyProductJob(api, context, env) { return { jobId: job.id, launchId: launch.id }; } +export function validateUploadedProductResults(details, selection, summary) { + if (details.length !== selection.cases.length || new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => { + const selected = selection.cases.find(value => value.id === row.testCaseId); + return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId + || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status); + })) fail('uploaded_identity_mismatch'); + const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status === status).length])); + if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch'); +} + export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) { admitProductTestOps(env); if (command === 'select') { @@ -114,13 +124,7 @@ export async function productTestOps(command, argument, env = process.env, exec await new Promise(resolve => setTimeout(resolve, 2_000)); } const details = await Promise.all(observed.content.map(row => api(`/api/testresult/${row.id}`))); - if (new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => { - const selected = selection.cases.find(value => value.id === row.testCaseId); - return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId - || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status?.name); - })) fail('uploaded_identity_mismatch'); - const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status.name === status).length])); - if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch'); + validateUploadedProductResults(details, selection, summary); return 0; } diff --git a/tests/runner-e2e/product-regression-testops.mjs b/tests/runner-e2e/product-regression-testops.mjs index f572b520316..dc4436c4d50 100644 --- a/tests/runner-e2e/product-regression-testops.mjs +++ b/tests/runner-e2e/product-regression-testops.mjs @@ -73,7 +73,16 @@ export function projectProductResults(campaign, selection, sourceSha) { || !Array.isArray(campaign.results) || !Array.isArray(campaign.expected) || JSON.stringify([...campaign.expected].sort()) !== JSON.stringify(selection.cases.map(row => row.executionId).sort()))) fail('report_binding'); const rows = campaign?.results ?? []; - if (rows.length > selection.cases.length * 10 || rows.some(row => !selection.cases.some(value => value.executionId === row.executionId))) fail('unexpected_result'); + if (rows.length > selection.cases.length || new Set(rows.map(row => row.executionId)).size !== rows.length + || rows.some(row => !selection.cases.some(value => value.executionId === row.executionId))) fail('unexpected_result'); + const accounting = campaign?.accounting; + if (accounting && (accounting.version !== 1 || !['retained_attempts', 'selected_results'].includes(accounting.scope) + || !Array.isArray(accounting.attempts) || accounting.attempts.length > selection.cases.length * 10 + || !Number.isSafeInteger(accounting.missingAttempts) || accounting.missingAttempts < 0 + || !Number.isSafeInteger(accounting.excludedArtifacts) || accounting.excludedArtifacts < 0 + || accounting.attempts.some(row => !selection.cases.some(value => value.executionId === row.executionId) + || !Number.isSafeInteger(row.attempt) || row.attempt < 1 || row.attempt > 10 + || !['passed', 'failed'].includes(row.status) || !['passed', 'failed', 'not_started'].includes(row.cleanup)))) fail('accounting_invalid'); const records = [], summaries = []; for (const selected of selection.cases) { const attempts = rows.filter(row => row.executionId === selected.executionId).sort((a, b) => a.attempt - b.attempt); @@ -108,6 +117,14 @@ export function projectProductResults(campaign, selection, sourceSha) { name: identifier(check.matcher?.path) ? check.matcher.path : identifier(check.matcher?.kind) ? check.matcher.kind : 'assertion', passed: check.passed, })), + retainedAttempts: (accounting?.attempts ?? []).filter(row => row.executionId === selected.executionId).map(row => ({ + attempt: row.attempt, status: row.status, cleanup: row.cleanup, + durationMs: Number.isFinite(row.durationMs) && row.durationMs >= 0 ? row.durationMs : null, + startedAt: Number.isFinite(Date.parse(row.startedAt)) ? new Date(row.startedAt).toISOString() : null, + finishedAt: Number.isFinite(Date.parse(row.finishedAt)) ? new Date(row.finishedAt).toISOString() : null, + })), + attemptAccounting: accounting ? { scope: accounting.scope, missingAttempts: accounting.missingAttempts, + excludedArtifacts: accounting.excludedArtifacts } : null, note: result && !missingArtifact ? 'Original full-stack assertions, attempts and diagnostics remain in the linked workflow report.' : 'No bound full-stack result was retained. This case is unverified and cannot pass.', }; diff --git a/tests/runner-e2e/product-regression-testops.test.mjs b/tests/runner-e2e/product-regression-testops.test.mjs index 8b1f40d6de9..746dd16d6e8 100644 --- a/tests/runner-e2e/product-regression-testops.test.mjs +++ b/tests/runner-e2e/product-regression-testops.test.mjs @@ -2,7 +2,7 @@ import test from 'node:test'; import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; import { productRegressionCases, selectProductRegression, validateSelection, projectProductResults, workflowUrl } from './product-regression-testops.mjs'; -import { admitProductTestOps, officialProductContext, verifyProductJob } from './product-regression-testops-cli.mjs'; +import { admitProductTestOps, officialProductContext, verifyProductJob, validateUploadedProductResults } from './product-regression-testops-cli.mjs'; const workflowSha = 'a'.repeat(40), targetSha = 'b'.repeat(40); const env = { @@ -88,17 +88,33 @@ test('missing results remain explicit broken cases and cannot improve the denomi assert.throws(() => projectProductResults(campaign([{ ...missing, status: 'passed' }]), selection([selected]), targetSha)); }); -test('original functional and infrastructure failures survive, and earlier failed retries remain in the export', () => { +test('original functional and infrastructure failures survive', () => { for (const [failureClass, expected] of [['candidate_failure', 'failed'], ['permanent_infrastructure', 'broken'], ['transient_infrastructure', 'broken']]) { const projected = projectProductResults(campaign([result({ status: 'failed', failureClass })]), selection([selected]), targetSha); assert.equal(projected.records[0].value.status, expected); } - const projected = projectProductResults(campaign([result({ status: 'failed', failureClass: 'candidate_failure' }), result({ attempt: 2 })]), selection([selected]), targetSha); - assert.deepEqual(projected.records.filter(row => row.file.endsWith('-result.json')).map(row => row.value.status), ['failed', 'passed']); - assert.equal(projected.summary.passed, 1); assert.throws(() => projectProductResults(campaign([result(), result()]), selection([selected]), targetSha)); }); +test('authoritative normalized accounting preserves earlier failed attempt verdicts beside the selected result', () => { + const measured = { ...campaign([result({ attempt: 2 })]), accounting: { version: 1, scope: 'retained_attempts', + missingAttempts: 0, excludedArtifacts: 0, attempts: [ + { id: 'retained-failure', executionId: selected.executionId, attempt: 1, status: 'failed', cleanup: 'passed', durationMs: 1000 }, + { id: 'selected-success', executionId: selected.executionId, attempt: 2, status: 'passed', cleanup: 'passed', durationMs: 1000 }, + ] } }; + const projected = projectProductResults(measured, selection([selected]), targetSha); + assert.equal(projected.summary.passed, 1); + assert.equal(projected.records.filter(row => row.file.endsWith('-result.json')).length, 1); + assert.deepEqual(projected.records[1].value.retainedAttempts.map(row => row.status), ['failed', 'passed']); + assert.deepEqual(projected.records[1].value.attemptAccounting, { scope: 'retained_attempts', missingAttempts: 0, excludedArtifacts: 0 }); + assert.throws(() => projectProductResults({ ...measured, accounting: { ...measured.accounting, attempts: [{ ...measured.accounting.attempts[0], status: 'unknown' }] } }, selection([selected]), targetSha)); + const openCode = productRegressionCases.find(row => row.id === 54); + const model = 'openrouter/deepseek/deepseek-v4.1-flash-1017'; + const record = result({ executionId: openCode.executionId, suiteId: openCode.suite, caseId: openCode.name, profileId: openCode.profile, model }); + const openCodeCampaign = { ...campaign([record]), expected: [openCode.executionId] }; + assert.equal(projectProductResults(openCodeCampaign, selection([openCode]), targetSha).records[1].value.model, model); +}); + test('the transmission projection excludes raw errors, provider IDs, narratives and unexpected fields', () => { const dirty = result({ error: 'secret-error', runIds: ['private-provider-session'], detail: 'private-context', usage: { credential: 'private-token' }, matcherResults: [{ passed: true, detail: 'secret-detail', @@ -117,6 +133,14 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: 'https://github.com/other/repo' } : values[route], { jobRunId: '456', launchId: 789 }, env)); }); +test('server verification accepts real Allure string statuses and rejects wrong verdicts or identities', () => { + const rows = [{ testCaseId: selected.id, fullName: selected.fullName, projectId: 1, launchId: 789, jobRun: { id: 456 }, status: 'passed' }]; + const summary = { passed: 1, failed: 0, broken: 0 }; + validateUploadedProductResults(rows, selection([selected]), summary); + assert.throws(() => validateUploadedProductResults([{ ...rows[0], status: 'failed' }], selection([selected]), summary)); + assert.throws(() => validateUploadedProductResults([{ ...rows[0], jobRun: { id: 999 } }], selection([selected]), summary)); +}); + test('workflow keeps reporting credentials and trusted code outside candidate/provider jobs', async () => { const workflow = await readFile(new URL('../../.github/workflows/runner-full-stack-e2e.yml', import.meta.url), 'utf8'); const selectionJob = workflow.split(' testops_selection:\n')[1].split('\n catalog:\n')[0];