test(evals): retain authoritative attempt verdict receipts

Use normalized campaign accounting for earlier failed attempts. Preserve provider model identifiers with slashes and validate the actual string status returned by TestOps. Cover missing-artifact placeholder rows and server-side status/identity controls.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/product-regression-testops-cli.mjs

@@ -52,6 +52,16 @@ export async function verifyProductJob(api, context, env) {
   return { jobId: job.id, launchId: launch.id };
 }
 
+export function validateUploadedProductResults(details, selection, summary) {
+  if (details.length !== selection.cases.length || new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => {
+    const selected = selection.cases.find(value => value.id === row.testCaseId);
+    return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId
+      || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status);
+  })) fail('uploaded_identity_mismatch');
+  const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status === status).length]));
+  if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch');
+}
+
 export async function productTestOps(command, argument, env = process.env, exec = execute, fetchImpl = fetch) {
   admitProductTestOps(env);
   if (command === 'select') {
@@ -114,13 +124,7 @@ export async function productTestOps(command, argument, env = process.env, exec
     await new Promise(resolve => setTimeout(resolve, 2_000));
   }
   const details = await Promise.all(observed.content.map(row => api(`/api/testresult/${row.id}`)));
-  if (new Set(details.map(row => row.testCaseId)).size !== selection.cases.length || details.some(row => {
-    const selected = selection.cases.find(value => value.id === row.testCaseId);
-    return !selected || row.fullName !== selected.fullName || row.projectId !== 1 || row.launchId !== selection.launchId
-      || row.jobRun?.id !== Number(selection.jobRunId) || !['passed', 'failed', 'broken'].includes(row.status?.name);
-  })) fail('uploaded_identity_mismatch');
-  const counts = Object.fromEntries(['passed', 'failed', 'broken'].map(status => [status, details.filter(row => row.status.name === status).length]));
-  if (Object.entries(counts).some(([status, count]) => summary[status] !== count)) fail('uploaded_status_mismatch');
+  validateUploadedProductResults(details, selection, summary);
   return 0;
 }
 

tests/runner-e2e/product-regression-testops.mjs

@@ -73,7 +73,16 @@ export function projectProductResults(campaign, selection, sourceSha) {
       || !Array.isArray(campaign.results) || !Array.isArray(campaign.expected)
       || JSON.stringify([...campaign.expected].sort()) !== JSON.stringify(selection.cases.map(row => row.executionId).sort()))) fail('report_binding');
   const rows = campaign?.results ?? [];
-  if (rows.length > selection.cases.length * 10 || rows.some(row => !selection.cases.some(value => value.executionId === row.executionId))) fail('unexpected_result');
+  if (rows.length > selection.cases.length || new Set(rows.map(row => row.executionId)).size !== rows.length
+      || rows.some(row => !selection.cases.some(value => value.executionId === row.executionId))) fail('unexpected_result');
+  const accounting = campaign?.accounting;
+  if (accounting && (accounting.version !== 1 || !['retained_attempts', 'selected_results'].includes(accounting.scope)
+      || !Array.isArray(accounting.attempts) || accounting.attempts.length > selection.cases.length * 10
+      || !Number.isSafeInteger(accounting.missingAttempts) || accounting.missingAttempts < 0
+      || !Number.isSafeInteger(accounting.excludedArtifacts) || accounting.excludedArtifacts < 0
+      || accounting.attempts.some(row => !selection.cases.some(value => value.executionId === row.executionId)
+        || !Number.isSafeInteger(row.attempt) || row.attempt < 1 || row.attempt > 10
+        || !['passed', 'failed'].includes(row.status) || !['passed', 'failed', 'not_started'].includes(row.cleanup)))) fail('accounting_invalid');
   const records = [], summaries = [];
   for (const selected of selection.cases) {
     const attempts = rows.filter(row => row.executionId === selected.executionId).sort((a, b) => a.attempt - b.attempt);
@@ -108,6 +117,14 @@ export function projectProductResults(campaign, selection, sourceSha) {
           name: identifier(check.matcher?.path) ? check.matcher.path : identifier(check.matcher?.kind) ? check.matcher.kind : 'assertion',
           passed: check.passed,
         })),
+        retainedAttempts: (accounting?.attempts ?? []).filter(row => row.executionId === selected.executionId).map(row => ({
+          attempt: row.attempt, status: row.status, cleanup: row.cleanup,
+          durationMs: Number.isFinite(row.durationMs) && row.durationMs >= 0 ? row.durationMs : null,
+          startedAt: Number.isFinite(Date.parse(row.startedAt)) ? new Date(row.startedAt).toISOString() : null,
+          finishedAt: Number.isFinite(Date.parse(row.finishedAt)) ? new Date(row.finishedAt).toISOString() : null,
+        })),
+        attemptAccounting: accounting ? { scope: accounting.scope, missingAttempts: accounting.missingAttempts,
+          excludedArtifacts: accounting.excludedArtifacts } : null,
         note: result && !missingArtifact ? 'Original full-stack assertions, attempts and diagnostics remain in the linked workflow report.'
           : 'No bound full-stack result was retained. This case is unverified and cannot pass.',
       };

tests/runner-e2e/product-regression-testops.test.mjs

@@ -2,7 +2,7 @@ import test from 'node:test';
 import assert from 'node:assert/strict';
 import { readFile } from 'node:fs/promises';
 import { productRegressionCases, selectProductRegression, validateSelection, projectProductResults, workflowUrl } from './product-regression-testops.mjs';
-import { admitProductTestOps, officialProductContext, verifyProductJob } from './product-regression-testops-cli.mjs';
+import { admitProductTestOps, officialProductContext, verifyProductJob, validateUploadedProductResults } from './product-regression-testops-cli.mjs';
 
 const workflowSha = 'a'.repeat(40), targetSha = 'b'.repeat(40);
 const env = {
@@ -88,17 +88,33 @@ test('missing results remain explicit broken cases and cannot improve the denomi
   assert.throws(() => projectProductResults(campaign([{ ...missing, status: 'passed' }]), selection([selected]), targetSha));
 });
 
-test('original functional and infrastructure failures survive, and earlier failed retries remain in the export', () => {
+test('original functional and infrastructure failures survive', () => {
   for (const [failureClass, expected] of [['candidate_failure', 'failed'], ['permanent_infrastructure', 'broken'], ['transient_infrastructure', 'broken']]) {
     const projected = projectProductResults(campaign([result({ status: 'failed', failureClass })]), selection([selected]), targetSha);
     assert.equal(projected.records[0].value.status, expected);
   }
-  const projected = projectProductResults(campaign([result({ status: 'failed', failureClass: 'candidate_failure' }), result({ attempt: 2 })]), selection([selected]), targetSha);
-  assert.deepEqual(projected.records.filter(row => row.file.endsWith('-result.json')).map(row => row.value.status), ['failed', 'passed']);
-  assert.equal(projected.summary.passed, 1);
   assert.throws(() => projectProductResults(campaign([result(), result()]), selection([selected]), targetSha));
 });
 
+test('authoritative normalized accounting preserves earlier failed attempt verdicts beside the selected result', () => {
+  const measured = { ...campaign([result({ attempt: 2 })]), accounting: { version: 1, scope: 'retained_attempts',
+    missingAttempts: 0, excludedArtifacts: 0, attempts: [
+      { id: 'retained-failure', executionId: selected.executionId, attempt: 1, status: 'failed', cleanup: 'passed', durationMs: 1000 },
+      { id: 'selected-success', executionId: selected.executionId, attempt: 2, status: 'passed', cleanup: 'passed', durationMs: 1000 },
+    ] } };
+  const projected = projectProductResults(measured, selection([selected]), targetSha);
+  assert.equal(projected.summary.passed, 1);
+  assert.equal(projected.records.filter(row => row.file.endsWith('-result.json')).length, 1);
+  assert.deepEqual(projected.records[1].value.retainedAttempts.map(row => row.status), ['failed', 'passed']);
+  assert.deepEqual(projected.records[1].value.attemptAccounting, { scope: 'retained_attempts', missingAttempts: 0, excludedArtifacts: 0 });
+  assert.throws(() => projectProductResults({ ...measured, accounting: { ...measured.accounting, attempts: [{ ...measured.accounting.attempts[0], status: 'unknown' }] } }, selection([selected]), targetSha));
+  const openCode = productRegressionCases.find(row => row.id === 54);
+  const model = 'openrouter/deepseek/deepseek-v4.1-flash-1017';
+  const record = result({ executionId: openCode.executionId, suiteId: openCode.suite, caseId: openCode.name, profileId: openCode.profile, model });
+  const openCodeCampaign = { ...campaign([record]), expected: [openCode.executionId] };
+  assert.equal(projectProductResults(openCodeCampaign, selection([openCode]), targetSha).records[1].value.model, model);
+});
+
 test('the transmission projection excludes raw errors, provider IDs, narratives and unexpected fields', () => {
   const dirty = result({ error: 'secret-error', runIds: ['private-provider-session'], detail: 'private-context',
     usage: { credential: 'private-token' }, matcherResults: [{ passed: true, detail: 'secret-detail',
@@ -117,6 +133,14 @@ test('a TestOps job run must belong to this workflow, source launch and GitHub e
   await assert.rejects(verifyProductJob(route => route === '/api/job/50' ? { ...values[route], url: 'https://github.com/other/repo' } : values[route], { jobRunId: '456', launchId: 789 }, env));
 });
 
+test('server verification accepts real Allure string statuses and rejects wrong verdicts or identities', () => {
+  const rows = [{ testCaseId: selected.id, fullName: selected.fullName, projectId: 1, launchId: 789, jobRun: { id: 456 }, status: 'passed' }];
+  const summary = { passed: 1, failed: 0, broken: 0 };
+  validateUploadedProductResults(rows, selection([selected]), summary);
+  assert.throws(() => validateUploadedProductResults([{ ...rows[0], status: 'failed' }], selection([selected]), summary));
+  assert.throws(() => validateUploadedProductResults([{ ...rows[0], jobRun: { id: 999 } }], selection([selected]), summary));
+});
+
 test('workflow keeps reporting credentials and trusted code outside candidate/provider jobs', async () => {
   const workflow = await readFile(new URL('../../.github/workflows/runner-full-stack-e2e.yml', import.meta.url), 'utf8');
   const selectionJob = workflow.split('  testops_selection:\n')[1].split('\n  catalog:\n')[0];