fix(evals): separate candidate outcomes and allowance certainty

tests/runner-e2e/README.md

@@ -3077,6 +3077,13 @@ and source-readiness enums. Extra fields are rejected. Build this projection
 from authenticated accounting metadata, preserving unknown allowances and the
 original qualification scope. Source review never supplies a result grade. The execution-record column distinguishes
 candidate measurements from never-run cases without changing main qualification.
+Allowance certainty is independent of a recorded measurement: a main pass or fail
+can retain `allowance: "unknown"`. Known exhausted rows bound the accounting total;
+unknown allowances can account for its remaining uncertainty. The optional
+`latestCandidateOutcome` enum (`passed`, `failed`, or `not-projected`) can record a
+later candidate measurement alongside a main outcome. It never changes
+`mainOutcome`, main counters, or the qualification scope. Omit it when no such
+additional outcome is projected; existing projections render unchanged.
 Expandable human requests and expected outcomes come only from the static YAML catalog,
 not from retained conversations or private artifacts.
 The renderer requires a new output directory and makes it private; it does not

tests/runner-e2e/catalog-overview.test.ts

@@ -62,6 +62,30 @@ describe("display-only catalog overview", () => {
     expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1');
     expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded');
   });
+  it("preserves a valid main outcome when its allowance is unknown", () => {
+    const data=fixture(); data.rows[0]!.allowance="unknown";
+    expect(parseCatalogOverview(data).accounting.exhaustedFinals).toBe(102);
+    const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[0]!.scenarioId)!;
+    expect(row.textContent).toContain('Recorded pass');
+    expect(row.textContent).toContain('Main measurement recorded');
+    expect(row.textContent).toContain('Allowance unknown');
+    expect(row.textContent).toContain('Recover ordinal evidence');
+    expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes');
+    expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,exhaustedFinals:104}})).toThrow('Inconsistent catalog accounting');
+  });
+  it("shows a later candidate pass without promoting the recorded main failure", () => {
+    const data=fixture(); data.rows[25]!.latestCandidateOutcome="passed";
+    const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[25]!.scenarioId)!;
+    expect(row.getAttribute('data-outcome')).toBe('failed');
+    expect(row.textContent).toContain('Recorded fail');
+    expect(row.textContent).toContain('Main measurement recorded · Latest candidate run: PASS');
+    expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes144Latest main failures75Without a main measurement');
+    expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('26 cases measured outside main');
+    expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0);
+    expect(parseCatalogOverview(data).rows[25]!.mainOutcome).toBe('failed');
+    expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===25?{...r,latestCandidateOutcome:'made-up'}:r)})).toThrow('Invalid catalog overview state');
+    expect(() => parseCatalogOverview({...fixture(),rows:fixture().rows.map((r,i)=>i===169?{...r,latestCandidateOutcome:'passed'}:r)})).toThrow('Inconsistent recorded');
+  });
   it.each([[152,"below"],[196,"equal to"],[200,"above"]] as const)("compares a ceiling of %i accurately", (ceiling, comparison) => {
     const data=fixture(); data.accounting.theoreticalMainCeiling=ceiling;
     expect(new JSDOM(page(data)).window.document.querySelector('main')!.textContent).toContain(`${ceiling}/244, ${comparison} the 196-case target`);

tests/runner-e2e/catalog-overview.ts

@@ -3,6 +3,7 @@ import { html } from "./report-format.js";
 const outcomes = { passed: "Recorded pass", failed: "Recorded fail", unmeasured: "No main measurement" } as const;
 const allowances = { "unused-initial": "Initial unused", "final-available": "Final available", exhausted: "Final consumed", unknown: "Allowance unknown" } as const;
 const executionRecords = { "main-measured": "Main measurement recorded", "candidate-measured": "Candidate measured · outcome not projected", "candidate-failed": "Candidate run: FAIL", "candidate-passed": "Candidate run: PASS", "never-run": "Never run", unknown: "Execution evidence unknown" } as const;
+const candidateOutcomes = { passed: "Latest candidate run: PASS", failed: "Latest candidate run: FAIL", "not-projected": "Latest candidate outcome not projected" } as const;
 const readiness = { "not-assessed": "Not assessed here", "reviewed-candidate": "Candidate source reviewed", "needs-preparation": "Preparation needed" } as const;
 
 export interface CatalogOverviewRow {
@@ -11,6 +12,7 @@ export interface CatalogOverviewRow {
   allowance: keyof typeof allowances;
   sourceReadiness: keyof typeof readiness;
   executionRecord: keyof typeof executionRecords;
+  latestCandidateOutcome?: keyof typeof candidateOutcomes;
 }
 
 export interface CatalogOverview {
@@ -21,10 +23,10 @@ export interface CatalogOverview {
   rows: CatalogOverviewRow[];
 }
 
-function object(value: unknown, keys: readonly string[]): Record<string, unknown> {
+function object(value: unknown, keys: readonly string[], optional: readonly string[] = []): Record<string, unknown> {
   if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid catalog overview object");
   const row = value as Record<string, unknown>;
-  if (Object.keys(row).length !== keys.length || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields");
+  if (Object.keys(row).some(key => !keys.includes(key) && !optional.includes(key)) || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields");
   return row;
 }
 function integer(value: unknown): number {
@@ -46,16 +48,20 @@ export function parseCatalogOverview(value: unknown): CatalogOverview {
   const accounting = { mainMeasurements: integer(a.mainMeasurements), retainedRawResults: integer(a.retainedRawResults), closedSlots: integer(a.closedSlots), exhaustedFinals: integer(a.exhaustedFinals), theoreticalMainCeiling: integer(a.theoreticalMainCeiling) };
   if (!Array.isArray(input.rows) || input.rows.length !== 244) throw new Error("Expected 244 catalog metadata rows");
   const rows = input.rows.map(value => {
-    const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"]);
+    const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"], ["latestCandidateOutcome"]);
     if (typeof row.scenarioId !== "string" || !/^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$/.test(row.scenarioId)) throw new Error("Invalid public catalog ID");
-    return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords) };
+    return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords),
+      ...(Object.hasOwn(row, "latestCandidateOutcome") ? { latestCandidateOutcome: choice(row.latestCandidateOutcome, candidateOutcomes) } : {}) };
   });
   if (new Set(rows.map(row => row.scenarioId)).size !== 244 || new Set(rows.map(row => row.scenarioId.split(".")[0])).size !== 35) throw new Error("Duplicate or incomplete catalog metadata");
   for (const row of rows) {
-    if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance === "unused-initial") !== (row.executionRecord === "never-run") || (row.allowance === "unknown") !== (row.executionRecord === "unknown")) throw new Error("Inconsistent recorded execution scope");
+    if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance !== "unknown" && (row.allowance === "unused-initial") !== (row.executionRecord === "never-run"))
+      || (row.latestCandidateOutcome !== undefined && ["never-run", "unknown"].includes(row.executionRecord))) throw new Error("Inconsistent recorded execution scope");
   }
   const measured = rows.filter(row => row.mainOutcome !== "unmeasured").length;
-  if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals !== rows.filter(row => row.allowance === "exhausted").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting");
+  const knownExhausted = rows.filter(row => row.allowance === "exhausted").length;
+  if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals < knownExhausted
+    || accounting.exhaustedFinals > knownExhausted + rows.filter(row => row.allowance === "unknown").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting");
   return { schema: "paperclip.catalog-overview/v2", asOf: input.asOf, definitions, accounting, rows };
 }
 
@@ -70,6 +76,9 @@ function nextAction(row: CatalogOverviewRow) {
   if (row.sourceReadiness === "needs-preparation") return "Prepare source, then review execution gates";
   return row.allowance === "unused-initial" ? "Review source and execution gates for the initial" : "Review the initial cause and final execution gates";
 }
+function executionLabel(row: CatalogOverviewRow) {
+  return executionRecords[row.executionRecord] + (row.latestCandidateOutcome ? ` · ${candidateOutcomes[row.latestCandidateOutcome]}` : "");
+}
 const exampleIds = ["projects.create-project", "projects.archive-project", "tasks.edit-properties", "tasks.comment-with-reference", "documents-artifacts.write-plan", "documents-artifacts.produce-download", "inbox-search.restore-archive"];
 
 export interface CatalogScenarioDescription { query: string; expectedOutcome: string }
@@ -84,12 +93,12 @@ export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<
     <header class="report-header"><div><p class="eyebrow">Chat feature catalog · private preview</p><h1 tabindex="-1">All 244 scenarios</h1><p class="lede">Recorded outcomes, remaining allowances and source readiness, shown separately.</p><p class="detail">Snapshot ${html(data.asOf)}</p></div></header>
     <div class="summary" aria-label="Catalog measurement summary"><div class="metric"><strong>${count("passed")}</strong><span>Recorded feature-work passes</span></div><div class="metric"><strong>${count("failed")}</strong><span>Latest main failures</span></div><div class="metric"><strong>${count("unmeasured")}</strong><span>Without a main measurement</span></div></div>
     <p class="detail">${count("passed") + count("failed")} main cases · ${data.accounting.mainMeasurements} main measurements. The ${count("unmeasured")} without a main measurement include ${data.rows.filter(row => row.allowance === "unused-initial").length} unused initials; these are different sets.</p>
-    <p class="detail" data-execution-scope>${data.rows.filter(row => row.executionRecord.startsWith("candidate-")).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.</p>
+    <p class="detail" data-execution-scope>${data.rows.filter(row => row.executionRecord.startsWith("candidate-") || row.latestCandidateOutcome !== undefined).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.</p>
     <aside class="catalog-scope" aria-label="Qualification scope"><strong>What these outcomes establish</strong><p>The recorded passes support their original feature-work checks. Fully native auxiliary setup and complete usage qualification are not established. Unknown usage and cost are unknown, never zero. Candidate outcomes do not add ordinary main credit.</p><p>${data.definitions.lazy} lazy definitions plus ${data.definitions.legacy} implemented legacy definitions cover 35 feature areas and 191 capabilities. The ${data.definitions.declaredGaps} declared source gaps describe coverage scope, not live results. Source readiness is a partial source review inventory; it does not change a recorded grade or grant execution approval.</p></aside>
     ${examples.length ? `<section aria-label="Recorded passing examples"><h2>Passing feature examples</h2><p class="evidence-links">${examples.map(id => `<a href="#catalog-${html(id)}">${html(label(id.split(".")[1]!))}</a>`).join(" · ")}</p></section>` : ""}
     <details class="campaign-billing-details"><summary>Attempt accounting and limits</summary><p>${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.</p><p>The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, ${ceilingComparison} the 196-case target. This is an upper bound, not a forecast or an execution approval.</p></details>
     <section class="catalog-filters report-filters" aria-label="Filter catalog metadata"><label class="report-search"><span>Search scenarios</span><input type="search" data-catalog-query placeholder="Scenario or feature area"></label><label><span>Feature area</span><select data-catalog-feature><option value="">All feature areas</option>${features.map(feature => `<option value="${html(feature)}">${html(label(feature))}</option>`).join("")}</select></label><label><span>Main outcome</span><select data-catalog-outcome><option value="">All outcomes</option>${Object.entries(outcomes).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><label><span>Allowance</span><select data-catalog-allowance><option value="">All allowances</option>${Object.entries(allowances).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><button class="filter-reset" type="button" data-catalog-reset>Reset filters</button><p data-catalog-count aria-live="polite">244 scenarios</p></section>
-    <div class="table-wrap"><table class="catalog-table"><thead><tr><th scope="col">Scenario</th><th scope="col">Main outcome</th><th scope="col">Recorded execution</th><th scope="col">Allowance</th><th scope="col">Source readiness</th><th scope="col">Next action</th></tr></thead><tbody>${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => `<tr id="catalog-${html(row.scenarioId)}" data-catalog-row data-feature="${html(row.scenarioId.split(".")[0])}" data-outcome="${row.mainOutcome}" data-allowance="${row.allowance}"><th scope="row"><strong>${html(label(row.scenarioId.split(".")[1]!))}</strong><small>${html(label(row.scenarioId.split(".")[0]!))}</small><code>${html(row.scenarioId)}</code>${descriptions.has(row.scenarioId) ? `<details><summary>What this test asks</summary><p><strong>Human request:</strong> ${html(descriptions.get(row.scenarioId)!.query)}</p><p><strong>Expected outcome:</strong> ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}</p></details>` : ""}</th><td>${outcomes[row.mainOutcome]}</td><td>${executionRecords[row.executionRecord]}</td><td>${allowances[row.allowance]}</td><td>${readiness[row.sourceReadiness]}</td><td>${html(nextAction(row))}</td></tr>`).join("")}</tbody></table></div>
+    <div class="table-wrap"><table class="catalog-table"><thead><tr><th scope="col">Scenario</th><th scope="col">Main outcome</th><th scope="col">Recorded execution</th><th scope="col">Allowance</th><th scope="col">Source readiness</th><th scope="col">Next action</th></tr></thead><tbody>${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => `<tr id="catalog-${html(row.scenarioId)}" data-catalog-row data-feature="${html(row.scenarioId.split(".")[0])}" data-outcome="${row.mainOutcome}" data-allowance="${row.allowance}"><th scope="row"><strong>${html(label(row.scenarioId.split(".")[1]!))}</strong><small>${html(label(row.scenarioId.split(".")[0]!))}</small><code>${html(row.scenarioId)}</code>${descriptions.has(row.scenarioId) ? `<details><summary>What this test asks</summary><p><strong>Human request:</strong> ${html(descriptions.get(row.scenarioId)!.query)}</p><p><strong>Expected outcome:</strong> ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}</p></details>` : ""}</th><td>${outcomes[row.mainOutcome]}</td><td>${executionLabel(row)}</td><td>${allowances[row.allowance]}</td><td>${readiness[row.sourceReadiness]}</td><td>${html(nextAction(row))}</td></tr>`).join("")}</tbody></table></div>
     <footer>Metadata overview only · No result cards or evidence files are synthesized</footer>
   </section>`;
 }