diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 83123c6e8a2..6b7a1c17c98 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3077,6 +3077,13 @@ and source-readiness enums. Extra fields are rejected. Build this projection from authenticated accounting metadata, preserving unknown allowances and the original qualification scope. Source review never supplies a result grade. The execution-record column distinguishes candidate measurements from never-run cases without changing main qualification. +Allowance certainty is independent of a recorded measurement: a main pass or fail +can retain `allowance: "unknown"`. Known exhausted rows bound the accounting total; +unknown allowances can account for its remaining uncertainty. The optional +`latestCandidateOutcome` enum (`passed`, `failed`, or `not-projected`) can record a +later candidate measurement alongside a main outcome. It never changes +`mainOutcome`, main counters, or the qualification scope. Omit it when no such +additional outcome is projected; existing projections render unchanged. Expandable human requests and expected outcomes come only from the static YAML catalog, not from retained conversations or private artifacts. The renderer requires a new output directory and makes it private; it does not diff --git a/tests/runner-e2e/catalog-overview.test.ts b/tests/runner-e2e/catalog-overview.test.ts index 3baeb9206ec..e8c7da68ff2 100644 --- a/tests/runner-e2e/catalog-overview.test.ts +++ b/tests/runner-e2e/catalog-overview.test.ts @@ -62,6 +62,30 @@ describe("display-only catalog overview", () => { expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1'); expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded'); }); + it("preserves a valid main outcome when its allowance is unknown", () => { + const data=fixture(); data.rows[0]!.allowance="unknown"; + expect(parseCatalogOverview(data).accounting.exhaustedFinals).toBe(102); + const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[0]!.scenarioId)!; + expect(row.textContent).toContain('Recorded pass'); + expect(row.textContent).toContain('Main measurement recorded'); + expect(row.textContent).toContain('Allowance unknown'); + expect(row.textContent).toContain('Recover ordinal evidence'); + expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes'); + expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,exhaustedFinals:104}})).toThrow('Inconsistent catalog accounting'); + }); + it("shows a later candidate pass without promoting the recorded main failure", () => { + const data=fixture(); data.rows[25]!.latestCandidateOutcome="passed"; + const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[25]!.scenarioId)!; + expect(row.getAttribute('data-outcome')).toBe('failed'); + expect(row.textContent).toContain('Recorded fail'); + expect(row.textContent).toContain('Main measurement recorded · Latest candidate run: PASS'); + expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes144Latest main failures75Without a main measurement'); + expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('26 cases measured outside main'); + expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0); + expect(parseCatalogOverview(data).rows[25]!.mainOutcome).toBe('failed'); + expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===25?{...r,latestCandidateOutcome:'made-up'}:r)})).toThrow('Invalid catalog overview state'); + expect(() => parseCatalogOverview({...fixture(),rows:fixture().rows.map((r,i)=>i===169?{...r,latestCandidateOutcome:'passed'}:r)})).toThrow('Inconsistent recorded'); + }); it.each([[152,"below"],[196,"equal to"],[200,"above"]] as const)("compares a ceiling of %i accurately", (ceiling, comparison) => { const data=fixture(); data.accounting.theoreticalMainCeiling=ceiling; expect(new JSDOM(page(data)).window.document.querySelector('main')!.textContent).toContain(`${ceiling}/244, ${comparison} the 196-case target`); diff --git a/tests/runner-e2e/catalog-overview.ts b/tests/runner-e2e/catalog-overview.ts index c2023ec5b06..a3dbcd4c26b 100644 --- a/tests/runner-e2e/catalog-overview.ts +++ b/tests/runner-e2e/catalog-overview.ts @@ -3,6 +3,7 @@ import { html } from "./report-format.js"; const outcomes = { passed: "Recorded pass", failed: "Recorded fail", unmeasured: "No main measurement" } as const; const allowances = { "unused-initial": "Initial unused", "final-available": "Final available", exhausted: "Final consumed", unknown: "Allowance unknown" } as const; const executionRecords = { "main-measured": "Main measurement recorded", "candidate-measured": "Candidate measured · outcome not projected", "candidate-failed": "Candidate run: FAIL", "candidate-passed": "Candidate run: PASS", "never-run": "Never run", unknown: "Execution evidence unknown" } as const; +const candidateOutcomes = { passed: "Latest candidate run: PASS", failed: "Latest candidate run: FAIL", "not-projected": "Latest candidate outcome not projected" } as const; const readiness = { "not-assessed": "Not assessed here", "reviewed-candidate": "Candidate source reviewed", "needs-preparation": "Preparation needed" } as const; export interface CatalogOverviewRow { @@ -11,6 +12,7 @@ export interface CatalogOverviewRow { allowance: keyof typeof allowances; sourceReadiness: keyof typeof readiness; executionRecord: keyof typeof executionRecords; + latestCandidateOutcome?: keyof typeof candidateOutcomes; } export interface CatalogOverview { @@ -21,10 +23,10 @@ export interface CatalogOverview { rows: CatalogOverviewRow[]; } -function object(value: unknown, keys: readonly string[]): Record { +function object(value: unknown, keys: readonly string[], optional: readonly string[] = []): Record { if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid catalog overview object"); const row = value as Record; - if (Object.keys(row).length !== keys.length || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields"); + if (Object.keys(row).some(key => !keys.includes(key) && !optional.includes(key)) || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields"); return row; } function integer(value: unknown): number { @@ -46,16 +48,20 @@ export function parseCatalogOverview(value: unknown): CatalogOverview { const accounting = { mainMeasurements: integer(a.mainMeasurements), retainedRawResults: integer(a.retainedRawResults), closedSlots: integer(a.closedSlots), exhaustedFinals: integer(a.exhaustedFinals), theoreticalMainCeiling: integer(a.theoreticalMainCeiling) }; if (!Array.isArray(input.rows) || input.rows.length !== 244) throw new Error("Expected 244 catalog metadata rows"); const rows = input.rows.map(value => { - const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"]); + const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"], ["latestCandidateOutcome"]); if (typeof row.scenarioId !== "string" || !/^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$/.test(row.scenarioId)) throw new Error("Invalid public catalog ID"); - return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords) }; + return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords), + ...(Object.hasOwn(row, "latestCandidateOutcome") ? { latestCandidateOutcome: choice(row.latestCandidateOutcome, candidateOutcomes) } : {}) }; }); if (new Set(rows.map(row => row.scenarioId)).size !== 244 || new Set(rows.map(row => row.scenarioId.split(".")[0])).size !== 35) throw new Error("Duplicate or incomplete catalog metadata"); for (const row of rows) { - if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance === "unused-initial") !== (row.executionRecord === "never-run") || (row.allowance === "unknown") !== (row.executionRecord === "unknown")) throw new Error("Inconsistent recorded execution scope"); + if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance !== "unknown" && (row.allowance === "unused-initial") !== (row.executionRecord === "never-run")) + || (row.latestCandidateOutcome !== undefined && ["never-run", "unknown"].includes(row.executionRecord))) throw new Error("Inconsistent recorded execution scope"); } const measured = rows.filter(row => row.mainOutcome !== "unmeasured").length; - if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals !== rows.filter(row => row.allowance === "exhausted").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting"); + const knownExhausted = rows.filter(row => row.allowance === "exhausted").length; + if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals < knownExhausted + || accounting.exhaustedFinals > knownExhausted + rows.filter(row => row.allowance === "unknown").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting"); return { schema: "paperclip.catalog-overview/v2", asOf: input.asOf, definitions, accounting, rows }; } @@ -70,6 +76,9 @@ function nextAction(row: CatalogOverviewRow) { if (row.sourceReadiness === "needs-preparation") return "Prepare source, then review execution gates"; return row.allowance === "unused-initial" ? "Review source and execution gates for the initial" : "Review the initial cause and final execution gates"; } +function executionLabel(row: CatalogOverviewRow) { + return executionRecords[row.executionRecord] + (row.latestCandidateOutcome ? ` · ${candidateOutcomes[row.latestCandidateOutcome]}` : ""); +} const exampleIds = ["projects.create-project", "projects.archive-project", "tasks.edit-properties", "tasks.comment-with-reference", "documents-artifacts.write-plan", "documents-artifacts.produce-download", "inbox-search.restore-archive"]; export interface CatalogScenarioDescription { query: string; expectedOutcome: string } @@ -84,12 +93,12 @@ export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<

Chat feature catalog · private preview

All 244 scenarios

Recorded outcomes, remaining allowances and source readiness, shown separately.

Snapshot ${html(data.asOf)}

${count("passed")}Recorded feature-work passes
${count("failed")}Latest main failures
${count("unmeasured")}Without a main measurement

${count("passed") + count("failed")} main cases · ${data.accounting.mainMeasurements} main measurements. The ${count("unmeasured")} without a main measurement include ${data.rows.filter(row => row.allowance === "unused-initial").length} unused initials; these are different sets.

-

${data.rows.filter(row => row.executionRecord.startsWith("candidate-")).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.

+

${data.rows.filter(row => row.executionRecord.startsWith("candidate-") || row.latestCandidateOutcome !== undefined).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.

${examples.length ? `

Passing feature examples

${examples.map(id => `${html(label(id.split(".")[1]!))}`).join(" · ")}

` : ""}
Attempt accounting and limits

${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.

The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, ${ceilingComparison} the 196-case target. This is an upper bound, not a forecast or an execution approval.

244 scenarios

-
${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => ``).join("")}
ScenarioMain outcomeRecorded executionAllowanceSource readinessNext action
${html(label(row.scenarioId.split(".")[1]!))}${html(label(row.scenarioId.split(".")[0]!))}${html(row.scenarioId)}${descriptions.has(row.scenarioId) ? `
What this test asks

Human request: ${html(descriptions.get(row.scenarioId)!.query)}

Expected outcome: ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}

` : ""}		${outcomes[row.mainOutcome]}${executionRecords[row.executionRecord]}${allowances[row.allowance]}${readiness[row.sourceReadiness]}${html(nextAction(row))}
+
${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => ``).join("")}
ScenarioMain outcomeRecorded executionAllowanceSource readinessNext action
${html(label(row.scenarioId.split(".")[1]!))}${html(label(row.scenarioId.split(".")[0]!))}${html(row.scenarioId)}${descriptions.has(row.scenarioId) ? `
What this test asks

Human request: ${html(descriptions.get(row.scenarioId)!.query)}

Expected outcome: ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}

` : ""}		${outcomes[row.mainOutcome]}${executionLabel(row)}${allowances[row.allowance]}${readiness[row.sourceReadiness]}${html(nextAction(row))}
`; }