diff --git a/tests/runner-e2e/README.md b/tests/runner-e2e/README.md index 3820b8f9541..6b7a1c17c98 100644 --- a/tests/runner-e2e/README.md +++ b/tests/runner-e2e/README.md @@ -3061,3 +3061,36 @@ they are not reported as charges or converted into reviewed lifecycle bounds. This mode therefore does not certify a full-lifecycle operating-cost ceiling. Omitting the policy preserves the stricter full-accounting gate. Historical journals and their holds are not rewritten by changing future campaign policy. + +## Private catalog metadata preview + +The existing dashboard renderer can show a display-only overview without creating +result rows or discovering historical evidence: + +```sh +pnpm test:e2e:runner:dashboard -- --catalog-overview --output +``` + +The input must match `CatalogOverview` in `catalog-overview.ts`, contain exactly +the frozen 244 YAML scenario IDs, and use only the declared outcome, allowance +and source-readiness enums. Extra fields are rejected. Build this projection +from authenticated accounting metadata, preserving unknown allowances and the +original qualification scope. Source review never supplies a result grade. The execution-record column distinguishes +candidate measurements from never-run cases without changing main qualification. +Allowance certainty is independent of a recorded measurement: a main pass or fail +can retain `allowance: "unknown"`. Known exhausted rows bound the accounting total; +unknown allowances can account for its remaining uncertainty. The optional +`latestCandidateOutcome` enum (`passed`, `failed`, or `not-projected`) can record a +later candidate measurement alongside a main outcome. It never changes +`mainOutcome`, main counters, or the qualification scope. Omit it when no such +additional outcome is projected; existing projections render unchanged. +Expandable human requests and expected outcomes come only from the static YAML catalog, +not from retained conversations or private artifacts. +The renderer requires a new output directory and makes it private; it does not +publish the preview or modify existing campaign files. Normal campaign rendering +and retained result cards continue to use the existing bundle command. + +The shared report navigation links the dated, immutable catalog snapshot. +Deploying this generator does not rewrite existing published history HTML. +Activating the link on an existing page requires a separately reviewed, +conditional navigation-only update. The metadata command publishes nothing. diff --git a/tests/runner-e2e/catalog-overview.test.ts b/tests/runner-e2e/catalog-overview.test.ts new file mode 100644 index 00000000000..d390c3a55a1 --- /dev/null +++ b/tests/runner-e2e/catalog-overview.test.ts @@ -0,0 +1,151 @@ +import { describe, expect, it } from "vitest"; +import { readFileSync, readdirSync } from "node:fs"; +import { createRequire } from "node:module"; +const { JSDOM } = createRequire(new URL("../../server/package.json", import.meta.url))("jsdom") as typeof import("../../server/node_modules/@types/jsdom/index.js"); +import { mkdtemp, writeFile, stat, rm, readFile, symlink } from "node:fs/promises"; +import path from "node:path"; +import os from "node:os"; +import { regenerateCatalogOverview } from "./dashboard-regenerate.js"; +import { parseCatalogOverview, type CatalogOverview } from "./catalog-overview.js"; +import { renderRunnerE2EDashboard } from "./dashboard.js"; + +function fixture(): CatalogOverview { + const catalog = new URL("./chat-scenarios/", import.meta.url); + const ids = readdirSync(catalog).filter(name => name.endsWith(".yaml")).flatMap(name => [...readFileSync(new URL(name,catalog),"utf8").matchAll(/^ - id: ([a-z0-9.-]+)$/gm)].map(match => match[1]!)); + const example = ids.indexOf("projects.create-project"); [ids[0],ids[example]]=[ids[example]!,ids[0]!]; + const rows: CatalogOverview["rows"] = ids.map((scenarioId,i)=>({scenarioId, mainOutcome:i<25?"passed":i<169?"failed":"unmeasured",allowance:i<102?"exhausted":i>=169&&i<218?"unused-initial":i===243?"unknown":"final-available",sourceReadiness:"not-assessed",executionRecord:i<169?"main-measured":i<218?"never-run":i===243?"unknown":"candidate-measured"})); + return { schema:"paperclip.catalog-overview/v2", asOf:"2026-10-29T17:54:00Z", definitions:{lazy:242,legacy:2,featureAreas:35,capabilities:191,declaredGaps:5}, accounting:{mainMeasurements:215,retainedRawResults:295,closedSlots:296,exhaustedFinals:102,theoreticalMainCeiling:152}, rows }; +} +function page(data: CatalogOverview) { + return renderRunnerE2EDashboard({title:"Catalog",generatedAt:data.asOf,expected:[],catalog:[],entries:[],catalogOverview:data}); +} + +describe("display-only catalog overview", () => { + it("requires the fixed scope and rejects private carrier fields rather than serializing them", () => { + const data=fixture(); + expect(parseCatalogOverview(data).rows).toHaveLength(244); + expect(() => parseCatalogOverview({...data,privatePath:"/private/hidden"})).toThrow("Unexpected"); + expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i? r : {...r,errorBody:"secret"})})).toThrow("Unexpected"); + expect(() => parseCatalogOverview({...data,definitions:{...data.definitions,legacy:0}})).toThrow("Frozen"); + }); + it("rejects duplicate rows, invalid IDs/states, impossible counts and mixed real results", () => { + const data=fixture(); + expect(() => parseCatalogOverview({...data,rows:[...data.rows.slice(1),data.rows[1]]})).toThrow("Duplicate"); + expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i?r:{...r,scenarioId:''})})).toThrow("Invalid public"); + expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i?r:{...r,mainOutcome:'candidate-pass'})})).toThrow("Invalid"); + expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,mainMeasurements:2}})).toThrow("Inconsistent"); + expect(() => renderRunnerE2EDashboard({title:"mixed",generatedAt:data.asOf,expected:["real-case"],catalog:[],entries:[],catalogOverview:data})).toThrow("cannot be mixed"); + }); + it("renders metadata counts and scopes without fake result cards or evidence links", () => { + const data=fixture(); const doc=new JSDOM(page(data)).window.document; + expect(doc.querySelectorAll('[data-catalog-row]')).toHaveLength(244); + expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0); + expect(doc.querySelector('[data-report-scope]')?.getAttribute('data-report-scope')).toBe('global'); + const text=doc.querySelector('main')!.textContent!; + for(const value of ['25','144','75','49 unused initials','295 retained results','296 closed ordinal slots','102 exhausted finals','152/244','242 lazy definitions plus 2 implemented legacy']) expect(text).toContain(value); + expect(text).toContain('Fully native auxiliary setup and complete usage qualification are not established'); + expect(text).not.toContain('No result artifact was uploaded'); + expect(text).not.toContain('2 unimplemented'); + expect(doc.querySelectorAll('a[href*="create-project"]')).toHaveLength(1); + expect(doc.querySelectorAll('img[src], a[href*="artifact"], a[href*="private/"]')).toHaveLength(0); + expect(doc.querySelectorAll('[data-catalog-row][data-allowance="unused-initial"]')).toHaveLength(49); + }); + it("separates candidate failures from unmeasured main and exposes only supplied public descriptions", () => { + const data=fixture(); data.rows[218]!.executionRecord="candidate-failed"; + const id=data.rows[218]!.scenarioId; + const html=renderRunnerE2EDashboard({title:"Catalog",generatedAt:data.asOf,expected:[],catalog:[],entries:[],catalogOverview:data,catalogOverviewDescriptions:new Map([[id,{query:'Explain & that',expectedOutcome:'A saved public outcome'}]])}); + const doc=new JSDOM(html).window.document,row=doc.getElementById('catalog-'+id)!; + expect(row.textContent).toContain('No main measurement'); + expect(row.textContent).toContain('Candidate run: FAIL'); + expect(row.querySelector('details')!.textContent).toContain('Explain & that'); + expect(html).toContain('Explain <this> & that'); + expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1'); + expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded'); + }); + it("preserves a valid main outcome when its allowance is unknown", () => { + const data=fixture(); data.rows[0]!.allowance="unknown"; + expect(parseCatalogOverview(data).accounting.exhaustedFinals).toBe(102); + const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[0]!.scenarioId)!; + expect(row.textContent).toContain('Recorded pass'); + expect(row.textContent).toContain('Main measurement recorded'); + expect(row.textContent).toContain('Allowance unknown'); + expect(row.textContent).toContain('Recover ordinal evidence'); + expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes'); + expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,exhaustedFinals:104}})).toThrow('Inconsistent catalog accounting'); + }); + it("rejects unknown allowance when execution is positively known never-run", () => { + const data=fixture(); data.rows[169]!.allowance="unknown"; + expect(() => parseCatalogOverview(data)).toThrow('Inconsistent recorded execution scope'); + }); + it("shows a later candidate pass without promoting the recorded main failure", () => { + const data=fixture(); data.rows[25]!.latestCandidateOutcome="passed"; + const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[25]!.scenarioId)!; + expect(row.getAttribute('data-outcome')).toBe('failed'); + expect(row.textContent).toContain('Recorded fail'); + expect(row.textContent).toContain('Main measurement recorded · Latest candidate run: PASS'); + expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes144Latest main failures75Without a main measurement'); + expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('26 cases measured outside main'); + expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0); + expect(parseCatalogOverview(data).rows[25]!.mainOutcome).toBe('failed'); + expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===25?{...r,latestCandidateOutcome:'made-up'}:r)})).toThrow('Invalid catalog overview state'); + expect(() => parseCatalogOverview({...fixture(),rows:fixture().rows.map((r,i)=>i===169?{...r,latestCandidateOutcome:'passed'}:r)})).toThrow('Inconsistent recorded'); + }); + it.each([[152,"below"],[196,"equal to"],[200,"above"]] as const)("compares a ceiling of %i accurately", (ceiling, comparison) => { + const data=fixture(); data.accounting.theoreticalMainCeiling=ceiling; + expect(new JSDOM(page(data)).window.document.querySelector('main')!.textContent).toContain(`${ceiling}/244, ${comparison} the 196-case target`); + }); + it("filters and resets actual DOM rows without treating unused initials as failures", () => { + const dom=new JSDOM(page(fixture()),{runScripts:'dangerously',pretendToBeVisual:true}); + const doc=dom.window.document; + for (const selector of ['[data-nav-view="attention"]','.report-global-nav a[href$="?view=history"]','.report-breadcrumbs']) expect(dom.window.getComputedStyle(doc.querySelector(selector)!).display).toBe('none'); + expect(dom.window.getComputedStyle(doc.querySelector('[data-nav-view="overview"]')!).display).not.toBe('none'); + const select=doc.querySelector('[data-catalog-outcome]')!;select.value='unmeasured';select.dispatchEvent(new dom.window.Event('input')); + expect([...doc.querySelectorAll('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(75); + const allowance=doc.querySelector('[data-catalog-allowance]')!;allowance.value='unknown';allowance.dispatchEvent(new dom.window.Event('input')); + expect(doc.querySelector('[data-catalog-count]')!.textContent).toBe('1 of 244 scenarios'); + doc.querySelector('[data-catalog-reset]')!.click(); + expect([...doc.querySelectorAll('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(244); + const search=doc.querySelector('[data-catalog-query]')!;search.value='create project';search.dispatchEvent(new dom.window.Event('input')); + expect(doc.querySelector('[data-catalog-count]')!.textContent).toBe('1 of 244 scenarios'); + dom.window.close(); + }); + it("loads only bounded display metadata and refuses existing outputs and symlink inputs", async () => { + const dir=await mkdtemp(path.join(os.tmpdir(),'catalog-overview-test-')); + try { + const metadata=path.join(dir,'display.json'); await writeFile(metadata,JSON.stringify(fixture())); + const output=path.join(dir,'preview'); + await regenerateCatalogOverview({metadataFile:metadata,outputDirectory:output}); + expect((await stat(output)).mode & 0o777).toBe(0o700); + expect((await stat(path.join(output,'index.html'))).mode & 0o777).toBe(0o600); + expect(await readFile(path.join(output,'index.html'),'utf8')).toContain('All 244 scenarios'); + expect(await readFile(path.join(output,'index.html'),'utf8')).toContain('What this test asks'); + await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:output})).rejects.toThrow(); + await writeFile(metadata,JSON.stringify({...fixture(),rows:fixture().rows.map((r,i)=>i?r:{...r,scenarioId:'projects.invented-scenario'})})); + await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:path.join(dir,'invented-out')})).rejects.toThrow('frozen scenario IDs'); + const link=path.join(dir,'link.json'); await symlink(metadata,link); + await expect(regenerateCatalogOverview({metadataFile:link,outputDirectory:path.join(dir,'link-out')})).rejects.toThrow(); + await writeFile(metadata,' '.repeat(256*1024+1)); + await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:path.join(dir,'big-out')})).rejects.toThrow('bounded'); + } finally { await rm(dir,{recursive:true,force:true}); } + }); + it("links the dated immutable snapshot from real campaign and metadata navigation", () => { + const data=fixture(); + const ordinary=renderRunnerE2EDashboard({title:"Campaign",generatedAt:data.asOf,expected:[],catalog:[],entries:[]}); + for (const html of [ordinary,page(data)]) { + const dom=new JSDOM(html),link=dom.window.document.querySelector('[data-nav-catalog]')!; + expect(link.textContent).toContain('Chat catalog · Oct 29'); + expect(link.href).toBe('https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/chat-catalog-overview-20261029-175400z/'); + expect(dom.window.getComputedStyle(link).display).not.toBe('none'); + expect(dom.window.document.querySelectorAll('[data-nav-catalog]')).toHaveLength(1); + dom.window.close(); + } + }); + it("keeps the normal campaign branch and its missing-result behavior", () => { + const normal=renderRunnerE2EDashboard({title:'Existing campaign',generatedAt:'2026-10-29T17:54:00Z',expected:[],catalog:[],entries:[]}); + expect(normal).toContain('aria-label="Campaign summary"'); + expect(normal).not.toContain('data-catalog-overview>'); + expect(normal).toContain('data-report-scope="campaign"'); + const window=new JSDOM(normal).window; + expect(window.getComputedStyle(window.document.querySelector('[data-nav-view="attention"]')!).display).not.toBe('none'); + }); +}); diff --git a/tests/runner-e2e/catalog-overview.ts b/tests/runner-e2e/catalog-overview.ts new file mode 100644 index 00000000000..5779ab8e2be --- /dev/null +++ b/tests/runner-e2e/catalog-overview.ts @@ -0,0 +1,141 @@ +import { html } from "./report-format.js"; + +const outcomes = { passed: "Recorded pass", failed: "Recorded fail", unmeasured: "No main measurement" } as const; +const allowances = { "unused-initial": "Initial unused", "final-available": "Final available", exhausted: "Final consumed", unknown: "Allowance unknown" } as const; +const executionRecords = { "main-measured": "Main measurement recorded", "candidate-measured": "Candidate measured · outcome not projected", "candidate-failed": "Candidate run: FAIL", "candidate-passed": "Candidate run: PASS", "never-run": "Never run", unknown: "Execution evidence unknown" } as const; +const candidateOutcomes = { passed: "Latest candidate run: PASS", failed: "Latest candidate run: FAIL", "not-projected": "Latest candidate outcome not projected" } as const; +const readiness = { "not-assessed": "Not assessed here", "reviewed-candidate": "Candidate source reviewed", "needs-preparation": "Preparation needed" } as const; + +export interface CatalogOverviewRow { + scenarioId: string; + mainOutcome: keyof typeof outcomes; + allowance: keyof typeof allowances; + sourceReadiness: keyof typeof readiness; + executionRecord: keyof typeof executionRecords; + latestCandidateOutcome?: keyof typeof candidateOutcomes; +} + +export interface CatalogOverview { + schema: "paperclip.catalog-overview/v2"; + asOf: string; + definitions: { lazy: number; legacy: number; featureAreas: number; capabilities: number; declaredGaps: number }; + accounting: { mainMeasurements: number; retainedRawResults: number; closedSlots: number; exhaustedFinals: number; theoreticalMainCeiling: number }; + rows: CatalogOverviewRow[]; +} + +function object(value: unknown, keys: readonly string[], optional: readonly string[] = []): Record { + if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid catalog overview object"); + const row = value as Record; + if (Object.keys(row).some(key => !keys.includes(key) && !optional.includes(key)) || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields"); + return row; +} +function integer(value: unknown): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) throw new Error("Invalid catalog overview count"); + return value as number; +} +function choice(value: unknown, choices: T): keyof T { + if (typeof value !== "string" || !Object.hasOwn(choices, value)) throw new Error("Invalid catalog overview state"); + return value as keyof T; +} + +export function parseCatalogOverview(value: unknown): CatalogOverview { + const input = object(value, ["schema", "asOf", "definitions", "accounting", "rows"]); + if (input.schema !== "paperclip.catalog-overview/v2" || typeof input.asOf !== "string" || !/^\d{4}-\d{2}-\d{2}T[\d:.]+Z$/.test(input.asOf) || !Number.isFinite(Date.parse(input.asOf))) throw new Error("Invalid catalog overview snapshot"); + const d = object(input.definitions, ["lazy", "legacy", "featureAreas", "capabilities", "declaredGaps"]); + const definitions = { lazy: integer(d.lazy), legacy: integer(d.legacy), featureAreas: integer(d.featureAreas), capabilities: integer(d.capabilities), declaredGaps: integer(d.declaredGaps) }; + if (definitions.lazy !== 242 || definitions.legacy !== 2 || definitions.featureAreas !== 35 || definitions.capabilities !== 191) throw new Error("Frozen catalog scope changed"); + const a = object(input.accounting, ["mainMeasurements", "retainedRawResults", "closedSlots", "exhaustedFinals", "theoreticalMainCeiling"]); + const accounting = { mainMeasurements: integer(a.mainMeasurements), retainedRawResults: integer(a.retainedRawResults), closedSlots: integer(a.closedSlots), exhaustedFinals: integer(a.exhaustedFinals), theoreticalMainCeiling: integer(a.theoreticalMainCeiling) }; + if (!Array.isArray(input.rows) || input.rows.length !== 244) throw new Error("Expected 244 catalog metadata rows"); + const rows = input.rows.map(value => { + const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"], ["latestCandidateOutcome"]); + if (typeof row.scenarioId !== "string" || !/^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$/.test(row.scenarioId)) throw new Error("Invalid public catalog ID"); + return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords), + ...(Object.hasOwn(row, "latestCandidateOutcome") ? { latestCandidateOutcome: choice(row.latestCandidateOutcome, candidateOutcomes) } : {}) }; + }); + if (new Set(rows.map(row => row.scenarioId)).size !== 244 || new Set(rows.map(row => row.scenarioId.split(".")[0])).size !== 35) throw new Error("Duplicate or incomplete catalog metadata"); + for (const row of rows) { + if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance === "unused-initial") !== (row.executionRecord === "never-run") + || (row.latestCandidateOutcome !== undefined && ["never-run", "unknown"].includes(row.executionRecord))) throw new Error("Inconsistent recorded execution scope"); + } + const measured = rows.filter(row => row.mainOutcome !== "unmeasured").length; + const knownExhausted = rows.filter(row => row.allowance === "exhausted").length; + if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals < knownExhausted + || accounting.exhaustedFinals > knownExhausted + rows.filter(row => row.allowance === "unknown").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting"); + return { schema: "paperclip.catalog-overview/v2", asOf: input.asOf, definitions, accounting, rows }; +} + +function label(value: string) { + const text = value.replaceAll("-", " "); + return text.charAt(0).toUpperCase() + text.slice(1); +} +function nextAction(row: CatalogOverviewRow) { + if (row.allowance === "unknown") return "Recover ordinal evidence; keep allowance unknown"; + if (row.mainOutcome === "passed") return "Retain the recorded outcome and its scope"; + if (row.allowance === "exhausted") return "Review the retained outcome; no further attempt"; + if (row.sourceReadiness === "needs-preparation") return "Prepare source, then review execution gates"; + return row.allowance === "unused-initial" ? "Review source and execution gates for the initial" : "Review the initial cause and final execution gates"; +} +function executionLabel(row: CatalogOverviewRow) { + return executionRecords[row.executionRecord] + (row.latestCandidateOutcome ? ` · ${candidateOutcomes[row.latestCandidateOutcome]}` : ""); +} +const exampleIds = ["projects.create-project", "projects.archive-project", "tasks.edit-properties", "tasks.comment-with-reference", "documents-artifacts.write-plan", "documents-artifacts.produce-download", "inbox-search.restore-archive"]; + +export interface CatalogScenarioDescription { query: string; expectedOutcome: string } + +export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap = new Map()) { + const data = parseCatalogOverview(value); + const count = (state: CatalogOverviewRow["mainOutcome"]) => data.rows.filter(row => row.mainOutcome === state).length; + const features = [...new Set(data.rows.map(row => row.scenarioId.split(".")[0]!))].sort(); + const ceilingComparison = data.accounting.theoreticalMainCeiling < 196 ? "below" : data.accounting.theoreticalMainCeiling === 196 ? "equal to" : "above"; + const examples = exampleIds.filter(id => data.rows.some(row => row.scenarioId === id && row.mainOutcome === "passed")); + return `
+

Chat feature catalog · private preview

All 244 scenarios

Recorded outcomes, remaining allowances and source readiness, shown separately.

Snapshot ${html(data.asOf)}

+
${count("passed")}Recorded feature-work passes
${count("failed")}Latest main failures
${count("unmeasured")}Without a main measurement
+

${count("passed") + count("failed")} main cases · ${data.accounting.mainMeasurements} main measurements. The ${count("unmeasured")} without a main measurement include ${data.rows.filter(row => row.allowance === "unused-initial").length} unused initials; these are different sets.

+

${data.rows.filter(row => row.executionRecord.startsWith("candidate-") || row.latestCandidateOutcome !== undefined).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.

+ + ${examples.length ? `

Passing feature examples

${examples.map(id => `${html(label(id.split(".")[1]!))}`).join(" · ")}

` : ""} +
Attempt accounting and limits

${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.

The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, ${ceilingComparison} the 196-case target. This is an upper bound, not a forecast or an execution approval.

+

244 scenarios

+
${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => ``).join("")}
ScenarioMain outcomeRecorded executionAllowanceSource readinessNext action
${html(label(row.scenarioId.split(".")[1]!))}${html(label(row.scenarioId.split(".")[0]!))}${html(row.scenarioId)}${descriptions.has(row.scenarioId) ? `
What this test asks

Human request: ${html(descriptions.get(row.scenarioId)!.query)}

Expected outcome: ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}

` : ""}		${outcomes[row.mainOutcome]}${executionLabel(row)}${allowances[row.allowance]}${readiness[row.sourceReadiness]}${html(nextAction(row))}
+
Metadata overview only · No result cards or evidence files are synthesized
+
`; +} + +export const catalogOverviewStyles = ` +body[data-catalog-metadata] .report-global-nav a[data-nav-view="attention"], +body[data-catalog-metadata] .report-global-nav a[href$="?view=history"], +body[data-catalog-metadata] .report-breadcrumbs { display:none; } +.catalog-scope { border:1px solid var(--border); border-radius:var(--radius); background:var(--raised); padding:20px; margin:24px 0; } +.catalog-scope p { margin-bottom:0; } +.catalog-table { width:100%; border-collapse:collapse; text-align:left; } +.catalog-table th, .catalog-table td { padding:14px; border-bottom:1px solid var(--border); vertical-align:top; } +.catalog-table thead { background:var(--raised); } +.catalog-table tbody th { min-width:220px; font-weight:400; } +.catalog-table small, .catalog-table code { display:block; color:var(--muted-foreground); margin-top:5px; } +.catalog-table code { font-size:11px; } +.catalog-table tr:target { background:var(--report-filter-tint); } +`; +export const catalogOverviewScript = String.raw` +(() => { + const root = document.querySelector('[data-catalog-overview]'); + if (!root) return; + const rows = [...root.querySelectorAll('[data-catalog-row]')]; + const controls = ['query','feature','outcome','allowance'].map(name => root.querySelector('[data-catalog-' + name + ']')); + const [query, feature, outcome, allowance] = controls; + const apply = () => { + let count = 0; + const search = query.value.trim().toLowerCase(); + for (const row of rows) { + row.hidden = Boolean((search && !row.textContent.toLowerCase().includes(search)) || (feature.value && row.dataset.feature !== feature.value) || (outcome.value && row.dataset.outcome !== outcome.value) || (allowance.value && row.dataset.allowance !== allowance.value)); + if (!row.hidden) count++; + } + root.querySelector('[data-catalog-count]').textContent = count + ' of ' + rows.length + ' scenarios'; + }; + controls.forEach(control => control.addEventListener('input', apply)); + root.querySelector('[data-catalog-reset]').addEventListener('click', () => { controls.forEach(control => { control.value = ''; }); apply(); }); + root.querySelectorAll('a[href^="#catalog-"]').forEach(link => link.addEventListener('click', () => { controls.forEach(control => { control.value = ''; }); apply(); })); + apply(); +})(); +`; diff --git a/tests/runner-e2e/dashboard-regenerate.ts b/tests/runner-e2e/dashboard-regenerate.ts index 5d94321e35c..7bb26fac94d 100644 --- a/tests/runner-e2e/dashboard-regenerate.ts +++ b/tests/runner-e2e/dashboard-regenerate.ts @@ -1,13 +1,11 @@ import path from "node:path"; -import { mkdir, readFile, readdir, writeFile } from "node:fs/promises"; -import { runnerMatrix } from "./catalog.js"; +import { mkdir, open, readFile, readdir, writeFile } from "node:fs/promises"; +import { constants } from "node:fs"; +import { isSeq, parseDocument } from "yaml"; +import { parseCatalogOverview, type CatalogScenarioDescription } from "./catalog-overview.js"; +import { stageDashboardBrandAssets } from "./report-assets.js"; import { renderRunnerE2EDashboard, type RunnerDashboardInput } from "./dashboard.js"; import { campaignFeedbackReviewText } from "./feedback-dispositions.js"; -import { - buildRunnerCampaign, - canonicalExecutionId, - upgradeRunnerResult, -} from "./history.js"; import type { RunnerE2ECampaign, RunnerE2EHistoryIndex, @@ -76,6 +74,41 @@ async function readOptionalHistory(file: string) { .catch(() => undefined); } +export async function regenerateCatalogOverview(input: { metadataFile: string; outputDirectory: string }) { + const file = await open(input.metadataFile, constants.O_RDONLY | constants.O_NOFOLLOW); + let metadata: unknown; + try { + const stat = await file.stat(); + if (!stat.isFile() || stat.size > 256 * 1024) throw new Error("Catalog overview input must be a bounded regular JSON file"); + metadata = JSON.parse(await file.readFile("utf8")); + } finally { + await file.close(); + } + const overview = parseCatalogOverview(metadata); + const catalogDirectory = new URL("./chat-scenarios/", import.meta.url); + const catalogIds = new Set(); + const descriptions = new Map(); + for (const name of (await readdir(catalogDirectory)).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml")) { + const document = parseDocument(await readFile(new URL(name, catalogDirectory), "utf8"), { uniqueKeys: true }); + if (document.errors.length) throw new Error("Invalid static chat catalog"); + const sequence = document.get("scenarios", true); + if (!isSeq(sequence)) throw new Error("Invalid static chat catalog"); + const scenarios: unknown = sequence.toJSON(); + if (!Array.isArray(scenarios)) throw new Error("Invalid static chat catalog"); + for (const row of scenarios) { + if (!row || typeof row.id !== "string" || catalogIds.has(row.id)) throw new Error("Invalid static chat catalog identity"); + catalogIds.add(row.id); + if (typeof row.query !== "string" || typeof row.expected_outcome !== "string") throw new Error("Missing public catalog description"); + descriptions.set(row.id, { query: row.query, expectedOutcome: row.expected_outcome }); + } + } + if (catalogIds.size !== 244 || overview.rows.some(row => !catalogIds.has(row.scenarioId))) throw new Error("Catalog overview must match the frozen scenario IDs"); + const dashboard = renderRunnerE2EDashboard({ title: "Chat feature catalog", generatedAt: overview.asOf, expected: [], catalog: [], entries: [], catalogOverview: overview, catalogOverviewDescriptions: descriptions }); + await mkdir(input.outputDirectory, { mode: 0o700 }); + await writeFile(path.join(input.outputDirectory, "index.html"), dashboard, { mode: 0o600, flag: "wx" }); + await stageDashboardBrandAssets(input.outputDirectory); +} + export async function regenerateRunnerDashboard(input: { bundle: string; historyFile?: string | null; @@ -83,6 +116,8 @@ export async function regenerateRunnerDashboard(input: { evidenceHrefPrefix?: string; feedbackReviewFile?: string; }) { + const { runnerMatrix } = await import("./catalog.js"); + const { buildRunnerCampaign, canonicalExecutionId, upgradeRunnerResult } = await import("./history.js"); const bundle = path.resolve(input.bundle); const outputDirectory = path.resolve(input.outputDirectory ?? bundle); const evidenceHrefPrefix = input.evidenceHrefPrefix?.replace( @@ -232,6 +267,13 @@ async function main() { const arguments_ = process.argv .slice(2) .filter((argument) => argument !== "--"); + if (arguments_.includes("--catalog-overview")) { + const index = arguments_.indexOf("--catalog-overview"); + const output = arguments_.indexOf("--output"); + if (index !== 0 || output !== 2 || arguments_.length !== 4 || !arguments_[1] || !arguments_[3]) throw new Error("Usage: dashboard --catalog-overview --output "); + await regenerateCatalogOverview({ metadataFile: arguments_[1], outputDirectory: arguments_[3] }); + return; + } const bundleArgument = arguments_.find( (argument) => !argument.startsWith("--"), ); diff --git a/tests/runner-e2e/dashboard.ts b/tests/runner-e2e/dashboard.ts index 65ba170db72..7de3ce065fe 100644 --- a/tests/runner-e2e/dashboard.ts +++ b/tests/runner-e2e/dashboard.ts @@ -1,3 +1,4 @@ +import { renderCatalogOverview, catalogOverviewStyles, catalogOverviewScript, type CatalogOverview, type CatalogScenarioDescription } from "./catalog-overview.js"; import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, summarizeCampaignTiming, summarizeWorkerPhaseTiming } from "./campaign-timing.js"; import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js"; import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js"; @@ -47,6 +48,8 @@ export interface RunnerDashboardInput { entries: readonly RunnerDashboardEntry[]; campaign?: RunnerE2ECampaign; history?: RunnerE2EHistoryIndex; + catalogOverview?: CatalogOverview; + catalogOverviewDescriptions?: ReadonlyMap; presentation?: { title?: string; description?: string; @@ -402,6 +405,8 @@ function renderSuiteResults(input: { } export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { + if (input.catalogOverview && (input.entries.length || input.expected.length || input.catalog.length || input.campaign)) throw new Error("Catalog metadata cannot be mixed with campaign results"); + const catalogOverview = input.catalogOverview ? renderCatalogOverview(input.catalogOverview, input.catalogOverviewDescriptions) : null; const presentationLinks = (input.presentation?.links ?? []).flatMap(link => { const href = reportEvidenceHref(link.href); return href ? [{ ...link, href }] : []; @@ -863,11 +868,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { .matrix thead th, .profile-cell, .profile-sticky { position: static; } } ${reportNavigationStyles} + ${catalogOverviewStyles} - - ${renderReportShell({ campaignId: input.campaign?.campaignId, catalog, history: input.history })} + + ${renderReportShell(catalogOverview ? { history: input.history } : { campaignId: input.campaign?.campaignId, catalog, history: input.history })}
+ ${catalogOverview ?? `
@@ -923,6 +930,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
${suiteSections}
Generated ${html(input.generatedAt)}${catalog.length} selected executions · Declared screenshots and normalized results published
+ `}
@@ -1078,7 +1086,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) { stage.addEventListener("pointercancel", () => { pointerStartX = null; }); })(); - + diff --git a/tests/runner-e2e/report-navigation.ts b/tests/runner-e2e/report-navigation.ts index ea1cf5daa0e..7a53b620d7f 100644 --- a/tests/runner-e2e/report-navigation.ts +++ b/tests/runner-e2e/report-navigation.ts @@ -37,6 +37,7 @@ export function renderReportShell(input: { ${paperclipReportLogo}Evaluations
${!campaign && presets.length ? `` : ""}${tree}
E2E tests${campaign ? `${input.catalog!.length} executions` : ""}