feat(evals): add a metadata overview for the chat catalog

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/README.md

@@ -3061,3 +3061,24 @@ they are not reported as charges or converted into reviewed lifecycle bounds.
 This mode therefore does not certify a full-lifecycle operating-cost ceiling.
 Omitting the policy preserves the stricter full-accounting gate. Historical
 journals and their holds are not rewritten by changing future campaign policy.
+
+## Private catalog metadata preview
+
+The existing dashboard renderer can show a display-only overview without creating
+result rows or discovering historical evidence:
+
+```sh
+pnpm test:e2e:runner:dashboard -- --catalog-overview <display-metadata.json> --output <new-private-directory>
+```
+
+The input must match `CatalogOverview` in `catalog-overview.ts`, contain exactly
+the frozen 244 YAML scenario IDs, and use only the declared outcome, allowance
+and source-readiness enums. Extra fields are rejected. Build this projection
+from authenticated accounting metadata, preserving unknown allowances and the
+original qualification scope. Source review never supplies a result grade. The execution-record column distinguishes
+candidate measurements from never-run cases without changing main qualification.
+Expandable human requests and expected outcomes come only from the static YAML catalog,
+not from retained conversations or private artifacts.
+The renderer requires a new output directory and makes it private; it does not
+publish the preview or modify existing campaign files. Normal campaign rendering
+and retained result cards continue to use the existing bundle command.

tests/runner-e2e/catalog-overview.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from "vitest";
+import { readFileSync, readdirSync } from "node:fs";
+import { createRequire } from "node:module";
+const { JSDOM } = createRequire(new URL("../../server/package.json", import.meta.url))("jsdom") as typeof import("../../server/node_modules/@types/jsdom/index.js");
+import { mkdtemp, writeFile, stat, rm, readFile, symlink } from "node:fs/promises";
+import path from "node:path";
+import os from "node:os";
+import { regenerateCatalogOverview } from "./dashboard-regenerate.js";
+import { parseCatalogOverview, type CatalogOverview } from "./catalog-overview.js";
+import { renderRunnerE2EDashboard } from "./dashboard.js";
+
+function fixture(): CatalogOverview {
+  const catalog = new URL("./chat-scenarios/", import.meta.url);
+  const ids = readdirSync(catalog).filter(name => name.endsWith(".yaml")).flatMap(name => [...readFileSync(new URL(name,catalog),"utf8").matchAll(/^  - id: ([a-z0-9.-]+)$/gm)].map(match => match[1]!));
+  const example = ids.indexOf("projects.create-project"); [ids[0],ids[example]]=[ids[example]!,ids[0]!];
+  const rows: CatalogOverview["rows"] = ids.map((scenarioId,i)=>({scenarioId, mainOutcome:i<25?"passed":i<169?"failed":"unmeasured",allowance:i<102?"exhausted":i>=169&&i<218?"unused-initial":i===243?"unknown":"final-available",sourceReadiness:"not-assessed",executionRecord:i<169?"main-measured":i<218?"never-run":i===243?"unknown":"candidate-measured"}));
+  return { schema:"paperclip.catalog-overview/v2", asOf:"2026-10-29T17:54:00Z", definitions:{lazy:242,legacy:2,featureAreas:35,capabilities:191,declaredGaps:5}, accounting:{mainMeasurements:215,retainedRawResults:295,closedSlots:296,exhaustedFinals:102,theoreticalMainCeiling:152}, rows };
+}
+function page(data: CatalogOverview) {
+  return renderRunnerE2EDashboard({title:"Catalog",generatedAt:data.asOf,expected:[],catalog:[],entries:[],catalogOverview:data});
+}
+
+describe("display-only catalog overview", () => {
+  it("requires the fixed scope and rejects private carrier fields rather than serializing them", () => {
+    const data=fixture();
+    expect(parseCatalogOverview(data).rows).toHaveLength(244);
+    expect(() => parseCatalogOverview({...data,privatePath:"/private/hidden"})).toThrow("Unexpected");
+    expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i? r : {...r,errorBody:"secret"})})).toThrow("Unexpected");
+    expect(() => parseCatalogOverview({...data,definitions:{...data.definitions,legacy:0}})).toThrow("Frozen");
+  });
+  it("rejects duplicate rows, invalid IDs/states, impossible counts and mixed real results", () => {
+    const data=fixture();
+    expect(() => parseCatalogOverview({...data,rows:[...data.rows.slice(1),data.rows[1]]})).toThrow("Duplicate");
+    expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i?r:{...r,scenarioId:'<img src=x onerror=alert(1)>'})})).toThrow("Invalid public");
+    expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i?r:{...r,mainOutcome:'candidate-pass'})})).toThrow("Invalid");
+    expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,mainMeasurements:2}})).toThrow("Inconsistent");
+    expect(() => renderRunnerE2EDashboard({title:"mixed",generatedAt:data.asOf,expected:["real-case"],catalog:[],entries:[],catalogOverview:data})).toThrow("cannot be mixed");
+  });
+  it("renders metadata counts and scopes without fake result cards or evidence links", () => {
+    const data=fixture(); const doc=new JSDOM(page(data)).window.document;
+    expect(doc.querySelectorAll('[data-catalog-row]')).toHaveLength(244);
+    expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0);
+    expect(doc.querySelector('[data-report-scope]')?.getAttribute('data-report-scope')).toBe('global');
+    const text=doc.querySelector('main')!.textContent!;
+    for(const value of ['25','144','75','49 unused initials','295 retained results','296 closed ordinal slots','102 exhausted finals','152/244','242 lazy definitions plus 2 implemented legacy']) expect(text).toContain(value);
+    expect(text).toContain('Fully native auxiliary setup and complete usage qualification are not established');
+    expect(text).not.toContain('No result artifact was uploaded');
+    expect(text).not.toContain('2 unimplemented');
+    expect(doc.querySelectorAll('a[href*="create-project"]')).toHaveLength(1);
+    expect(doc.querySelectorAll('img[src], a[href*="artifact"], a[href*="private/"]')).toHaveLength(0);
+    expect(doc.querySelectorAll('[data-catalog-row][data-allowance="unused-initial"]')).toHaveLength(49);
+  });
+  it("separates candidate failures from unmeasured main and exposes only supplied public descriptions", () => {
+    const data=fixture(); data.rows[218]!.executionRecord="candidate-failed";
+    const id=data.rows[218]!.scenarioId;
+    const html=renderRunnerE2EDashboard({title:"Catalog",generatedAt:data.asOf,expected:[],catalog:[],entries:[],catalogOverview:data,catalogOverviewDescriptions:new Map([[id,{query:'Explain <this> & that',expectedOutcome:'A saved public outcome'}]])});
+    const doc=new JSDOM(html).window.document,row=doc.getElementById('catalog-'+id)!;
+    expect(row.textContent).toContain('No main measurement');
+    expect(row.textContent).toContain('Candidate run: FAIL');
+    expect(row.querySelector('details')!.textContent).toContain('Explain <this> & that');
+    expect(html).toContain('Explain &lt;this&gt; &amp; that');
+    expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1');
+    expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded');
+  });
+  it("filters and resets actual DOM rows without treating unused initials as failures", () => {
+    const dom=new JSDOM(page(fixture()),{runScripts:'dangerously',pretendToBeVisual:true});
+    const doc=dom.window.document;
+    const select=doc.querySelector<HTMLSelectElement>('[data-catalog-outcome]')!;select.value='unmeasured';select.dispatchEvent(new dom.window.Event('input'));
+    expect([...doc.querySelectorAll<HTMLElement>('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(75);
+    const allowance=doc.querySelector<HTMLSelectElement>('[data-catalog-allowance]')!;allowance.value='unknown';allowance.dispatchEvent(new dom.window.Event('input'));
+    expect(doc.querySelector('[data-catalog-count]')!.textContent).toBe('1 of 244 scenarios');
+    doc.querySelector<HTMLButtonElement>('[data-catalog-reset]')!.click();
+    expect([...doc.querySelectorAll<HTMLElement>('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(244);
+    const search=doc.querySelector<HTMLInputElement>('[data-catalog-query]')!;search.value='create project';search.dispatchEvent(new dom.window.Event('input'));
+    expect(doc.querySelector('[data-catalog-count]')!.textContent).toBe('1 of 244 scenarios');
+    dom.window.close();
+  });
+  it("loads only bounded display metadata and refuses existing outputs and symlink inputs", async () => {
+    const dir=await mkdtemp(path.join(os.tmpdir(),'catalog-overview-test-'));
+    try {
+      const metadata=path.join(dir,'display.json'); await writeFile(metadata,JSON.stringify(fixture()));
+      const output=path.join(dir,'preview');
+      await regenerateCatalogOverview({metadataFile:metadata,outputDirectory:output});
+      expect((await stat(output)).mode & 0o777).toBe(0o700);
+      expect((await stat(path.join(output,'index.html'))).mode & 0o777).toBe(0o600);
+      expect(await readFile(path.join(output,'index.html'),'utf8')).toContain('All 244 scenarios');
+      expect(await readFile(path.join(output,'index.html'),'utf8')).toContain('What this test asks');
+      await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:output})).rejects.toThrow();
+      await writeFile(metadata,JSON.stringify({...fixture(),rows:fixture().rows.map((r,i)=>i?r:{...r,scenarioId:'projects.invented-scenario'})}));
+      await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:path.join(dir,'invented-out')})).rejects.toThrow('frozen scenario IDs');
+      const link=path.join(dir,'link.json'); await symlink(metadata,link);
+      await expect(regenerateCatalogOverview({metadataFile:link,outputDirectory:path.join(dir,'link-out')})).rejects.toThrow();
+      await writeFile(metadata,' '.repeat(256*1024+1));
+      await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:path.join(dir,'big-out')})).rejects.toThrow('bounded');
+    } finally { await rm(dir,{recursive:true,force:true}); }
+  });
+  it("keeps the normal campaign branch and its missing-result behavior", () => {
+    const normal=renderRunnerE2EDashboard({title:'Existing campaign',generatedAt:'2026-10-29T17:54:00Z',expected:[],catalog:[],entries:[]});
+    expect(normal).toContain('aria-label="Campaign summary"');
+    expect(normal).not.toContain('data-catalog-overview>');
+    expect(normal).toContain('data-report-scope="campaign"');
+  });
+});

tests/runner-e2e/catalog-overview.ts

@@ -0,0 +1,128 @@
+import { html } from "./report-format.js";
+
+const outcomes = { passed: "Recorded pass", failed: "Recorded fail", unmeasured: "No main measurement" } as const;
+const allowances = { "unused-initial": "Initial unused", "final-available": "Final available", exhausted: "Final consumed", unknown: "Allowance unknown" } as const;
+const executionRecords = { "main-measured": "Main measurement recorded", "candidate-measured": "Candidate measured · outcome not projected", "candidate-failed": "Candidate run: FAIL", "candidate-passed": "Candidate run: PASS", "never-run": "Never run", unknown: "Execution evidence unknown" } as const;
+const readiness = { "not-assessed": "Not assessed here", "reviewed-candidate": "Candidate source reviewed", "needs-preparation": "Preparation needed" } as const;
+
+export interface CatalogOverviewRow {
+  scenarioId: string;
+  mainOutcome: keyof typeof outcomes;
+  allowance: keyof typeof allowances;
+  sourceReadiness: keyof typeof readiness;
+  executionRecord: keyof typeof executionRecords;
+}
+
+export interface CatalogOverview {
+  schema: "paperclip.catalog-overview/v2";
+  asOf: string;
+  definitions: { lazy: number; legacy: number; featureAreas: number; capabilities: number; declaredGaps: number };
+  accounting: { mainMeasurements: number; retainedRawResults: number; closedSlots: number; exhaustedFinals: number; theoreticalMainCeiling: number };
+  rows: CatalogOverviewRow[];
+}
+
+function object(value: unknown, keys: readonly string[]): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid catalog overview object");
+  const row = value as Record<string, unknown>;
+  if (Object.keys(row).length !== keys.length || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields");
+  return row;
+}
+function integer(value: unknown): number {
+  if (!Number.isSafeInteger(value) || (value as number) < 0) throw new Error("Invalid catalog overview count");
+  return value as number;
+}
+function choice<T extends object>(value: unknown, choices: T): keyof T {
+  if (typeof value !== "string" || !Object.hasOwn(choices, value)) throw new Error("Invalid catalog overview state");
+  return value as keyof T;
+}
+
+export function parseCatalogOverview(value: unknown): CatalogOverview {
+  const input = object(value, ["schema", "asOf", "definitions", "accounting", "rows"]);
+  if (input.schema !== "paperclip.catalog-overview/v2" || typeof input.asOf !== "string" || !/^\d{4}-\d{2}-\d{2}T[\d:.]+Z$/.test(input.asOf) || !Number.isFinite(Date.parse(input.asOf))) throw new Error("Invalid catalog overview snapshot");
+  const d = object(input.definitions, ["lazy", "legacy", "featureAreas", "capabilities", "declaredGaps"]);
+  const definitions = { lazy: integer(d.lazy), legacy: integer(d.legacy), featureAreas: integer(d.featureAreas), capabilities: integer(d.capabilities), declaredGaps: integer(d.declaredGaps) };
+  if (definitions.lazy !== 242 || definitions.legacy !== 2 || definitions.featureAreas !== 35 || definitions.capabilities !== 191) throw new Error("Frozen catalog scope changed");
+  const a = object(input.accounting, ["mainMeasurements", "retainedRawResults", "closedSlots", "exhaustedFinals", "theoreticalMainCeiling"]);
+  const accounting = { mainMeasurements: integer(a.mainMeasurements), retainedRawResults: integer(a.retainedRawResults), closedSlots: integer(a.closedSlots), exhaustedFinals: integer(a.exhaustedFinals), theoreticalMainCeiling: integer(a.theoreticalMainCeiling) };
+  if (!Array.isArray(input.rows) || input.rows.length !== 244) throw new Error("Expected 244 catalog metadata rows");
+  const rows = input.rows.map(value => {
+    const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"]);
+    if (typeof row.scenarioId !== "string" || !/^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$/.test(row.scenarioId)) throw new Error("Invalid public catalog ID");
+    return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords) };
+  });
+  if (new Set(rows.map(row => row.scenarioId)).size !== 244 || new Set(rows.map(row => row.scenarioId.split(".")[0])).size !== 35) throw new Error("Duplicate or incomplete catalog metadata");
+  for (const row of rows) {
+    if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance === "unused-initial") !== (row.executionRecord === "never-run") || (row.allowance === "unknown") !== (row.executionRecord === "unknown")) throw new Error("Inconsistent recorded execution scope");
+  }
+  const measured = rows.filter(row => row.mainOutcome !== "unmeasured").length;
+  if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals !== rows.filter(row => row.allowance === "exhausted").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting");
+  return { schema: "paperclip.catalog-overview/v2", asOf: input.asOf, definitions, accounting, rows };
+}
+
+function label(value: string) {
+  const text = value.replaceAll("-", " ");
+  return text.charAt(0).toUpperCase() + text.slice(1);
+}
+function nextAction(row: CatalogOverviewRow) {
+  if (row.allowance === "unknown") return "Recover ordinal evidence; keep allowance unknown";
+  if (row.mainOutcome === "passed") return "Retain the recorded outcome and its scope";
+  if (row.allowance === "exhausted") return "Review the retained outcome; no further attempt";
+  if (row.sourceReadiness === "needs-preparation") return "Prepare source, then review execution gates";
+  return row.allowance === "unused-initial" ? "Review source and execution gates for the initial" : "Review the initial cause and final execution gates";
+}
+const exampleIds = ["projects.create-project", "projects.archive-project", "tasks.edit-properties", "tasks.comment-with-reference", "documents-artifacts.write-plan", "documents-artifacts.produce-download", "inbox-search.restore-archive"];
+
+export interface CatalogScenarioDescription { query: string; expectedOutcome: string }
+
+export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<string, CatalogScenarioDescription> = new Map()) {
+  const data = parseCatalogOverview(value);
+  const count = (state: CatalogOverviewRow["mainOutcome"]) => data.rows.filter(row => row.mainOutcome === state).length;
+  const features = [...new Set(data.rows.map(row => row.scenarioId.split(".")[0]!))].sort();
+  const examples = exampleIds.filter(id => data.rows.some(row => row.scenarioId === id && row.mainOutcome === "passed"));
+  return `<section data-catalog-overview>
+    <header class="report-header"><div><p class="eyebrow">Chat feature catalog · private preview</p><h1 tabindex="-1">All 244 scenarios</h1><p class="lede">Recorded outcomes, remaining allowances and source readiness, shown separately.</p><p class="detail">Snapshot ${html(data.asOf)}</p></div></header>
+    <div class="summary" aria-label="Catalog measurement summary"><div class="metric"><strong>${count("passed")}</strong><span>Recorded feature-work passes</span></div><div class="metric"><strong>${count("failed")}</strong><span>Latest main failures</span></div><div class="metric"><strong>${count("unmeasured")}</strong><span>Without a main measurement</span></div></div>
+    <p class="detail">${count("passed") + count("failed")} main cases · ${data.accounting.mainMeasurements} main measurements. The ${count("unmeasured")} without a main measurement include ${data.rows.filter(row => row.allowance === "unused-initial").length} unused initials; these are different sets.</p>
+    <p class="detail" data-execution-scope>${data.rows.filter(row => row.executionRecord.startsWith("candidate-")).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.</p>
+    <aside class="catalog-scope" aria-label="Qualification scope"><strong>What these outcomes establish</strong><p>The recorded passes support their original feature-work checks. Fully native auxiliary setup and complete usage qualification are not established. Unknown usage and cost are unknown, never zero. Candidate outcomes do not add ordinary main credit.</p><p>${data.definitions.lazy} lazy definitions plus ${data.definitions.legacy} implemented legacy definitions cover 35 feature areas and 191 capabilities. The ${data.definitions.declaredGaps} declared source gaps describe coverage scope, not live results. Source readiness is a partial source review inventory; it does not change a recorded grade or grant execution approval.</p></aside>
+    ${examples.length ? `<section aria-label="Recorded passing examples"><h2>Passing feature examples</h2><p class="evidence-links">${examples.map(id => `<a href="#catalog-${html(id)}">${html(label(id.split(".")[1]!))}</a>`).join(" · ")}</p></section>` : ""}
+    <details class="campaign-billing-details"><summary>Attempt accounting and limits</summary><p>${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.</p><p>The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, below the 196-case target. This is an upper bound, not a forecast or an execution approval.</p></details>
+    <section class="catalog-filters report-filters" aria-label="Filter catalog metadata"><label class="report-search"><span>Search scenarios</span><input type="search" data-catalog-query placeholder="Scenario or feature area"></label><label><span>Feature area</span><select data-catalog-feature><option value="">All feature areas</option>${features.map(feature => `<option value="${html(feature)}">${html(label(feature))}</option>`).join("")}</select></label><label><span>Main outcome</span><select data-catalog-outcome><option value="">All outcomes</option>${Object.entries(outcomes).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><label><span>Allowance</span><select data-catalog-allowance><option value="">All allowances</option>${Object.entries(allowances).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><button class="filter-reset" type="button" data-catalog-reset>Reset filters</button><p data-catalog-count aria-live="polite">244 scenarios</p></section>
+    <div class="table-wrap"><table class="catalog-table"><thead><tr><th scope="col">Scenario</th><th scope="col">Main outcome</th><th scope="col">Recorded execution</th><th scope="col">Allowance</th><th scope="col">Source readiness</th><th scope="col">Next action</th></tr></thead><tbody>${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => `<tr id="catalog-${html(row.scenarioId)}" data-catalog-row data-feature="${html(row.scenarioId.split(".")[0])}" data-outcome="${row.mainOutcome}" data-allowance="${row.allowance}"><th scope="row"><strong>${html(label(row.scenarioId.split(".")[1]!))}</strong><small>${html(label(row.scenarioId.split(".")[0]!))}</small><code>${html(row.scenarioId)}</code>${descriptions.has(row.scenarioId) ? `<details><summary>What this test asks</summary><p><strong>Human request:</strong> ${html(descriptions.get(row.scenarioId)!.query)}</p><p><strong>Expected outcome:</strong> ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}</p></details>` : ""}</th><td>${outcomes[row.mainOutcome]}</td><td>${executionRecords[row.executionRecord]}</td><td>${allowances[row.allowance]}</td><td>${readiness[row.sourceReadiness]}</td><td>${html(nextAction(row))}</td></tr>`).join("")}</tbody></table></div>
+    <footer>Metadata overview only · No result cards or evidence files are synthesized</footer>
+  </section>`;
+}
+
+export const catalogOverviewStyles = `
+.catalog-scope { border:1px solid var(--border); border-radius:var(--radius); background:var(--raised); padding:20px; margin:24px 0; }
+.catalog-scope p { margin-bottom:0; }
+.catalog-table { width:100%; border-collapse:collapse; text-align:left; }
+.catalog-table th, .catalog-table td { padding:14px; border-bottom:1px solid var(--border); vertical-align:top; }
+.catalog-table thead { background:var(--raised); }
+.catalog-table tbody th { min-width:220px; font-weight:400; }
+.catalog-table small, .catalog-table code { display:block; color:var(--muted-foreground); margin-top:5px; }
+.catalog-table code { font-size:11px; }
+.catalog-table tr:target { background:var(--report-filter-tint); }
+`;
+export const catalogOverviewScript = String.raw`
+(() => {
+  const root = document.querySelector('[data-catalog-overview]');
+  if (!root) return;
+  const rows = [...root.querySelectorAll('[data-catalog-row]')];
+  const controls = ['query','feature','outcome','allowance'].map(name => root.querySelector('[data-catalog-' + name + ']'));
+  const [query, feature, outcome, allowance] = controls;
+  const apply = () => {
+    let count = 0;
+    const search = query.value.trim().toLowerCase();
+    for (const row of rows) {
+      row.hidden = Boolean((search && !row.textContent.toLowerCase().includes(search)) || (feature.value && row.dataset.feature !== feature.value) || (outcome.value && row.dataset.outcome !== outcome.value) || (allowance.value && row.dataset.allowance !== allowance.value));
+      if (!row.hidden) count++;
+    }
+    root.querySelector('[data-catalog-count]').textContent = count + ' of ' + rows.length + ' scenarios';
+  };
+  controls.forEach(control => control.addEventListener('input', apply));
+  root.querySelector('[data-catalog-reset]').addEventListener('click', () => { controls.forEach(control => { control.value = ''; }); apply(); });
+  root.querySelectorAll('a[href^="#catalog-"]').forEach(link => link.addEventListener('click', () => { controls.forEach(control => { control.value = ''; }); apply(); }));
+  apply();
+})();
+`;

tests/runner-e2e/dashboard-regenerate.ts

@@ -1,13 +1,11 @@
 import path from "node:path";
-import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
-import { runnerMatrix } from "./catalog.js";
+import { mkdir, open, readFile, readdir, writeFile } from "node:fs/promises";
+import { constants } from "node:fs";
+import { isSeq, parseDocument } from "yaml";
+import { parseCatalogOverview, type CatalogScenarioDescription } from "./catalog-overview.js";
+import { stageDashboardBrandAssets } from "./report-assets.js";
 import { renderRunnerE2EDashboard, type RunnerDashboardInput } from "./dashboard.js";
 import { campaignFeedbackReviewText } from "./feedback-dispositions.js";
-import {
-  buildRunnerCampaign,
-  canonicalExecutionId,
-  upgradeRunnerResult,
-} from "./history.js";
 import type {
   RunnerE2ECampaign,
   RunnerE2EHistoryIndex,
@@ -76,6 +74,41 @@ async function readOptionalHistory(file: string) {
     .catch(() => undefined);
 }
 
+export async function regenerateCatalogOverview(input: { metadataFile: string; outputDirectory: string }) {
+  const file = await open(input.metadataFile, constants.O_RDONLY | constants.O_NOFOLLOW);
+  let metadata: unknown;
+  try {
+    const stat = await file.stat();
+    if (!stat.isFile() || stat.size > 256 * 1024) throw new Error("Catalog overview input must be a bounded regular JSON file");
+    metadata = JSON.parse(await file.readFile("utf8"));
+  } finally {
+    await file.close();
+  }
+  const overview = parseCatalogOverview(metadata);
+  const catalogDirectory = new URL("./chat-scenarios/", import.meta.url);
+  const catalogIds = new Set<string>();
+  const descriptions = new Map<string, CatalogScenarioDescription>();
+  for (const name of (await readdir(catalogDirectory)).filter(name => name.endsWith(".yaml") && name !== "defaults.yaml")) {
+    const document = parseDocument(await readFile(new URL(name, catalogDirectory), "utf8"), { uniqueKeys: true });
+    if (document.errors.length) throw new Error("Invalid static chat catalog");
+    const sequence = document.get("scenarios", true);
+    if (!isSeq(sequence)) throw new Error("Invalid static chat catalog");
+    const scenarios: unknown = sequence.toJSON();
+    if (!Array.isArray(scenarios)) throw new Error("Invalid static chat catalog");
+    for (const row of scenarios) {
+      if (!row || typeof row.id !== "string" || catalogIds.has(row.id)) throw new Error("Invalid static chat catalog identity");
+      catalogIds.add(row.id);
+      if (typeof row.query !== "string" || typeof row.expected_outcome !== "string") throw new Error("Missing public catalog description");
+      descriptions.set(row.id, { query: row.query, expectedOutcome: row.expected_outcome });
+    }
+  }
+  if (catalogIds.size !== 244 || overview.rows.some(row => !catalogIds.has(row.scenarioId))) throw new Error("Catalog overview must match the frozen scenario IDs");
+  const dashboard = renderRunnerE2EDashboard({ title: "Chat feature catalog", generatedAt: overview.asOf, expected: [], catalog: [], entries: [], catalogOverview: overview, catalogOverviewDescriptions: descriptions });
+  await mkdir(input.outputDirectory, { mode: 0o700 });
+  await writeFile(path.join(input.outputDirectory, "index.html"), dashboard, { mode: 0o600, flag: "wx" });
+  await stageDashboardBrandAssets(input.outputDirectory);
+}
+
 export async function regenerateRunnerDashboard(input: {
   bundle: string;
   historyFile?: string | null;
@@ -83,6 +116,8 @@ export async function regenerateRunnerDashboard(input: {
   evidenceHrefPrefix?: string;
   feedbackReviewFile?: string;
 }) {
+  const { runnerMatrix } = await import("./catalog.js");
+  const { buildRunnerCampaign, canonicalExecutionId, upgradeRunnerResult } = await import("./history.js");
   const bundle = path.resolve(input.bundle);
   const outputDirectory = path.resolve(input.outputDirectory ?? bundle);
   const evidenceHrefPrefix = input.evidenceHrefPrefix?.replace(
@@ -232,6 +267,13 @@ async function main() {
   const arguments_ = process.argv
     .slice(2)
     .filter((argument) => argument !== "--");
+  if (arguments_.includes("--catalog-overview")) {
+    const index = arguments_.indexOf("--catalog-overview");
+    const output = arguments_.indexOf("--output");
+    if (index !== 0 || output !== 2 || arguments_.length !== 4 || !arguments_[1] || !arguments_[3]) throw new Error("Usage: dashboard --catalog-overview <display-metadata.json> --output <new-private-directory>");
+    await regenerateCatalogOverview({ metadataFile: arguments_[1], outputDirectory: arguments_[3] });
+    return;
+  }
   const bundleArgument = arguments_.find(
     (argument) => !argument.startsWith("--"),
   );

tests/runner-e2e/dashboard.ts

@@ -1,3 +1,4 @@
+import { renderCatalogOverview, catalogOverviewStyles, catalogOverviewScript, type CatalogOverview, type CatalogScenarioDescription } from "./catalog-overview.js";
 import { campaignAttemptTimeLabel, campaignTimingScope, renderWorkerPhaseTiming, summarizeCampaignTiming, summarizeWorkerPhaseTiming } from "./campaign-timing.js";
 import { html, durationLabel, tokenLabel, llmCostLabel, usdLabel } from "./report-format.js";
 import { renderReportShell, reportNavigationStyles, reportShellScript } from "./report-navigation.js";
@@ -47,6 +48,8 @@ export interface RunnerDashboardInput {
   entries: readonly RunnerDashboardEntry[];
   campaign?: RunnerE2ECampaign;
   history?: RunnerE2EHistoryIndex;
+  catalogOverview?: CatalogOverview;
+  catalogOverviewDescriptions?: ReadonlyMap<string, CatalogScenarioDescription>;
   presentation?: {
     title?: string;
     description?: string;
@@ -402,6 +405,8 @@ function renderSuiteResults(input: {
 }
 
 export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
+  if (input.catalogOverview && (input.entries.length || input.expected.length || input.catalog.length || input.campaign)) throw new Error("Catalog metadata cannot be mixed with campaign results");
+  const catalogOverview = input.catalogOverview ? renderCatalogOverview(input.catalogOverview, input.catalogOverviewDescriptions) : null;
   const presentationLinks = (input.presentation?.links ?? []).flatMap(link => {
     const href = reportEvidenceHref(link.href);
     return href ? [{ ...link, href }] : [];
@@ -863,11 +868,13 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
       .matrix thead th, .profile-cell, .profile-sticky { position: static; }
     }
     ${reportNavigationStyles}
+    ${catalogOverviewStyles}
   </style>
 </head>
 <body id="top">
-  ${renderReportShell({ campaignId: input.campaign?.campaignId, catalog, history: input.history })}
+  ${renderReportShell(catalogOverview ? { history: input.history } : { campaignId: input.campaign?.campaignId, catalog, history: input.history })}
   <main id="overview">
+    ${catalogOverview ?? `
     <section class="case-page-heading" data-case-page-heading hidden><p class="eyebrow">Test case</p><h1 data-case-page-title tabindex="-1"></h1><a href="?view=all" data-nav-return>← Back to campaign results</a></section>
     <section data-campaign-overview>
     <header class="report-header">
@@ -923,6 +930,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
     </section>
     ${suiteSections}
     <footer><span>Generated ${html(input.generatedAt)}</span><span>${catalog.length} selected executions · Declared screenshots and normalized results published</span></footer>
+    `}
   </main>
   </div></div>
   <a class="back-to-top" href="#top" data-back-to-top hidden><span aria-hidden="true">↑</span>Back to top</a>
@@ -1078,7 +1086,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
       stage.addEventListener("pointercancel", () => { pointerStartX = null; });
     })();
   </script>
-  <script>${reportShellScript}${reportBrowserScript}</script>
+  <script>${reportShellScript}${catalogOverview ? catalogOverviewScript : reportBrowserScript}</script>
 
 </body>
 </html>