mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-31 06:04:11 +01:00
fix(evals): correct metadata limits and navigation
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
4cd50e8f9e
commit
5956f59c7b
3 files changed
+14
-2
No files matched your search
@@ -62,9 +62,15 @@ describe("display-only catalog overview", () => {
|
||||
expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1');
|
||||
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded');
|
||||
});
|
||||
it.each([[152,"below"],[196,"equal to"],[200,"above"]] as const)("compares a ceiling of %i accurately", (ceiling, comparison) => {
|
||||
const data=fixture(); data.accounting.theoreticalMainCeiling=ceiling;
|
||||
expect(new JSDOM(page(data)).window.document.querySelector('main')!.textContent).toContain(`${ceiling}/244, ${comparison} the 196-case target`);
|
||||
});
|
||||
it("filters and resets actual DOM rows without treating unused initials as failures", () => {
|
||||
const dom=new JSDOM(page(fixture()),{runScripts:'dangerously',pretendToBeVisual:true});
|
||||
const doc=dom.window.document;
|
||||
for (const selector of ['[data-nav-view="attention"]','.report-global-nav a[href$="?view=history"]','.report-breadcrumbs']) expect(dom.window.getComputedStyle(doc.querySelector(selector)!).display).toBe('none');
|
||||
expect(dom.window.getComputedStyle(doc.querySelector('[data-nav-view="overview"]')!).display).not.toBe('none');
|
||||
const select=doc.querySelector<HTMLSelectElement>('[data-catalog-outcome]')!;select.value='unmeasured';select.dispatchEvent(new dom.window.Event('input'));
|
||||
expect([...doc.querySelectorAll<HTMLElement>('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(75);
|
||||
const allowance=doc.querySelector<HTMLSelectElement>('[data-catalog-allowance]')!;allowance.value='unknown';allowance.dispatchEvent(new dom.window.Event('input'));
|
||||
@@ -99,5 +105,7 @@ describe("display-only catalog overview", () => {
|
||||
expect(normal).toContain('aria-label="Campaign summary"');
|
||||
expect(normal).not.toContain('data-catalog-overview>');
|
||||
expect(normal).toContain('data-report-scope="campaign"');
|
||||
const window=new JSDOM(normal).window;
|
||||
expect(window.getComputedStyle(window.document.querySelector('[data-nav-view="attention"]')!).display).not.toBe('none');
|
||||
});
|
||||
});
|
||||
@@ -78,6 +78,7 @@ export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<
|
||||
const data = parseCatalogOverview(value);
|
||||
const count = (state: CatalogOverviewRow["mainOutcome"]) => data.rows.filter(row => row.mainOutcome === state).length;
|
||||
const features = [...new Set(data.rows.map(row => row.scenarioId.split(".")[0]!))].sort();
|
||||
const ceilingComparison = data.accounting.theoreticalMainCeiling < 196 ? "below" : data.accounting.theoreticalMainCeiling === 196 ? "equal to" : "above";
|
||||
const examples = exampleIds.filter(id => data.rows.some(row => row.scenarioId === id && row.mainOutcome === "passed"));
|
||||
return `<section data-catalog-overview>
|
||||
<header class="report-header"><div><p class="eyebrow">Chat feature catalog · private preview</p><h1 tabindex="-1">All 244 scenarios</h1><p class="lede">Recorded outcomes, remaining allowances and source readiness, shown separately.</p><p class="detail">Snapshot ${html(data.asOf)}</p></div></header>
|
||||
@@ -86,7 +87,7 @@ export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<
|
||||
<p class="detail" data-execution-scope>${data.rows.filter(row => row.executionRecord.startsWith("candidate-")).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.</p>
|
||||
<aside class="catalog-scope" aria-label="Qualification scope"><strong>What these outcomes establish</strong><p>The recorded passes support their original feature-work checks. Fully native auxiliary setup and complete usage qualification are not established. Unknown usage and cost are unknown, never zero. Candidate outcomes do not add ordinary main credit.</p><p>${data.definitions.lazy} lazy definitions plus ${data.definitions.legacy} implemented legacy definitions cover 35 feature areas and 191 capabilities. The ${data.definitions.declaredGaps} declared source gaps describe coverage scope, not live results. Source readiness is a partial source review inventory; it does not change a recorded grade or grant execution approval.</p></aside>
|
||||
${examples.length ? `<section aria-label="Recorded passing examples"><h2>Passing feature examples</h2><p class="evidence-links">${examples.map(id => `<a href="#catalog-${html(id)}">${html(label(id.split(".")[1]!))}</a>`).join(" · ")}</p></section>` : ""}
|
||||
<details class="campaign-billing-details"><summary>Attempt accounting and limits</summary><p>${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.</p><p>The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, below the 196-case target. This is an upper bound, not a forecast or an execution approval.</p></details>
|
||||
<details class="campaign-billing-details"><summary>Attempt accounting and limits</summary><p>${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.</p><p>The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, ${ceilingComparison} the 196-case target. This is an upper bound, not a forecast or an execution approval.</p></details>
|
||||
<section class="catalog-filters report-filters" aria-label="Filter catalog metadata"><label class="report-search"><span>Search scenarios</span><input type="search" data-catalog-query placeholder="Scenario or feature area"></label><label><span>Feature area</span><select data-catalog-feature><option value="">All feature areas</option>${features.map(feature => `<option value="${html(feature)}">${html(label(feature))}</option>`).join("")}</select></label><label><span>Main outcome</span><select data-catalog-outcome><option value="">All outcomes</option>${Object.entries(outcomes).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><label><span>Allowance</span><select data-catalog-allowance><option value="">All allowances</option>${Object.entries(allowances).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><button class="filter-reset" type="button" data-catalog-reset>Reset filters</button><p data-catalog-count aria-live="polite">244 scenarios</p></section>
|
||||
<div class="table-wrap"><table class="catalog-table"><thead><tr><th scope="col">Scenario</th><th scope="col">Main outcome</th><th scope="col">Recorded execution</th><th scope="col">Allowance</th><th scope="col">Source readiness</th><th scope="col">Next action</th></tr></thead><tbody>${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => `<tr id="catalog-${html(row.scenarioId)}" data-catalog-row data-feature="${html(row.scenarioId.split(".")[0])}" data-outcome="${row.mainOutcome}" data-allowance="${row.allowance}"><th scope="row"><strong>${html(label(row.scenarioId.split(".")[1]!))}</strong><small>${html(label(row.scenarioId.split(".")[0]!))}</small><code>${html(row.scenarioId)}</code>${descriptions.has(row.scenarioId) ? `<details><summary>What this test asks</summary><p><strong>Human request:</strong> ${html(descriptions.get(row.scenarioId)!.query)}</p><p><strong>Expected outcome:</strong> ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}</p></details>` : ""}</th><td>${outcomes[row.mainOutcome]}</td><td>${executionRecords[row.executionRecord]}</td><td>${allowances[row.allowance]}</td><td>${readiness[row.sourceReadiness]}</td><td>${html(nextAction(row))}</td></tr>`).join("")}</tbody></table></div>
|
||||
<footer>Metadata overview only · No result cards or evidence files are synthesized</footer>
|
||||
@@ -94,6 +95,9 @@ export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<
|
||||
}
|
||||
|
||||
export const catalogOverviewStyles = `
|
||||
body[data-catalog-metadata] .report-global-nav a[data-nav-view="attention"],
|
||||
body[data-catalog-metadata] .report-global-nav a[href$="?view=history"],
|
||||
body[data-catalog-metadata] .report-breadcrumbs { display:none; }
|
||||
.catalog-scope { border:1px solid var(--border); border-radius:var(--radius); background:var(--raised); padding:20px; margin:24px 0; }
|
||||
.catalog-scope p { margin-bottom:0; }
|
||||
.catalog-table { width:100%; border-collapse:collapse; text-align:left; }
|
||||
|
||||
@@ -871,7 +871,7 @@ export function renderRunnerE2EDashboard(input: RunnerDashboardInput) {
|
||||
${catalogOverviewStyles}
|
||||
</style>
|
||||
</head>
|
||||
<body id="top">
|
||||
<body id="top"${catalogOverview ? " data-catalog-metadata" : ""}>
|
||||
${renderReportShell(catalogOverview ? { history: input.history } : { campaignId: input.campaign?.campaignId, catalog, history: input.history })}
|
||||
<main id="overview">
|
||||
${catalogOverview ?? `
|
||||
|
||||
Reference in new issue
Block a user