mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-31 06:04:11 +01:00
fix(evals): separate candidate outcomes and allowance certainty
This commit is contained in:
Dotta committed
1 parent
43af4c4844
commit
e8686bf373
3 files changed
+48
-8
No files matched your search
@@ -3077,6 +3077,13 @@ and source-readiness enums. Extra fields are rejected. Build this projection
|
||||
from authenticated accounting metadata, preserving unknown allowances and the
|
||||
original qualification scope. Source review never supplies a result grade. The execution-record column distinguishes
|
||||
candidate measurements from never-run cases without changing main qualification.
|
||||
Allowance certainty is independent of a recorded measurement: a main pass or fail
|
||||
can retain `allowance: "unknown"`. Known exhausted rows bound the accounting total;
|
||||
unknown allowances can account for its remaining uncertainty. The optional
|
||||
`latestCandidateOutcome` enum (`passed`, `failed`, or `not-projected`) can record a
|
||||
later candidate measurement alongside a main outcome. It never changes
|
||||
`mainOutcome`, main counters, or the qualification scope. Omit it when no such
|
||||
additional outcome is projected; existing projections render unchanged.
|
||||
Expandable human requests and expected outcomes come only from the static YAML catalog,
|
||||
not from retained conversations or private artifacts.
|
||||
The renderer requires a new output directory and makes it private; it does not
|
||||
|
||||
@@ -62,6 +62,30 @@ describe("display-only catalog overview", () => {
|
||||
expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1');
|
||||
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded');
|
||||
});
|
||||
it("preserves a valid main outcome when its allowance is unknown", () => {
|
||||
const data=fixture(); data.rows[0]!.allowance="unknown";
|
||||
expect(parseCatalogOverview(data).accounting.exhaustedFinals).toBe(102);
|
||||
const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[0]!.scenarioId)!;
|
||||
expect(row.textContent).toContain('Recorded pass');
|
||||
expect(row.textContent).toContain('Main measurement recorded');
|
||||
expect(row.textContent).toContain('Allowance unknown');
|
||||
expect(row.textContent).toContain('Recover ordinal evidence');
|
||||
expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes');
|
||||
expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,exhaustedFinals:104}})).toThrow('Inconsistent catalog accounting');
|
||||
});
|
||||
it("shows a later candidate pass without promoting the recorded main failure", () => {
|
||||
const data=fixture(); data.rows[25]!.latestCandidateOutcome="passed";
|
||||
const doc=new JSDOM(page(data)).window.document,row=doc.getElementById('catalog-'+data.rows[25]!.scenarioId)!;
|
||||
expect(row.getAttribute('data-outcome')).toBe('failed');
|
||||
expect(row.textContent).toContain('Recorded fail');
|
||||
expect(row.textContent).toContain('Main measurement recorded · Latest candidate run: PASS');
|
||||
expect(doc.querySelector('.summary')!.textContent).toContain('25Recorded feature-work passes144Latest main failures75Without a main measurement');
|
||||
expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('26 cases measured outside main');
|
||||
expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0);
|
||||
expect(parseCatalogOverview(data).rows[25]!.mainOutcome).toBe('failed');
|
||||
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===25?{...r,latestCandidateOutcome:'made-up'}:r)})).toThrow('Invalid catalog overview state');
|
||||
expect(() => parseCatalogOverview({...fixture(),rows:fixture().rows.map((r,i)=>i===169?{...r,latestCandidateOutcome:'passed'}:r)})).toThrow('Inconsistent recorded');
|
||||
});
|
||||
it.each([[152,"below"],[196,"equal to"],[200,"above"]] as const)("compares a ceiling of %i accurately", (ceiling, comparison) => {
|
||||
const data=fixture(); data.accounting.theoreticalMainCeiling=ceiling;
|
||||
expect(new JSDOM(page(data)).window.document.querySelector('main')!.textContent).toContain(`${ceiling}/244, ${comparison} the 196-case target`);
|
||||
|
||||
@@ -3,6 +3,7 @@ import { html } from "./report-format.js";
|
||||
const outcomes = { passed: "Recorded pass", failed: "Recorded fail", unmeasured: "No main measurement" } as const;
|
||||
const allowances = { "unused-initial": "Initial unused", "final-available": "Final available", exhausted: "Final consumed", unknown: "Allowance unknown" } as const;
|
||||
const executionRecords = { "main-measured": "Main measurement recorded", "candidate-measured": "Candidate measured · outcome not projected", "candidate-failed": "Candidate run: FAIL", "candidate-passed": "Candidate run: PASS", "never-run": "Never run", unknown: "Execution evidence unknown" } as const;
|
||||
const candidateOutcomes = { passed: "Latest candidate run: PASS", failed: "Latest candidate run: FAIL", "not-projected": "Latest candidate outcome not projected" } as const;
|
||||
const readiness = { "not-assessed": "Not assessed here", "reviewed-candidate": "Candidate source reviewed", "needs-preparation": "Preparation needed" } as const;
|
||||
|
||||
export interface CatalogOverviewRow {
|
||||
@@ -11,6 +12,7 @@ export interface CatalogOverviewRow {
|
||||
allowance: keyof typeof allowances;
|
||||
sourceReadiness: keyof typeof readiness;
|
||||
executionRecord: keyof typeof executionRecords;
|
||||
latestCandidateOutcome?: keyof typeof candidateOutcomes;
|
||||
}
|
||||
|
||||
export interface CatalogOverview {
|
||||
@@ -21,10 +23,10 @@ export interface CatalogOverview {
|
||||
rows: CatalogOverviewRow[];
|
||||
}
|
||||
|
||||
function object(value: unknown, keys: readonly string[]): Record<string, unknown> {
|
||||
function object(value: unknown, keys: readonly string[], optional: readonly string[] = []): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid catalog overview object");
|
||||
const row = value as Record<string, unknown>;
|
||||
if (Object.keys(row).length !== keys.length || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields");
|
||||
if (Object.keys(row).some(key => !keys.includes(key) && !optional.includes(key)) || keys.some(key => !Object.hasOwn(row, key))) throw new Error("Unexpected catalog overview fields");
|
||||
return row;
|
||||
}
|
||||
function integer(value: unknown): number {
|
||||
@@ -46,16 +48,20 @@ export function parseCatalogOverview(value: unknown): CatalogOverview {
|
||||
const accounting = { mainMeasurements: integer(a.mainMeasurements), retainedRawResults: integer(a.retainedRawResults), closedSlots: integer(a.closedSlots), exhaustedFinals: integer(a.exhaustedFinals), theoreticalMainCeiling: integer(a.theoreticalMainCeiling) };
|
||||
if (!Array.isArray(input.rows) || input.rows.length !== 244) throw new Error("Expected 244 catalog metadata rows");
|
||||
const rows = input.rows.map(value => {
|
||||
const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"]);
|
||||
const row = object(value, ["scenarioId", "mainOutcome", "allowance", "sourceReadiness", "executionRecord"], ["latestCandidateOutcome"]);
|
||||
if (typeof row.scenarioId !== "string" || !/^[a-z][a-z0-9-]*\.[a-z][a-z0-9-]*$/.test(row.scenarioId)) throw new Error("Invalid public catalog ID");
|
||||
return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords) };
|
||||
return { scenarioId: row.scenarioId, mainOutcome: choice(row.mainOutcome, outcomes), allowance: choice(row.allowance, allowances), sourceReadiness: choice(row.sourceReadiness, readiness), executionRecord: choice(row.executionRecord, executionRecords),
|
||||
...(Object.hasOwn(row, "latestCandidateOutcome") ? { latestCandidateOutcome: choice(row.latestCandidateOutcome, candidateOutcomes) } : {}) };
|
||||
});
|
||||
if (new Set(rows.map(row => row.scenarioId)).size !== 244 || new Set(rows.map(row => row.scenarioId.split(".")[0])).size !== 35) throw new Error("Duplicate or incomplete catalog metadata");
|
||||
for (const row of rows) {
|
||||
if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance === "unused-initial") !== (row.executionRecord === "never-run") || (row.allowance === "unknown") !== (row.executionRecord === "unknown")) throw new Error("Inconsistent recorded execution scope");
|
||||
if ((row.mainOutcome !== "unmeasured") !== (row.executionRecord === "main-measured") || (row.allowance !== "unknown" && (row.allowance === "unused-initial") !== (row.executionRecord === "never-run"))
|
||||
|| (row.latestCandidateOutcome !== undefined && ["never-run", "unknown"].includes(row.executionRecord))) throw new Error("Inconsistent recorded execution scope");
|
||||
}
|
||||
const measured = rows.filter(row => row.mainOutcome !== "unmeasured").length;
|
||||
if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals !== rows.filter(row => row.allowance === "exhausted").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting");
|
||||
const knownExhausted = rows.filter(row => row.allowance === "exhausted").length;
|
||||
if (accounting.mainMeasurements < measured || accounting.retainedRawResults < accounting.mainMeasurements || accounting.closedSlots < accounting.retainedRawResults || accounting.exhaustedFinals < knownExhausted
|
||||
|| accounting.exhaustedFinals > knownExhausted + rows.filter(row => row.allowance === "unknown").length || accounting.theoreticalMainCeiling > 244) throw new Error("Inconsistent catalog accounting");
|
||||
return { schema: "paperclip.catalog-overview/v2", asOf: input.asOf, definitions, accounting, rows };
|
||||
}
|
||||
|
||||
@@ -70,6 +76,9 @@ function nextAction(row: CatalogOverviewRow) {
|
||||
if (row.sourceReadiness === "needs-preparation") return "Prepare source, then review execution gates";
|
||||
return row.allowance === "unused-initial" ? "Review source and execution gates for the initial" : "Review the initial cause and final execution gates";
|
||||
}
|
||||
function executionLabel(row: CatalogOverviewRow) {
|
||||
return executionRecords[row.executionRecord] + (row.latestCandidateOutcome ? ` · ${candidateOutcomes[row.latestCandidateOutcome]}` : "");
|
||||
}
|
||||
const exampleIds = ["projects.create-project", "projects.archive-project", "tasks.edit-properties", "tasks.comment-with-reference", "documents-artifacts.write-plan", "documents-artifacts.produce-download", "inbox-search.restore-archive"];
|
||||
|
||||
export interface CatalogScenarioDescription { query: string; expectedOutcome: string }
|
||||
@@ -84,12 +93,12 @@ export function renderCatalogOverview(value: unknown, descriptions: ReadonlyMap<
|
||||
<header class="report-header"><div><p class="eyebrow">Chat feature catalog · private preview</p><h1 tabindex="-1">All 244 scenarios</h1><p class="lede">Recorded outcomes, remaining allowances and source readiness, shown separately.</p><p class="detail">Snapshot ${html(data.asOf)}</p></div></header>
|
||||
<div class="summary" aria-label="Catalog measurement summary"><div class="metric"><strong>${count("passed")}</strong><span>Recorded feature-work passes</span></div><div class="metric"><strong>${count("failed")}</strong><span>Latest main failures</span></div><div class="metric"><strong>${count("unmeasured")}</strong><span>Without a main measurement</span></div></div>
|
||||
<p class="detail">${count("passed") + count("failed")} main cases · ${data.accounting.mainMeasurements} main measurements. The ${count("unmeasured")} without a main measurement include ${data.rows.filter(row => row.allowance === "unused-initial").length} unused initials; these are different sets.</p>
|
||||
<p class="detail" data-execution-scope>${data.rows.filter(row => row.executionRecord.startsWith("candidate-")).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.</p>
|
||||
<p class="detail" data-execution-scope>${data.rows.filter(row => row.executionRecord.startsWith("candidate-") || row.latestCandidateOutcome !== undefined).length} cases measured outside main · ${data.rows.filter(row => row.executionRecord === "never-run").length} never run · ${data.rows.filter(row => row.executionRecord === "unknown").length} with unknown execution evidence. A missing main measurement is not a claim that the case never ran. Candidate PASS and FAIL stay outside the main qualification total.</p>
|
||||
<aside class="catalog-scope" aria-label="Qualification scope"><strong>What these outcomes establish</strong><p>The recorded passes support their original feature-work checks. Fully native auxiliary setup and complete usage qualification are not established. Unknown usage and cost are unknown, never zero. Candidate outcomes do not add ordinary main credit.</p><p>${data.definitions.lazy} lazy definitions plus ${data.definitions.legacy} implemented legacy definitions cover 35 feature areas and 191 capabilities. The ${data.definitions.declaredGaps} declared source gaps describe coverage scope, not live results. Source readiness is a partial source review inventory; it does not change a recorded grade or grant execution approval.</p></aside>
|
||||
${examples.length ? `<section aria-label="Recorded passing examples"><h2>Passing feature examples</h2><p class="evidence-links">${examples.map(id => `<a href="#catalog-${html(id)}">${html(label(id.split(".")[1]!))}</a>`).join(" · ")}</p></section>` : ""}
|
||||
<details class="campaign-billing-details"><summary>Attempt accounting and limits</summary><p>${data.accounting.retainedRawResults} retained results · ${data.accounting.closedSlots} closed ordinal slots · ${data.accounting.exhaustedFinals} exhausted finals · ${data.rows.filter(row => row.allowance === "unknown").length} unknown allowance.</p><p>The theoretical eligible main ceiling is ${data.accounting.theoreticalMainCeiling}/244, ${ceilingComparison} the 196-case target. This is an upper bound, not a forecast or an execution approval.</p></details>
|
||||
<section class="catalog-filters report-filters" aria-label="Filter catalog metadata"><label class="report-search"><span>Search scenarios</span><input type="search" data-catalog-query placeholder="Scenario or feature area"></label><label><span>Feature area</span><select data-catalog-feature><option value="">All feature areas</option>${features.map(feature => `<option value="${html(feature)}">${html(label(feature))}</option>`).join("")}</select></label><label><span>Main outcome</span><select data-catalog-outcome><option value="">All outcomes</option>${Object.entries(outcomes).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><label><span>Allowance</span><select data-catalog-allowance><option value="">All allowances</option>${Object.entries(allowances).map(([key, text]) => `<option value="${key}">${text}</option>`).join("")}</select></label><button class="filter-reset" type="button" data-catalog-reset>Reset filters</button><p data-catalog-count aria-live="polite">244 scenarios</p></section>
|
||||
<div class="table-wrap"><table class="catalog-table"><thead><tr><th scope="col">Scenario</th><th scope="col">Main outcome</th><th scope="col">Recorded execution</th><th scope="col">Allowance</th><th scope="col">Source readiness</th><th scope="col">Next action</th></tr></thead><tbody>${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => `<tr id="catalog-${html(row.scenarioId)}" data-catalog-row data-feature="${html(row.scenarioId.split(".")[0])}" data-outcome="${row.mainOutcome}" data-allowance="${row.allowance}"><th scope="row"><strong>${html(label(row.scenarioId.split(".")[1]!))}</strong><small>${html(label(row.scenarioId.split(".")[0]!))}</small><code>${html(row.scenarioId)}</code>${descriptions.has(row.scenarioId) ? `<details><summary>What this test asks</summary><p><strong>Human request:</strong> ${html(descriptions.get(row.scenarioId)!.query)}</p><p><strong>Expected outcome:</strong> ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}</p></details>` : ""}</th><td>${outcomes[row.mainOutcome]}</td><td>${executionRecords[row.executionRecord]}</td><td>${allowances[row.allowance]}</td><td>${readiness[row.sourceReadiness]}</td><td>${html(nextAction(row))}</td></tr>`).join("")}</tbody></table></div>
|
||||
<div class="table-wrap"><table class="catalog-table"><thead><tr><th scope="col">Scenario</th><th scope="col">Main outcome</th><th scope="col">Recorded execution</th><th scope="col">Allowance</th><th scope="col">Source readiness</th><th scope="col">Next action</th></tr></thead><tbody>${[...data.rows].sort((a,b) => a.scenarioId.localeCompare(b.scenarioId)).map(row => `<tr id="catalog-${html(row.scenarioId)}" data-catalog-row data-feature="${html(row.scenarioId.split(".")[0])}" data-outcome="${row.mainOutcome}" data-allowance="${row.allowance}"><th scope="row"><strong>${html(label(row.scenarioId.split(".")[1]!))}</strong><small>${html(label(row.scenarioId.split(".")[0]!))}</small><code>${html(row.scenarioId)}</code>${descriptions.has(row.scenarioId) ? `<details><summary>What this test asks</summary><p><strong>Human request:</strong> ${html(descriptions.get(row.scenarioId)!.query)}</p><p><strong>Expected outcome:</strong> ${html(descriptions.get(row.scenarioId)!.expectedOutcome)}</p></details>` : ""}</th><td>${outcomes[row.mainOutcome]}</td><td>${executionLabel(row)}</td><td>${allowances[row.allowance]}</td><td>${readiness[row.sourceReadiness]}</td><td>${html(nextAction(row))}</td></tr>`).join("")}</tbody></table></div>
|
||||
<footer>Metadata overview only · No result cards or evidence files are synthesized</footer>
|
||||
</section>`;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user