mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-10-31 06:04:11 +01:00
124 lines
10 KiB
TypeScript
124 lines
10 KiB
TypeScript
import { describe, expect, it } from "vitest";
|
|
import { readFileSync, readdirSync } from "node:fs";
|
|
import { createRequire } from "node:module";
|
|
const { JSDOM } = createRequire(new URL("../../server/package.json", import.meta.url))("jsdom") as typeof import("../../server/node_modules/@types/jsdom/index.js");
|
|
import { mkdtemp, writeFile, stat, rm, readFile, symlink } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import os from "node:os";
|
|
import { regenerateCatalogOverview } from "./dashboard-regenerate.js";
|
|
import { parseCatalogOverview, type CatalogOverview } from "./catalog-overview.js";
|
|
import { renderRunnerE2EDashboard } from "./dashboard.js";
|
|
|
|
function fixture(): CatalogOverview {
|
|
const catalog = new URL("./chat-scenarios/", import.meta.url);
|
|
const ids = readdirSync(catalog).filter(name => name.endsWith(".yaml")).flatMap(name => [...readFileSync(new URL(name,catalog),"utf8").matchAll(/^ - id: ([a-z0-9.-]+)$/gm)].map(match => match[1]!));
|
|
const example = ids.indexOf("projects.create-project"); [ids[0],ids[example]]=[ids[example]!,ids[0]!];
|
|
const rows: CatalogOverview["rows"] = ids.map((scenarioId,i)=>({scenarioId, mainOutcome:i<25?"passed":i<169?"failed":"unmeasured",allowance:i<102?"exhausted":i>=169&&i<218?"unused-initial":i===243?"unknown":"final-available",sourceReadiness:"not-assessed",executionRecord:i<169?"main-measured":i<218?"never-run":i===243?"unknown":"candidate-measured"}));
|
|
return { schema:"paperclip.catalog-overview/v2", asOf:"2026-10-29T17:54:00Z", definitions:{lazy:242,legacy:2,featureAreas:35,capabilities:191,declaredGaps:5}, accounting:{mainMeasurements:215,retainedRawResults:295,closedSlots:296,exhaustedFinals:102,theoreticalMainCeiling:152}, rows };
|
|
}
|
|
function page(data: CatalogOverview) {
|
|
return renderRunnerE2EDashboard({title:"Catalog",generatedAt:data.asOf,expected:[],catalog:[],entries:[],catalogOverview:data});
|
|
}
|
|
|
|
describe("display-only catalog overview", () => {
|
|
it("requires the fixed scope and rejects private carrier fields rather than serializing them", () => {
|
|
const data=fixture();
|
|
expect(parseCatalogOverview(data).rows).toHaveLength(244);
|
|
expect(() => parseCatalogOverview({...data,privatePath:"/private/hidden"})).toThrow("Unexpected");
|
|
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i? r : {...r,errorBody:"secret"})})).toThrow("Unexpected");
|
|
expect(() => parseCatalogOverview({...data,definitions:{...data.definitions,legacy:0}})).toThrow("Frozen");
|
|
});
|
|
it("rejects duplicate rows, invalid IDs/states, impossible counts and mixed real results", () => {
|
|
const data=fixture();
|
|
expect(() => parseCatalogOverview({...data,rows:[...data.rows.slice(1),data.rows[1]]})).toThrow("Duplicate");
|
|
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i?r:{...r,scenarioId:'<img src=x onerror=alert(1)>'})})).toThrow("Invalid public");
|
|
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i?r:{...r,mainOutcome:'candidate-pass'})})).toThrow("Invalid");
|
|
expect(() => parseCatalogOverview({...data,accounting:{...data.accounting,mainMeasurements:2}})).toThrow("Inconsistent");
|
|
expect(() => renderRunnerE2EDashboard({title:"mixed",generatedAt:data.asOf,expected:["real-case"],catalog:[],entries:[],catalogOverview:data})).toThrow("cannot be mixed");
|
|
});
|
|
it("renders metadata counts and scopes without fake result cards or evidence links", () => {
|
|
const data=fixture(); const doc=new JSDOM(page(data)).window.document;
|
|
expect(doc.querySelectorAll('[data-catalog-row]')).toHaveLength(244);
|
|
expect(doc.querySelectorAll('[data-report-case]')).toHaveLength(0);
|
|
expect(doc.querySelector('[data-report-scope]')?.getAttribute('data-report-scope')).toBe('global');
|
|
const text=doc.querySelector('main')!.textContent!;
|
|
for(const value of ['25','144','75','49 unused initials','295 retained results','296 closed ordinal slots','102 exhausted finals','152/244','242 lazy definitions plus 2 implemented legacy']) expect(text).toContain(value);
|
|
expect(text).toContain('Fully native auxiliary setup and complete usage qualification are not established');
|
|
expect(text).not.toContain('No result artifact was uploaded');
|
|
expect(text).not.toContain('2 unimplemented');
|
|
expect(doc.querySelectorAll('a[href*="create-project"]')).toHaveLength(1);
|
|
expect(doc.querySelectorAll('img[src], a[href*="artifact"], a[href*="private/"]')).toHaveLength(0);
|
|
expect(doc.querySelectorAll('[data-catalog-row][data-allowance="unused-initial"]')).toHaveLength(49);
|
|
});
|
|
it("separates candidate failures from unmeasured main and exposes only supplied public descriptions", () => {
|
|
const data=fixture(); data.rows[218]!.executionRecord="candidate-failed";
|
|
const id=data.rows[218]!.scenarioId;
|
|
const html=renderRunnerE2EDashboard({title:"Catalog",generatedAt:data.asOf,expected:[],catalog:[],entries:[],catalogOverview:data,catalogOverviewDescriptions:new Map([[id,{query:'Explain <this> & that',expectedOutcome:'A saved public outcome'}]])});
|
|
const doc=new JSDOM(html).window.document,row=doc.getElementById('catalog-'+id)!;
|
|
expect(row.textContent).toContain('No main measurement');
|
|
expect(row.textContent).toContain('Candidate run: FAIL');
|
|
expect(row.querySelector('details')!.textContent).toContain('Explain <this> & that');
|
|
expect(html).toContain('Explain <this> & that');
|
|
expect(doc.querySelector('[data-execution-scope]')!.textContent).toContain('25 cases measured outside main · 49 never run · 1');
|
|
expect(() => parseCatalogOverview({...data,rows:data.rows.map((r,i)=>i===218?{...r,executionRecord:'never-run'}:r)})).toThrow('Inconsistent recorded');
|
|
});
|
|
it.each([[152,"below"],[196,"equal to"],[200,"above"]] as const)("compares a ceiling of %i accurately", (ceiling, comparison) => {
|
|
const data=fixture(); data.accounting.theoreticalMainCeiling=ceiling;
|
|
expect(new JSDOM(page(data)).window.document.querySelector('main')!.textContent).toContain(`${ceiling}/244, ${comparison} the 196-case target`);
|
|
});
|
|
it("filters and resets actual DOM rows without treating unused initials as failures", () => {
|
|
const dom=new JSDOM(page(fixture()),{runScripts:'dangerously',pretendToBeVisual:true});
|
|
const doc=dom.window.document;
|
|
for (const selector of ['[data-nav-view="attention"]','.report-global-nav a[href$="?view=history"]','.report-breadcrumbs']) expect(dom.window.getComputedStyle(doc.querySelector(selector)!).display).toBe('none');
|
|
expect(dom.window.getComputedStyle(doc.querySelector('[data-nav-view="overview"]')!).display).not.toBe('none');
|
|
const select=doc.querySelector<HTMLSelectElement>('[data-catalog-outcome]')!;select.value='unmeasured';select.dispatchEvent(new dom.window.Event('input'));
|
|
expect([...doc.querySelectorAll<HTMLElement>('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(75);
|
|
const allowance=doc.querySelector<HTMLSelectElement>('[data-catalog-allowance]')!;allowance.value='unknown';allowance.dispatchEvent(new dom.window.Event('input'));
|
|
expect(doc.querySelector('[data-catalog-count]')!.textContent).toBe('1 of 244 scenarios');
|
|
doc.querySelector<HTMLButtonElement>('[data-catalog-reset]')!.click();
|
|
expect([...doc.querySelectorAll<HTMLElement>('[data-catalog-row]')].filter(r=>!r.hidden)).toHaveLength(244);
|
|
const search=doc.querySelector<HTMLInputElement>('[data-catalog-query]')!;search.value='create project';search.dispatchEvent(new dom.window.Event('input'));
|
|
expect(doc.querySelector('[data-catalog-count]')!.textContent).toBe('1 of 244 scenarios');
|
|
dom.window.close();
|
|
});
|
|
it("loads only bounded display metadata and refuses existing outputs and symlink inputs", async () => {
|
|
const dir=await mkdtemp(path.join(os.tmpdir(),'catalog-overview-test-'));
|
|
try {
|
|
const metadata=path.join(dir,'display.json'); await writeFile(metadata,JSON.stringify(fixture()));
|
|
const output=path.join(dir,'preview');
|
|
await regenerateCatalogOverview({metadataFile:metadata,outputDirectory:output});
|
|
expect((await stat(output)).mode & 0o777).toBe(0o700);
|
|
expect((await stat(path.join(output,'index.html'))).mode & 0o777).toBe(0o600);
|
|
expect(await readFile(path.join(output,'index.html'),'utf8')).toContain('All 244 scenarios');
|
|
expect(await readFile(path.join(output,'index.html'),'utf8')).toContain('What this test asks');
|
|
await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:output})).rejects.toThrow();
|
|
await writeFile(metadata,JSON.stringify({...fixture(),rows:fixture().rows.map((r,i)=>i?r:{...r,scenarioId:'projects.invented-scenario'})}));
|
|
await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:path.join(dir,'invented-out')})).rejects.toThrow('frozen scenario IDs');
|
|
const link=path.join(dir,'link.json'); await symlink(metadata,link);
|
|
await expect(regenerateCatalogOverview({metadataFile:link,outputDirectory:path.join(dir,'link-out')})).rejects.toThrow();
|
|
await writeFile(metadata,' '.repeat(256*1024+1));
|
|
await expect(regenerateCatalogOverview({metadataFile:metadata,outputDirectory:path.join(dir,'big-out')})).rejects.toThrow('bounded');
|
|
} finally { await rm(dir,{recursive:true,force:true}); }
|
|
});
|
|
it("links the dated immutable snapshot from real campaign and metadata navigation", () => {
|
|
const data=fixture();
|
|
const ordinary=renderRunnerE2EDashboard({title:"Campaign",generatedAt:data.asOf,expected:[],catalog:[],entries:[]});
|
|
for (const html of [ordinary,page(data)]) {
|
|
const dom=new JSDOM(html),link=dom.window.document.querySelector<HTMLAnchorElement>('[data-nav-catalog]')!;
|
|
expect(link.textContent).toContain('Chat catalog · Oct 29');
|
|
expect(link.href).toBe('https://d1p6rlowie26tp.cloudfront.net/runner-e2e/campaigns/chat-catalog-overview-20261029-175400z/');
|
|
expect(dom.window.getComputedStyle(link).display).not.toBe('none');
|
|
expect(dom.window.document.querySelectorAll('[data-nav-catalog]')).toHaveLength(1);
|
|
dom.window.close();
|
|
}
|
|
});
|
|
it("keeps the normal campaign branch and its missing-result behavior", () => {
|
|
const normal=renderRunnerE2EDashboard({title:'Existing campaign',generatedAt:'2026-10-29T17:54:00Z',expected:[],catalog:[],entries:[]});
|
|
expect(normal).toContain('aria-label="Campaign summary"');
|
|
expect(normal).not.toContain('data-catalog-overview>');
|
|
expect(normal).toContain('data-report-scope="campaign"');
|
|
const window=new JSDOM(normal).window;
|
|
expect(window.getComputedStyle(window.document.querySelector('[data-nav-view="attention"]')!).display).not.toBe('none');
|
|
});
|
|
});
|