From 36ec1aa4fbacea42df6b8221db5411b901a3f2f3 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 00:27:42 -0600 Subject: [PATCH] test(evals): inspect authored checklist as rich text blocks Co-Authored-By: Paperclip --- .../integrator-skill-authoring-v2.test.ts | 13 ++++++- .../chat-catalog-domains/integrator-skills.ts | 39 ++++++++++++++----- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts index fd9d3195598..0692d4256db 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts @@ -4,7 +4,7 @@ import type { InternalObservation } from "./internal-support.js"; vi.mock("@playwright/test",()=>({expect:Object.assign((node:any)=>({toBeVisible:async()=>{if(!node.visible)throw Error("Missing real UI item");},toBeEditable:async()=>{if(!node.editable)throw Error("Noneditable real UI");},toHaveURL:async(predicate:(url:URL)=>boolean)=>{if(!predicate(new URL(node.url())))throw Error("Wrong actual Studio identity");}}),{poll:(read:()=>Promise)=>({toBe:async(expected:boolean)=>{for(let i=0;i<3;i++)if(await read()===expected)return;throw Error("UI content did not hydrate to the saved checklist");}})})})); const observed=vi.hoisted(()=>({base:undefined as unknown})); vi.mock("./integrator-support.js",async original=>({...await original(),observeInternal:vi.fn(async(_ctx:unknown,state:unknown)=>({...observed.base as object,state}))})); -import { gradeIntegratorSkills, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js"; +import { gradeIntegratorSkills, preservesVisibleWebsiteChecklist, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js"; afterEach(()=>vi.restoreAllMocks()); function fixture(){ const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0; @@ -13,7 +13,7 @@ function fixture(){ const state:Record={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:[],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation; observed.base=o; const ui={library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,delayedHydration:false};const events:string[]=[]; - const editor={get visible(){return ui.studio;},get editable(){return ui.editable;},innerText:async()=>ui.delayedHydration&&editorReads++===0?"":ui.changedAfterReload&&reloads?"Lost checklist":ui.body}; + const editor={get visible(){return ui.studio;},get editable(){return ui.editable;},locator:()=>({evaluateAll:async()=>ui.delayedHydration&&editorReads++===0?[]:(ui.changedAfterReload&&reloads?"Lost checklist":ui.body).split("\n")})}; const page={url:()=>current,getByRole:(role:string,options?:{name?:string})=>{ if(role==="textbox")return editor; if(role==="link")return{first:()=>({click:async()=>{events.push(options!.name!);current=options?.name==="Skills"?"https://fixture.test/ACORN/skills":"https://fixture.test/ACORN/skills/studio";}})}; @@ -69,3 +69,12 @@ it("waits for actual saved editor content instead of rejecting its visible empty await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).resolves.toBeUndefined(); expect(f.state.authoredSkillInspection).toMatchObject({checklistVisible:true,reloaded:true}); }); + +it("inspects real rich-text blocks without applying Markdown marker assumptions to unrelated prefaces", () => { + const blocks=["Do not mark a check passed without evidence.",...websiteChecklist]; + expect(preservesVisibleWebsiteChecklist(blocks)).toBe(true); + expect(preservesVisibleWebsiteChecklist([blocks[0]!,...websiteChecklist.toReversed()])).toBe(false); + for(let index=0;index<4;index++)expect(preservesVisibleWebsiteChecklist(websiteChecklist.filter((_,i)=>i!==index))).toBe(false); + expect(preservesVisibleWebsiteChecklist(["Website review checklist","Type here to edit"])).toBe(false); + expect(preservesVisibleWebsiteChecklist([...websiteChecklist.slice(0,3),"375 pixel viewport"])).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index 6aa21608fd8..90343be52fe 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -10,17 +10,37 @@ import { checks, observation, project, type InternalObservation } from "./intern import { integratorDefinition, groundedStatement, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; export const websiteChecklist = ["Navigate every interactive control using only the keyboard.", "Verify every form field has a visible label.", "Check text contrast meets WCAG AA.", "Test the landing page at a 375 pixel viewport without horizontal scrolling."]; -export function preservesWebsiteChecklist(markdown: string): boolean { +function websiteChecklistChecks(markdown: string): boolean[] { const instructions = markdown.split(/\n+|(?<=[.!?])\s+/).filter(Boolean); // A numbered check and its indented detail form one instruction. Keep // separate checks isolated so an unrelated earlier width cannot ground it. const blocks = markdown.split(/\n(?=\s*(?:\d+[.)]|#{1,6})\s)/).map(block => block.replace(/\s+/g, " ")); const forbidden = /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i; const instruction = (requirements: RegExp[]) => instructions.some(text => requirements.every(pattern => pattern.test(text)) && !forbidden.test(text)); - return instruction([/\bkeyboard\b/i, /(?:\b(?:all|every)\b.*\b(?:interactive|control|element|action|navigation)|\b(?:interactive|control|element|action|navigation)\b.*\b(?:all|every)\b)/i, /(?:only|alone|without (?:a |the )?mouse|no mouse)/i]) - && instruction([/\b(?:all|every|each)\b/i, /\b(?:form|input)\b/i, /\b(?:field|control|input)s?\b/i, /\bvisible\b/i, /\blabels?\b/i]) - && instruction([/\b(?:text|foreground)\b/i, /\bcontrast\b/i, /\bWCAG(?:\s+2\.\d)?\s+(?:level\s+)?AA\b/i, /\b(?:check|verify|ensure|measure|assess|meet|meets|satisfy)\b/i]) - && blocks.some(text => !forbidden.test(text) && [/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally|(?:confirm|check|verify)[^.!?]{0,70}whether[^.!?]{0,70}horizontal scrolling)/i].every(pattern => pattern.test(text))); + return [instruction([/\bkeyboard\b/i, /(?:\b(?:all|every)\b.*\b(?:interactive|control|element|action|navigation)|\b(?:interactive|control|element|action|navigation)\b.*\b(?:all|every)\b)/i, /(?:only|alone|without (?:a |the )?mouse|no mouse)/i]), + instruction([/\b(?:all|every|each)\b/i, /\b(?:form|input)\b/i, /\b(?:field|control|input)s?\b/i, /\bvisible\b/i, /\blabels?\b/i]), + instruction([/\b(?:text|foreground)\b/i, /\bcontrast\b/i, /\bWCAG(?:\s+2\.\d)?\s+(?:level\s+)?AA\b/i, /\b(?:check|verify|ensure|measure|assess|meet|meets|satisfy)\b/i]), + blocks.some(text => !forbidden.test(text) && [/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally|(?:confirm|check|verify)[^.!?]{0,70}whether[^.!?]{0,70}horizontal scrolling)/i].every(pattern => pattern.test(text)))]; +} +export function preservesWebsiteChecklist(markdown: string): boolean { + return websiteChecklistChecks(markdown).every(Boolean); +} +/** Rich editor paragraphs/list items are semantic blocks, not Markdown source. */ +export function preservesVisibleWebsiteChecklist(blocks: readonly string[]): boolean { + if (!Array.isArray(blocks) || !blocks.length || blocks.some(block => typeof block !== "string")) return false; + const checks = blocks.map(websiteChecklistChecks); + let previous = 0; + for (let requirement = 0; requirement < 4; requirement++) { + const index = checks.findIndex((row, index) => index >= previous && row[requirement]); + if (index < 0) return false; + previous = index; + } + return true; +} +/** Serializable Playwright DOM projection; no editor text or Markdown is invented. */ +export function authoredChecklistTextBlocks(nodes: Element[]): string[] { + return nodes.filter(node => node.tagName === "LI" ? !node.parentElement?.closest("li") : !node.closest("li")) + .map(node => node.textContent ?? ""); } interface SkillsObservation extends InternalObservation { skills: CompanySkillDetail[]; versions: Record; @@ -96,12 +116,13 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor await expect(ctx.input.page.getByText(skill.name, { exact: true }).first()).toBeVisible({ timeout: remaining() }); const editor = ctx.input.page.getByRole("textbox", { name: "editable markdown", exact: true }); await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() }); - await expect.poll(async () => preservesWebsiteChecklist(await editor.innerText()), { timeout: remaining() }).toBe(true); - const body = await editor.innerText(); + const readBlocks = () => editor.locator("p, li").evaluateAll(authoredChecklistTextBlocks); + await expect.poll(async () => preservesVisibleWebsiteChecklist(await readBlocks()), { timeout: remaining() }).toBe(true); + const body = await readBlocks(); await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); await expect(ctx.input.page).toHaveURL(url => url.pathname === route, { timeout: remaining() }); await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() }); - await expect.poll(async () => await editor.innerText() === body, { timeout: remaining() }).toBe(true); + await expect.poll(async () => JSON.stringify(await readBlocks()) === JSON.stringify(body), { timeout: remaining() }).toBe(true); const after = await ctx.input.api.get(`${root(ctx)}/skills/${encodeURIComponent(skill.id)}`, { timeout: remaining() }); if (after.id !== skill.id || after.companyId !== observed.companyId || !after.editable || after.currentVersionId !== skill.currentVersionId || after.markdown !== markdown) throw new Error("Authored skill identity, saved version or bytes changed during inspection"); remaining(); @@ -114,7 +135,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => { const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill); base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"]; - return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), + return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx"], queryAttachments: factualContextQueryAttachment("skill-context.md"), async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); }, async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); }, } : base;