diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts index 7140270093c..190b9402d65 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts @@ -1,10 +1,11 @@ +import { createRequire } from "node:module"; import { afterEach, expect, it, vi } from "vitest"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; import type { InternalObservation } from "./internal-support.js"; vi.mock("@playwright/test",()=>({expect:Object.assign((node:any)=>({toBeVisible:async()=>{if(!node.visible)throw Error("Missing real UI item");},toBeEditable:async()=>{if(!node.editable)throw Error("Noneditable real UI");},toHaveURL:async(predicate:(url:URL)=>boolean)=>{if(!predicate(new URL(node.url())))throw Error("Wrong actual Studio identity");}}),{poll:(read:()=>Promise)=>({toBe:async(expected:boolean)=>{for(let i=0;i<3;i++)if(await read()===expected)return;throw Error("UI content did not hydrate to the saved checklist");}})})})); const observed=vi.hoisted(()=>({base:undefined as unknown})); vi.mock("./integrator-support.js",async original=>({...await original(),observeInternal:vi.fn(async(_ctx:unknown,state:unknown)=>({...observed.base as object,state}))})); -import { gradeIntegratorSkills, preservesVisibleWebsiteChecklist, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js"; +import { gradeIntegratorSkills, authoredChecklistTextBlocks, preservesVisibleWebsiteChecklist, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js"; afterEach(()=>vi.restoreAllMocks()); function fixture(){ const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0; @@ -89,3 +90,25 @@ it.each([undefined, null, ""])("rejects missing company prefix %s before any UI await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("company prefix"); expect(f.events).toEqual([]);expect(f.state.authoredSkillInspection).toBeUndefined(); }); + +const { JSDOM } = createRequire(import.meta.url)("../../../server/node_modules/jsdom"); +const sectionMarkup = `

Keyboard-only navigation

+

Visible form labels

+

WCAG AA text contrast

+

375px viewport without horizontal scrolling

`; +function renderedSections(markup: string) { + const document = new JSDOM(`
${markup}
`).window.document; + return authoredChecklistTextBlocks(Array.from(document.querySelector('[role="textbox"]')!.querySelectorAll("h1,h2,h3,h4,h5,h6,p,li")) as Element[]); +} +it("reads actual editor heading context with its following steps rather than isolated list items",()=>{ + expect(preservesVisibleWebsiteChecklist(renderedSections(sectionMarkup))).toBe(true); +}); +it.each(["wrong section","missing check","heading only","negated directive","negated keyboard"])("rejects incomplete editor sections: %s",fault=>{ + let markup=sectionMarkup; + if(fault==="wrong section")markup=markup.replace("

375px viewport without horizontal scrolling", "

Archived dimensions

The earlier width was 375 pixels.

Responsive review").replace("375 pixel wide viewport","wide viewport"); + if(fault==="missing check")markup=markup.replace(/

WCAG AA text contrast<\/h3>
    .*?<\/ul>/,""); + if(fault==="heading only")markup=markup.replace("
    • Load the landing page at a 375 pixel wide viewport.
    • Confirm no horizontal scrollbar appears.
    ",""); + if(fault==="negated directive")markup=markup.replace("Load the landing page", "Do not test the landing page"); + if(fault==="negated keyboard")markup=markup.replace("Do not use a mouse or touch input.", "Do not use the keyboard."); + expect(preservesVisibleWebsiteChecklist(renderedSections(markup))).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index c0e2fe7c886..97005dab5ed 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -11,12 +11,14 @@ import { checks, observation, project, type InternalObservation } from "./intern import { integratorDefinition, groundedStatement, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; export const websiteChecklist = ["Navigate every interactive control using only the keyboard.", "Verify every form field has a visible label.", "Check text contrast meets WCAG AA.", "Test the landing page at a 375 pixel viewport without horizontal scrolling."]; -function websiteChecklistChecks(markdown: string): boolean[] { - const instructions = markdown.split(/\n+|(?<=[.!?])\s+/).filter(Boolean); +function websiteChecklistChecks(markdown: string, section = false): boolean[] { + const instructions = section ? [markdown.replace(/\s+/g, " ")] : markdown.split(/\n+|(?<=[.!?])\s+/).filter(Boolean); // A numbered check and its indented detail form one instruction. Keep // separate checks isolated so an unrelated earlier width cannot ground it. const blocks = markdown.split(/\n(?=\s*(?:\d+[.)]|#{1,6})\s)/).map(block => block.replace(/\s+/g, " ")); - const forbidden = /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i; + const forbidden = section + ? /\b(?:skip|omit|ignore|need not|not required|optional)\b|\b(?:don't|do not|never)\s+(?:navigate|check|verify|test|review|inspect|assess|measure|ensure|use\s+(?:(?:the|a)\s+)?keyboard)\b/i + : /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i; const mobileRequirements = [/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally|(?:confirm|check|verify)[^.!?]{0,70}whether[^.!?]{0,70}horizontal scrolling)/i]; const mobileWaiver = (text: string) => { const clauses = text.split(/[.!?;\n]+|,\s*(?:but\s+)?(?=(?:don't|do not|never|avoid|no need to)\b)/i); @@ -48,7 +50,7 @@ export function preservesWebsiteChecklist(markdown: string): boolean { /** Rich editor paragraphs/list items are semantic blocks, not Markdown source. */ export function preservesVisibleWebsiteChecklist(blocks: readonly string[]): boolean { if (!Array.isArray(blocks) || !blocks.length || blocks.some(block => typeof block !== "string")) return false; - const checks = blocks.map(websiteChecklistChecks); + const checks = blocks.map(block => websiteChecklistChecks(block, true)); let previous = 0; for (let requirement = 0; requirement < 4; requirement++) { const index = checks.findIndex((row, index) => index >= previous && row[requirement]); @@ -59,8 +61,15 @@ export function preservesVisibleWebsiteChecklist(blocks: readonly string[]): boo } /** Serializable Playwright DOM projection; no editor text or Markdown is invented. */ export function authoredChecklistTextBlocks(nodes: Element[]): string[] { - return nodes.filter(node => node.tagName === "LI" ? !node.parentElement?.closest("li") : !node.closest("li")) - .map(node => node.textContent ?? ""); + const blocks: string[] = []; let heading: string | undefined, steps: string[] = []; + const flush = () => { if (steps.length) blocks.push([heading, ...steps].filter(Boolean).join("\n")); steps = []; }; + for (const node of nodes.filter(node => node.tagName === "LI" ? !node.parentElement?.closest("li") : !node.closest("li"))) { + const text = node.textContent ?? ""; + if (/^H[1-6]$/.test(node.tagName)) { flush(); heading = text; } + else if (heading !== undefined) { if (text.trim()) steps.push(text); } + else if (text.trim()) blocks.push(text); + } + flush(); return blocks; } interface SkillsObservation extends InternalObservation { skills: CompanySkillDetail[]; versions: Record; @@ -173,7 +182,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor await expect(ctx.input.page.getByText(skill.name, { exact: true }).first()).toBeVisible({ timeout: remaining() }); const editor = ctx.input.page.getByRole("textbox", { name: "editable markdown", exact: true }); await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() }); - const readBlocks = () => editor.locator("p, li").evaluateAll(authoredChecklistTextBlocks); + const readBlocks = () => editor.locator("h1, h2, h3, h4, h5, h6, p, li").evaluateAll(authoredChecklistTextBlocks); await expect.poll(async () => preservesVisibleWebsiteChecklist(await readBlocks()), { timeout: remaining() }).toBe(true); const body = await readBlocks(); await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() });