Inspect authored skill checklist within editor sections

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts

@@ -1,10 +1,11 @@
+import { createRequire } from "node:module";
 import { afterEach, expect, it, vi } from "vitest";
 import type { ChatCatalogContext } from "../chat-catalog-contract.js";
 import type { InternalObservation } from "./internal-support.js";
 vi.mock("@playwright/test",()=>({expect:Object.assign((node:any)=>({toBeVisible:async()=>{if(!node.visible)throw Error("Missing real UI item");},toBeEditable:async()=>{if(!node.editable)throw Error("Noneditable real UI");},toHaveURL:async(predicate:(url:URL)=>boolean)=>{if(!predicate(new URL(node.url())))throw Error("Wrong actual Studio identity");}}),{poll:(read:()=>Promise<boolean>)=>({toBe:async(expected:boolean)=>{for(let i=0;i<3;i++)if(await read()===expected)return;throw Error("UI content did not hydrate to the saved checklist");}})})}));
 const observed=vi.hoisted(()=>({base:undefined as unknown}));
 vi.mock("./integrator-support.js",async original=>({...await original<typeof import("./integrator-support.js")>(),observeInternal:vi.fn(async(_ctx:unknown,state:unknown)=>({...observed.base as object,state}))}));
-import { gradeIntegratorSkills, preservesVisibleWebsiteChecklist, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js";
+import { gradeIntegratorSkills, authoredChecklistTextBlocks, preservesVisibleWebsiteChecklist, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js";
 afterEach(()=>vi.restoreAllMocks());
 function fixture(){
  const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0;
@@ -89,3 +90,25 @@ it.each([undefined, null, ""])("rejects missing company prefix %s before any UI
  await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("company prefix");
  expect(f.events).toEqual([]);expect(f.state.authoredSkillInspection).toBeUndefined();
 });
+
+const { JSDOM } = createRequire(import.meta.url)("../../../server/node_modules/jsdom");
+const sectionMarkup = `<h3>Keyboard-only navigation</h3><ul><li>Do not use a mouse or touch input.</li><li>Press Tab through every interactive control.</li></ul>
+<h3>Visible form labels</h3><ul><li>Locate every form field.</li><li>Confirm each field has a visible label.</li></ul>
+<h3>WCAG AA text contrast</h3><ul><li>Check text contrast meets WCAG AA.</li></ul>
+<h3>375px viewport without horizontal scrolling</h3><ul><li>Load the landing page at a 375 pixel wide viewport.</li><li>Confirm no horizontal scrollbar appears.</li></ul>`;
+function renderedSections(markup: string) {
+ const document = new JSDOM(`<aside>375px viewport without horizontal scrolling</aside><div role="textbox" aria-label="editable markdown">${markup}</div>`).window.document;
+ return authoredChecklistTextBlocks(Array.from(document.querySelector('[role="textbox"]')!.querySelectorAll("h1,h2,h3,h4,h5,h6,p,li")) as Element[]);
+}
+it("reads actual editor heading context with its following steps rather than isolated list items",()=>{
+ expect(preservesVisibleWebsiteChecklist(renderedSections(sectionMarkup))).toBe(true);
+});
+it.each(["wrong section","missing check","heading only","negated directive","negated keyboard"])("rejects incomplete editor sections: %s",fault=>{
+ let markup=sectionMarkup;
+ if(fault==="wrong section")markup=markup.replace("<h3>375px viewport without horizontal scrolling", "<h3>Archived dimensions</h3><p>The earlier width was 375 pixels.</p><h3>Responsive review").replace("375 pixel wide viewport","wide viewport");
+ if(fault==="missing check")markup=markup.replace(/<h3>WCAG AA text contrast<\/h3><ul>.*?<\/ul>/,"");
+ if(fault==="heading only")markup=markup.replace("<ul><li>Load the landing page at a 375 pixel wide viewport.</li><li>Confirm no horizontal scrollbar appears.</li></ul>","");
+ if(fault==="negated directive")markup=markup.replace("Load the landing page", "Do not test the landing page");
+ if(fault==="negated keyboard")markup=markup.replace("Do not use a mouse or touch input.", "Do not use the keyboard.");
+ expect(preservesVisibleWebsiteChecklist(renderedSections(markup))).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/integrator-skills.ts

@@ -11,12 +11,14 @@ import { checks, observation, project, type InternalObservation } from "./intern
 import { integratorDefinition, groundedStatement, inspectResult, resultHref, observeInternal } from "./integrator-support.js";
 
 export const websiteChecklist = ["Navigate every interactive control using only the keyboard.", "Verify every form field has a visible label.", "Check text contrast meets WCAG AA.", "Test the landing page at a 375 pixel viewport without horizontal scrolling."];
-function websiteChecklistChecks(markdown: string): boolean[] {
-  const instructions = markdown.split(/\n+|(?<=[.!?])\s+/).filter(Boolean);
+function websiteChecklistChecks(markdown: string, section = false): boolean[] {
+  const instructions = section ? [markdown.replace(/\s+/g, " ")] : markdown.split(/\n+|(?<=[.!?])\s+/).filter(Boolean);
   // A numbered check and its indented detail form one instruction. Keep
   // separate checks isolated so an unrelated earlier width cannot ground it.
   const blocks = markdown.split(/\n(?=\s*(?:\d+[.)]|#{1,6})\s)/).map(block => block.replace(/\s+/g, " "));
-  const forbidden = /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i;
+  const forbidden = section
+    ? /\b(?:skip|omit|ignore|need not|not required|optional)\b|\b(?:don't|do not|never)\s+(?:navigate|check|verify|test|review|inspect|assess|measure|ensure|use\s+(?:(?:the|a)\s+)?keyboard)\b/i
+    : /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i;
   const mobileRequirements = [/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally|(?:confirm|check|verify)[^.!?]{0,70}whether[^.!?]{0,70}horizontal scrolling)/i];
   const mobileWaiver = (text: string) => {
     const clauses = text.split(/[.!?;\n]+|,\s*(?:but\s+)?(?=(?:don't|do not|never|avoid|no need to)\b)/i);
@@ -48,7 +50,7 @@ export function preservesWebsiteChecklist(markdown: string): boolean {
 /** Rich editor paragraphs/list items are semantic blocks, not Markdown source. */
 export function preservesVisibleWebsiteChecklist(blocks: readonly string[]): boolean {
   if (!Array.isArray(blocks) || !blocks.length || blocks.some(block => typeof block !== "string")) return false;
-  const checks = blocks.map(websiteChecklistChecks);
+  const checks = blocks.map(block => websiteChecklistChecks(block, true));
   let previous = 0;
   for (let requirement = 0; requirement < 4; requirement++) {
     const index = checks.findIndex((row, index) => index >= previous && row[requirement]);
@@ -59,8 +61,15 @@ export function preservesVisibleWebsiteChecklist(blocks: readonly string[]): boo
 }
 /** Serializable Playwright DOM projection; no editor text or Markdown is invented. */
 export function authoredChecklistTextBlocks(nodes: Element[]): string[] {
-  return nodes.filter(node => node.tagName === "LI" ? !node.parentElement?.closest("li") : !node.closest("li"))
-    .map(node => node.textContent ?? "");
+  const blocks: string[] = []; let heading: string | undefined, steps: string[] = [];
+  const flush = () => { if (steps.length) blocks.push([heading, ...steps].filter(Boolean).join("\n")); steps = []; };
+  for (const node of nodes.filter(node => node.tagName === "LI" ? !node.parentElement?.closest("li") : !node.closest("li"))) {
+    const text = node.textContent ?? "";
+    if (/^H[1-6]$/.test(node.tagName)) { flush(); heading = text; }
+    else if (heading !== undefined) { if (text.trim()) steps.push(text); }
+    else if (text.trim()) blocks.push(text);
+  }
+  flush(); return blocks;
 }
 interface SkillsObservation extends InternalObservation {
   skills: CompanySkillDetail[]; versions: Record<string, CompanySkillVersion[]>;
@@ -173,7 +182,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor
   await expect(ctx.input.page.getByText(skill.name, { exact: true }).first()).toBeVisible({ timeout: remaining() });
   const editor = ctx.input.page.getByRole("textbox", { name: "editable markdown", exact: true });
   await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() });
-  const readBlocks = () => editor.locator("p, li").evaluateAll(authoredChecklistTextBlocks);
+  const readBlocks = () => editor.locator("h1, h2, h3, h4, h5, h6, p, li").evaluateAll(authoredChecklistTextBlocks);
   await expect.poll(async () => preservesVisibleWebsiteChecklist(await readBlocks()), { timeout: remaining() }).toBe(true);
   const body = await readBlocks();
   await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() });