diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts index b29e9f32607..953a83f20df 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { gradeIntegratorSkills, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js"; +import { gradeIntegratorSkills, integratorSkillsCases, websiteChecklist, preservesWebsiteChecklist } from "./integrator-skills.js"; function valid(id: string) { const markdown = websiteChecklist.join("\n"); @@ -38,4 +38,13 @@ describe("authored skill outcomes", () => { const changed = structuredClone(o); changed.skills[0]!.markdown = "Skip all checks."; expect(gradeIntegratorSkills(changed).passed).toBe(false); }); + it("accepts substantive checklist paraphrases but rejects dropped, negated and wrong-threshold checks", () => { + const paraphrase = "Ensure all interactive elements are reachable and usable with the keyboard alone.\nEach form input needs a visible label.\nMeasure foreground text contrast to verify WCAG 2.2 level AA.\nUse a viewport 375px wide; there must be no horizontal overflow."; + expect(preservesWebsiteChecklist(paraphrase)).toBe(true); + const o = valid("skills.author-skill"); o.skills[0]!.markdown = paraphrase; o.versions.skill[0]!.fileInventory[0]!.content = paraphrase; + expect(gradeIntegratorSkills(o).passed).toBe(true); + for (const changed of [paraphrase.replace("level AA", "level A"), paraphrase.replace("375px", "768px"), paraphrase.replace("Measure foreground", "Do not measure foreground"), paraphrase.replace("Each form input needs a visible label.", "Labels are optional."), paraphrase.replace("no horizontal overflow", "horizontal overflow")]) { + expect(preservesWebsiteChecklist(changed), changed).toBe(false); + } + }); }); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index 03380f141ba..283f04775d6 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -5,6 +5,15 @@ import { checks, observation, project, task, putDocument, type InternalObservati import { integratorDefinition, groundedStatement, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; export const websiteChecklist = ["Navigate every interactive control using only the keyboard.", "Verify every form field has a visible label.", "Check text contrast meets WCAG AA.", "Test the landing page at a 375 pixel viewport without horizontal scrolling."]; +export function preservesWebsiteChecklist(markdown: string): boolean { + const instructions = markdown.split(/\n+|(?<=[.!?])\s+/).filter(Boolean); + const forbidden = /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i; + const instruction = (requirements: RegExp[]) => instructions.some(text => requirements.every(pattern => pattern.test(text)) && !forbidden.test(text)); + return instruction([/\bkeyboard\b/i, /(?:\b(?:all|every)\b.*\b(?:interactive|control|element|action|navigation)|\b(?:interactive|control|element|action|navigation)\b.*\b(?:all|every)\b)/i, /(?:only|alone|without (?:a |the )?mouse|no mouse)/i]) + && instruction([/\b(?:all|every|each)\b/i, /\b(?:form|input)\b/i, /\b(?:field|control|input)s?\b/i, /\bvisible\b/i, /\blabels?\b/i]) + && instruction([/\b(?:text|foreground)\b/i, /\bcontrast\b/i, /\bWCAG(?:\s+2\.\d)?\s+(?:level\s+)?AA\b/i, /\b(?:check|verify|ensure|measure|assess|meet|meets|satisfy)\b/i]) + && instruction([/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally)/i]); +} interface SkillsObservation extends InternalObservation { skills: CompanySkillDetail[]; versions: Record; policy: EffectiveSkillPolicy; editDecision: SkillPolicyDecision; @@ -35,7 +44,7 @@ export function gradeIntegratorSkills(value: unknown) { const version = skill && o.versions[skill.id]?.find(row => row.id === skill.currentVersionId); const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content; c.add("actual-checklist-in-current-revision", !!skill && !!version && version.authorAgentId === o.leadId && typeof markdown === "string" - && websiteChecklist.every(item => markdown.includes(item)) && markdown === skill.markdown, "The current saved SKILL.md revision preserves all four actual checklist requirements."); + && preservesWebsiteChecklist(markdown) && markdown === skill.markdown, "The current saved SKILL.md revision preserves keyboard-only controls, visible form labels, WCAG AA text contrast and a 375px viewport without horizontal overflow; paraphrase is permitted."); c.add("inspectable-skill-link", !!skill && !!resultHref(o, skill.id, "studio"), "Chat links the saved editable skill in Studio."); c.add("creation-not-global-assignment", !!skill && skill.usedByAgents.length === 0 && !/\b(?:enabled|assigned|activated)\s+(?:it|the skill|this skill)?\s*(?:for\s+)?(?:all|every)\b/i.test(o.reply), "Creating a reusable skill does not silently enable it for all agents or claim that happened."); } else if (o.scenarioId === "skills.explicit-restriction") {