diff --git a/tests/runner-e2e/CHAT-FEATURES.md b/tests/runner-e2e/CHAT-FEATURES.md index 15240d235a7..f2c0ea6f59d 100644 --- a/tests/runner-e2e/CHAT-FEATURES.md +++ b/tests/runner-e2e/CHAT-FEATURES.md @@ -241,12 +241,14 @@ Only `steering.preserve-draft` and `tasks.attachment-brief` may currently declar ### Authored skill inspection -`skills.author-skill` v3 follows the normal library card to the exact saved skill. +`skills.author-skill` v4 follows the normal library card to the exact saved skill. It accepts the canonical company-scoped detail route or the exact Studio route. The detail view opens Overview first; select Files, then its normal Edit control. Both paths must show the editable saved checklist, survive reload, and match the same public current version. The receipt identifies the actual editor surface. The original case -clock, query, and one-human/one-native limits remain unchanged. Older failed +clock, query, and one-human/one-native limits remain unchanged. The 375px check +rejects omission or waiver of the required review; a protective instruction +that content must not overflow does not waive that review. Older failed attempts are retained; component controls do not establish a native pass. ### Six prerequisite probes are not completed journeys diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 403781f8d84..ef369bae7f5 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5085,7 +5085,7 @@ "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts" ], - "oracleVersion": "paperclip.chat-catalog.skill-authoring.v3", + "oracleVersion": "paperclip.chat-catalog.skill-authoring.v4", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts index 09d28b4f441..90d39e10f8a 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts @@ -63,8 +63,17 @@ it("preserves a numbered viewport check and its actual multiline inspection deta expect(preservesWebsiteChecklist(markdown.replace("Confirm whether", "Ignore whether"))).toBe(false); }); -it("authoring v3 accepts ordinary Skills/Studio inspection without a prescribed reply link",()=>{ +it("authoring v4 accepts ordinary Skills/Studio inspection without a prescribed reply link",()=>{ const o=valid("skills.author-skill");o.links=[]; Object.assign(o.state,{authoredSkillInspection:{companyId:"company",skillId:"skill",versionId:"revision",libraryVisible:true,editorVisible:true,editorSurface:"studio",editable:true,checklistVisible:true,reloaded:true,readbackMatched:true}}); expect(gradeIntegratorSkills(o).passed).toBe(true); }); + +it("preserves required mobile checks beside protective nonoverflow instructions without allowing a waiver", () => { + const original = websiteChecklist.join("\n"); + const protection = "Confirm images, tables, and code blocks do not overflow."; + expect(preservesWebsiteChecklist(`${original}\n- ${protection}`)).toBe(true); + expect(preservesWebsiteChecklist(original.replace(websiteChecklist[3]!, `${websiteChecklist[3]!.slice(0, -1)}; ${protection}`))).toBe(true); + expect(preservesWebsiteChecklist(`${original.replace(websiteChecklist[3]!, `Do not test the landing page at a 375 pixel viewport without horizontal scrolling.`)}\n- ${protection}`)).toBe(false); + expect(preservesWebsiteChecklist(`${original.replace(websiteChecklist[3]!, `Testing the landing page at a 375 pixel viewport without horizontal scrolling is optional.`)}\n- ${protection}`)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index a12d5c09f96..c9708dd3db0 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -17,11 +17,12 @@ function websiteChecklistChecks(markdown: string): boolean[] { // separate checks isolated so an unrelated earlier width cannot ground it. const blocks = markdown.split(/\n(?=\s*(?:\d+[.)]|#{1,6})\s)/).map(block => block.replace(/\s+/g, " ")); const forbidden = /\b(?:skip|omit|ignore|don't|do not|need not|not required|optional)\b/i; + const mobileWaiver = /\b(?:skip|omit|ignore|need not|not required|optional)\b|\b(?:don't|do not)\s+(?:check|test|verify|confirm|assess|measure|review)\b/i; const instruction = (requirements: RegExp[]) => instructions.some(text => requirements.every(pattern => pattern.test(text)) && !forbidden.test(text)); return [instruction([/\bkeyboard\b/i, /(?:\b(?:all|every)\b.*\b(?:interactive|control|element|action|navigation)|\b(?:interactive|control|element|action|navigation)\b.*\b(?:all|every)\b)/i, /(?:only|alone|without (?:a |the )?mouse|no mouse)/i]), instruction([/\b(?:all|every|each)\b/i, /\b(?:form|input)\b/i, /\b(?:field|control|input)s?\b/i, /\bvisible\b/i, /\blabels?\b/i]), instruction([/\b(?:text|foreground)\b/i, /\bcontrast\b/i, /\bWCAG(?:\s+2\.\d)?\s+(?:level\s+)?AA\b/i, /\b(?:check|verify|ensure|measure|assess|meet|meets|satisfy)\b/i]), - blocks.some(text => !forbidden.test(text) && [/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally|(?:confirm|check|verify)[^.!?]{0,70}whether[^.!?]{0,70}horizontal scrolling)/i].every(pattern => pattern.test(text)))]; + blocks.some(text => !mobileWaiver.test(text) && [/\b375\s*(?:px|pixels?)?\b/i, /\b(?:viewport|wide|width)\b/i, /(?:without horizontal (?:scrolling|overflow)|no horizontal (?:scrolling|overflow)|(?:must not|does not|doesn't) (?:scroll|overflow) horizontally|(?:confirm|check|verify)[^.!?]{0,70}whether[^.!?]{0,70}horizontal scrolling)/i].every(pattern => pattern.test(text)))]; } export function preservesWebsiteChecklist(markdown: string): boolean { return websiteChecklistChecks(markdown).every(Boolean); @@ -148,7 +149,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => { const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill); base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"]; - return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v3", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), + return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v4", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); }, async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); }, } : base;