diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts new file mode 100644 index 00000000000..acf67a8889f --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts @@ -0,0 +1,71 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { InternalObservation } from "./internal-support.js"; +vi.mock("@playwright/test",()=>({expect:Object.assign((node:any)=>({toBeVisible:async()=>{if(!node.visible)throw Error("Missing real UI item");},toBeEditable:async()=>{if(!node.editable)throw Error("Noneditable real UI");},toHaveURL:async(predicate:(url:URL)=>boolean)=>{if(!predicate(new URL(node.url())))throw Error("Wrong actual Studio identity");}}),{poll:(read:()=>Promise)=>({toBe:async(expected:boolean)=>{for(let i=0;i<3;i++)if(await read()===expected)return;throw Error("UI content did not hydrate to the saved checklist");}})})})); +const observed=vi.hoisted(()=>({base:undefined as unknown})); +vi.mock("./integrator-support.js",async original=>({...await original(),observeInternal:vi.fn(async(_ctx:unknown,state:unknown)=>({...observed.base as object,state}))})); +import { gradeIntegratorSkills, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js"; +afterEach(()=>vi.restoreAllMocks()); +function fixture(){ + const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0; + const version={id:"revision",companyId:"company",companySkillId:"skill",revisionNumber:1,authorAgentId:"ceo",authorUserId:null,fileInventory:[{path:"SKILL.md",kind:"skill",content:markdown}]}; + const skill={id:"skill",companyId:"company",name:"Website review checklist",editable:true,currentVersionId:"revision",markdown,usedByAgents:[]}; + const state:Record={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:[],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation; + observed.base=o; + const ui={library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,delayedHydration:false};const events:string[]=[]; + const editor={get visible(){return ui.studio;},get editable(){return ui.editable;},innerText:async()=>ui.delayedHydration&&editorReads++===0?"":ui.changedAfterReload&&reloads?"Lost checklist":ui.body}; + const page={url:()=>current,getByRole:(role:string,options?:{name?:string})=>{ + if(role==="textbox")return editor; + if(role==="link")return{first:()=>({click:async()=>{events.push(options!.name!);current=options?.name==="Skills"?"https://fixture.test/ACORN/skills":"https://fixture.test/ACORN/skills/studio";}})}; + return{filter:()=>({first:()=>({get visible(){return ui.library;},click:async()=>{events.push("open-skill");current=ui.wrongRoute?"https://fixture.test/ACORN/skills/studio/other":"https://fixture.test/ACORN/skills/studio/skill";}})})}; + },getByText:()=>({first:()=>({get visible(){return ui.studio;}})}),getByTestId:()=>({get visible(){return current===source;}}),reload:async()=>{reloads++;events.push("reload");},goto:async(url:string)=>{current=url;events.push("return-chat");}}; + const after=structuredClone(skill),get=vi.fn(async(route:string)=>{ + if(route.endsWith("/skills"))return[{id:"skill"}]; + if(route.endsWith("/skills/skill"))return after; + if(route.endsWith("/versions"))return[{...version,fileInventory:[{path:"SKILL.md",kind:"skill"}]}]; + if(route.endsWith("/versions/revision"))return version; + if(route.endsWith("/skill-policy"))return{}; + throw Error("Unexpected public inspection read"); + });const post=vi.fn(async()=>({allowed:true})),evidence=vi.fn(); + const ctx={scenario:{id:"skills.author-skill"},observationDeadlineAt:Date.now()+270000,input:{fixtures:{company:{id:"company",issuePrefix:"ACORN"},agent:{id:"ceo"}},api:{baseURL:o.origin,get,post},page,evidence},refreshIssue:async()=>{},issue:()=>({conversationState:"waiting"}),allRuns:async()=>o.runs} as unknown as ChatCatalogContext; + return{ctx,state,o,skill,version,ui,events,after,get,post,evidence}; +} +it("the actual authoring act performs readonly normal library/Studio inspection and keeps a single native/human turn",async()=>{ + const f=fixture(),d=integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!; + await d.act!(f.ctx,f.state); + expect(f.events).toEqual(["Skills","My Skills","open-skill","reload","return-chat"]); + expect(f.state.authoredSkillInspection).toMatchObject({skillId:"skill",versionId:"revision",readbackMatched:true}); + expect(gradeIntegratorSkills({...f.o,state:f.state}).passed).toBe(true); + expect(d.expectedRunCount).toBe(1);expect(d.expectedHumanMessageCount).toBe(1); + expect(f.post).toHaveBeenCalledExactlyOnceWith("/api/companies/company/skill-policy/evaluate",expect.anything()); +}); +it.each(["missing library","missing Studio","readonly","wrong UI route","wrong UI content","lost after reload","wrong saved version","wrong author","foreign company","no saved version"])("rejects %s before recording inspection success",async invalid=>{ + const f=fixture(); + if(invalid==="missing library")f.ui.library=false; + if(invalid==="missing Studio")f.ui.studio=false; + if(invalid==="readonly")f.ui.editable=false; + if(invalid==="wrong UI route")f.ui.wrongRoute=true; + if(invalid==="wrong UI content")f.ui.body="Review the site."; + if(invalid==="lost after reload")f.ui.changedAfterReload=true; + if(invalid==="wrong saved version")f.after.currentVersionId="other"; + if(invalid==="wrong author")f.version.authorAgentId="other"; + if(invalid==="foreign company")f.skill.companyId="foreign"; + if(invalid==="no saved version")f.skill.currentVersionId="missing"; + await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow();expect(f.state.authoredSkillInspection).toBeUndefined(); +}); +it("rejects invented inspection receipts and keeps other skill cases on their original contract",()=>{ + const f=fixture();expect(gradeIntegratorSkills(f.o).passed).toBe(false); + f.state.authoredSkillInspection={companyId:"company",skillId:"other",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true}; + expect(gradeIntegratorSkills(f.o).passed).toBe(false); + expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1"); +}); +it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{ + const f=fixture();f.ctx.observationDeadlineAt=Date.now()-1; + await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("budget");expect(f.events).toEqual([]); +}); + +it("waits for actual saved editor content instead of rejecting its visible empty hydration shell",async()=>{ + const f=fixture();f.ui.delayedHydration=true; + await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).resolves.toBeUndefined(); + expect(f.state.authoredSkillInspection).toMatchObject({checklistVisible:true,reloaded:true}); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts index 87e647f5963..5d14f9ceaf9 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts @@ -3,10 +3,10 @@ import { gradeIntegratorSkills, integratorSkillsCases, websiteChecklist, preserv function valid(id: string) { const markdown = websiteChecklist.join("\n"); - const version = { id: "revision", companySkillId: "skill", authorAgentId: "ceo", fileInventory: [{ path: "SKILL.md", content: markdown }] }; + const version = { id: "revision", companyId: "company", companySkillId: "skill", authorAgentId: "ceo", fileInventory: [{ path: "SKILL.md", content: markdown }] }; const skill = { id: "skill", companyId: "company", name: "Accessibility review", editable: true, markdown, currentVersionId: "revision", usedByAgents: [] }; const denied = id === "skills.explicit-restriction"; - return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: denied ? { skill: structuredClone(skill), versions: [structuredClone(version)], policyRevision: 3 } : {}, + return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: denied ? { skill: structuredClone(skill), versions: [structuredClone(version)], policyRevision: 3 } : { authoredSkillInspection: {companyId:"company",skillId:"skill",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true} }, tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }], activity: denied ? [] : [{ id: "create", entityId: "skill", action: "company.skill_created", agentId: "ceo" }], reply: denied ? "Company policy denies my skill edits. Ask an administrator to request a change." : "Created the reusable review skill; it is available in Studio. This does not enable it for all agents.", @@ -26,7 +26,7 @@ describe("authored skill outcomes", () => { expect(gradeIntegratorSkills({ ...o, activity: [] }).passed).toBe(false); const missing = structuredClone(o); missing.skills[0]!.markdown = "Review the website."; missing.versions.skill[0]!.fileInventory[0]!.content = "Review the website."; expect(gradeIntegratorSkills(missing).passed).toBe(false); - expect(gradeIntegratorSkills({ ...o, links: ["https://other.test/ACORN/skills/studio/skill"] }).passed).toBe(false); + const notInspected = structuredClone(o); delete (notInspected.state as {authoredSkillInspection?:unknown}).authoredSkillInspection; expect(gradeIntegratorSkills(notInspected).passed).toBe(false); expect(gradeIntegratorSkills({ ...o, reply: "Created and enabled the skill for every agent." }).passed).toBe(false); }); it("rejects a policy denial explained as success, a fictional source limitation, or a stealth fork", () => { @@ -62,3 +62,9 @@ it("preserves a numbered viewport check and its actual multiline inspection deta expect(preservesWebsiteChecklist(markdown.replace("375 pixel", "768 pixel"))).toBe(false); expect(preservesWebsiteChecklist(markdown.replace("Confirm whether", "Ignore whether"))).toBe(false); }); + +it("authoring v2 accepts ordinary Skills/Studio inspection without a prescribed reply link",()=>{ + const o=valid("skills.author-skill");o.links=[]; + Object.assign(o.state,{authoredSkillInspection:{companyId:"company",skillId:"skill",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true}}); + expect(gradeIntegratorSkills(o).passed).toBe(true); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index 7349d05435b..6aa21608fd8 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -1,5 +1,7 @@ import { readSkillVersionHistory } from "./integrator-skill-versions.js"; import { saveSkillConversationContext } from "./integrator-skill-context.js"; +import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js"; +import { expect } from "@playwright/test"; import { factualContextQueryAttachment } from "../chat-query-attachments.js"; import type { CompanySkill, CompanySkillDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js"; import type { EffectiveSkillPolicy, SkillPolicyDecision } from "../../../packages/shared/src/validators/skill-policy.js"; @@ -51,7 +53,9 @@ export function gradeIntegratorSkills(value: unknown) { const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content; c.add("actual-checklist-in-current-revision", !!skill && !!version && version.authorAgentId === o.leadId && typeof markdown === "string" && preservesWebsiteChecklist(markdown) && markdown === skill.markdown, "The current saved SKILL.md revision preserves keyboard-only controls, visible form labels, WCAG AA text contrast and a 375px viewport without horizontal overflow; paraphrase is permitted."); - c.add("inspectable-skill-link", !!skill && !!resultHref(o, skill.id, "studio"), "Chat links the saved editable skill in Studio."); + const inspection = o.state.authoredSkillInspection as AuthoredSkillInspection | undefined; + c.add("inspectable-authored-skill", !!skill && inspection?.companyId === o.companyId && inspection.skillId === skill.id && inspection.versionId === skill.currentVersionId && inspection.libraryVisible === true && inspection.studioVisible === true && inspection.editable === true && inspection.checklistVisible === true && inspection.reloaded === true && inspection.readbackMatched === true, + "The actual saved skill is discoverable through ordinary Skills/My Skills, opens editable Studio content, and survives reload with the same saved version. No specific reply-link format is required."); c.add("creation-not-global-assignment", !!skill && skill.usedByAgents.length === 0 && !/\b(?:enabled|assigned|activated)\s+(?:it|the skill|this skill)?\s*(?:for\s+)?(?:all|every)\b/i.test(o.reply), "Creating a reusable skill does not silently enable it for all agents or claim that happened."); } else if (o.scenarioId === "skills.explicit-restriction") { const before = o.state.skill as CompanySkillDetail; const current = o.skills.find(row => row.id === before?.id); @@ -73,10 +77,47 @@ async function verifySkill(ctx: ChatCatalogContext, state: Record, o: InternalObservation): Promise { + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); + const observed = skillsObservation(o); + const authored = observed?.skills.filter(skill => observed.activity.some(event => event.action === "company.skill_created" && event.entityId === skill.id && event.agentId === observed.leadId)) ?? []; + const skill = authored[0], version = skill && observed?.versions[skill.id]?.find(row => row.id === skill.currentVersionId); + const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content; + if (!observed || observed.companyId !== ctx.input.fixtures.company.id || observed.leadId !== ctx.input.fixtures.agent.id || authored.length !== 1 || !skill || skill.companyId !== observed.companyId || !skill.editable || !skill.currentVersionId || version?.authorAgentId !== observed.leadId || version.companyId !== observed.companyId || version.companySkillId !== skill.id || typeof markdown !== "string" || markdown !== skill.markdown || !preservesWebsiteChecklist(markdown)) throw new Error("No exact authored saved checklist version is available to inspect"); + const source = ctx.input.page.url(); + await ctx.input.page.getByRole("link", { name: "Skills", exact: true }).first().click({ timeout: remaining() }); + await ctx.input.page.getByRole("link", { name: "My Skills", exact: true }).first().click({ timeout: remaining() }); + const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByText(skill.name, { exact: true }) }).first(); + await expect(card).toBeVisible({ timeout: remaining() }); await card.click({ timeout: remaining() }); + const route = `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`; + await expect(ctx.input.page).toHaveURL(url => url.pathname === route, { timeout: remaining() }); + await expect(ctx.input.page.getByText(skill.name, { exact: true }).first()).toBeVisible({ timeout: remaining() }); + const editor = ctx.input.page.getByRole("textbox", { name: "editable markdown", exact: true }); + await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() }); + await expect.poll(async () => preservesWebsiteChecklist(await editor.innerText()), { timeout: remaining() }).toBe(true); + const body = await editor.innerText(); + await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(ctx.input.page).toHaveURL(url => url.pathname === route, { timeout: remaining() }); + await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() }); + await expect.poll(async () => await editor.innerText() === body, { timeout: remaining() }).toBe(true); + const after = await ctx.input.api.get(`${root(ctx)}/skills/${encodeURIComponent(skill.id)}`, { timeout: remaining() }); + if (after.id !== skill.id || after.companyId !== observed.companyId || !after.editable || after.currentVersionId !== skill.currentVersionId || after.markdown !== markdown) throw new Error("Authored skill identity, saved version or bytes changed during inspection"); + remaining(); + state.authoredSkillInspection = { companyId: observed.companyId, skillId: skill.id, versionId: skill.currentVersionId, libraryVisible: true, studioVisible: true, editable: true, checklistVisible: true, reloaded: true, readbackMatched: true } satisfies AuthoredSkillInspection; + await ctx.input.evidence("chat-catalog-authored-skill-ui.json", state.authoredSkillInspection); + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); + await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() }); + remaining(); +} const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => { const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill); base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"]; - return id === "skills.author-skill" ? { ...base, authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md") } : base; + return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), + async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); }, + async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); }, + } : base; }; export const integratorSkillsCases: readonly ChatCatalogCase[] = [ define("skills.author-skill", async ctx => { diff --git a/tests/runner-e2e/chat-scenarios/skills.yaml b/tests/runner-e2e/chat-scenarios/skills.yaml index f9e8cb09d38..e95e1db89f8 100644 --- a/tests/runner-e2e/chat-scenarios/skills.yaml +++ b/tests/runner-e2e/chat-scenarios/skills.yaml @@ -26,8 +26,9 @@ scenarios: Turn our website review checklist into a reusable skill. expected_outcome: >- A writable company skill captures the checklist with clear instructions and a saved - revision. Chat links to the skill so the person can inspect or edit it. The agent - preserves the actual checklist instead of merely posting a suggestion or pretending + revision. The person can discover it through ordinary Skills/My Skills navigation, open + its editable Studio content and reload the same saved revision. No particular chat-link + format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending creation automatically enables the skill for every agent. - id: skills.fork-read-only capabilities: