diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index e2cabeea59b..39f3fa315bd 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -5120,7 +5120,7 @@ "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts" ], - "oracleVersion": "paperclip.chat-catalog.skill-authoring.v6", + "oracleVersion": "paperclip.chat-catalog.skill-authoring.v7", "expectedRunCount": 1, "expectedHumanMessageCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts index 35ed48645d2..0779ea48275 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts @@ -11,7 +11,7 @@ function fixture(){ const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0; const version={id:"revision",companyId:"company",companySkillId:"skill",revisionNumber:1,authorAgentId:"ceo",authorUserId:null,fileInventory:[{path:"SKILL.md",kind:"skill",content:markdown}]}; const skill={id:"skill",companyId:"company",name:"Website review checklist",slug:"website-review-checklist",editable:true,currentVersionId:"revision",markdown,usedByAgents:[]}; - const state:Record={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:[],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation; + const state:Record={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:["/skills/studio/skill"],renderedLinkEvidence:[{threadTestId:"task-chat-thread",href:"/skills/studio/skill",source:"reply",conversationId:"conversation",anchorCommentId:"query",commentId:"reply"}],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation; observed.base=o; const ui={library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,foreignRoute:false,directStudio:false,delayedHydration:false,inEdit:false,filesSelected:false};const events:string[]=[]; const editor={get visible(){return ui.studio && (ui.inEdit || ui.directStudio);},get editable(){return ui.editable;},locator:()=>({evaluateAll:async()=>ui.delayedHydration&&editorReads++===0?[]:(ui.changedAfterReload&&reloads?"Lost checklist":ui.body).split("\n")})}; @@ -21,7 +21,7 @@ function fixture(){ if(role==="button" && options?.name==="Edit")return{click:async()=>{if(!ui.editable||!ui.filesSelected)throw Error("No Edit control outside Files");ui.inEdit=true;events.push("Edit");}}; if(role==="link")return{first:()=>({click:async()=>{events.push(options!.name!);current=options?.name==="Skills"?"https://fixture.test/ACORN/skills":"https://fixture.test/ACORN/skills/studio";}})}; return{filter:()=>({first:()=>({get visible(){return ui.library;},click:async()=>{events.push("open-skill");current=ui.foreignRoute?"https://fixture.test/FOREIGN/skills/website-review-checklist":ui.wrongRoute?"https://fixture.test/ACORN/skills/other":ui.directStudio?"https://fixture.test/ACORN/skills/studio/skill":"https://fixture.test/ACORN/skills/website-review-checklist";}})})}; - },getByText:()=>({first:()=>({get visible(){return ui.studio;}})}),getByTestId:()=>({get visible(){return current===source;}}),reload:async()=>{reloads++;ui.inEdit=false;ui.filesSelected=false;events.push("reload");},goto:async(url:string)=>{current=url;events.push("return-chat");}}; + },getByText:()=>({first:()=>({get visible(){return ui.studio;}})}),locator:()=>({}),getByTestId:()=>({get visible(){return current===source;},filter:()=>({locator:(selector:string)=>({first:()=>({click:async()=>{if(!selector.includes('"/skills/studio/skill"'))throw Error("Not the rendered skill link");events.push("reply-link");current="https://fixture.test/ACORN/skills/studio/skill";}})})})}),reload:async()=>{reloads++;ui.inEdit=false;ui.filesSelected=false;events.push("reload");},goto:async(url:string)=>{current=url;events.push("return-chat");}}; const after=structuredClone(skill),get=vi.fn(async(route:string)=>{ if(route.endsWith("/skills"))return[{id:"skill"}]; if(route.endsWith("/skills/skill"))return after; @@ -36,7 +36,7 @@ function fixture(){ it("the actual authoring act performs nonmutating canonical-library/Edit inspection and keeps a single native/human turn",async()=>{ const f=fixture(),d=integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!; await d.act!(f.ctx,f.state); - expect(f.events).toEqual(["Skills","My Skills","open-skill","Files","Edit","reload","Files","Edit","return-chat"]); + expect(f.events).toEqual(["reply-link","My Skills","open-skill","Files","Edit","reload","Files","Edit","return-chat"]); expect(f.state.authoredSkillInspection).toMatchObject({skillId:"skill",versionId:"revision",editorSurface:"detail",editorVisible:true,readbackMatched:true}); expect(gradeIntegratorSkills({...f.o,state:f.state}).passed).toBe(true); expect(d.expectedRunCount).toBe(1);expect(d.expectedHumanMessageCount).toBe(1); @@ -62,7 +62,12 @@ it("rejects invented inspection receipts and keeps other skill cases on their or f.state.authoredSkillInspection={companyId:"company",skillId:"other",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true}; expect(gradeIntegratorSkills(f.o).passed).toBe(false); expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1"); - expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v6"); + expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v7"); +}); +it.each(["no link","link to another skill"])("rejects a reply with %s before any UI navigation",async fault=>{ + const f=fixture(),href=fault==="no link"?null:"/skills/studio/other"; + Object.assign(f.o,{links:href?[href]:[],renderedLinkEvidence:href?[{threadTestId:"task-chat-thread",href,source:"reply",conversationId:"conversation",anchorCommentId:"query",commentId:"reply"}]:[]}); + await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("did not link the saved skill");expect(f.events).toEqual([]);expect(f.state.authoredSkillInspection).toBeUndefined(); }); it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{ const f=fixture();Object.assign(f.ctx,{observationDeadlineAt:Date.now()-1}); @@ -84,7 +89,7 @@ it("inspects real rich-text blocks without applying Markdown marker assumptions expect(preservesVisibleWebsiteChecklist([...websiteChecklist.slice(0,3),"375 pixel viewport"])).toBe(false); }); -it("retains exact direct Studio card navigation when the UI uses it", async()=>{ const f=fixture();f.ui.directStudio=true;await inspectAuthoredSkill(f.ctx,f.state,f.o);expect(f.events).toEqual(["Skills","My Skills","open-skill","reload","return-chat"]);expect(f.state.authoredSkillInspection).toMatchObject({readbackMatched:true,editorSurface:"studio"}); }); +it("retains exact direct Studio card navigation when the UI uses it", async()=>{ const f=fixture();f.ui.directStudio=true;await inspectAuthoredSkill(f.ctx,f.state,f.o);expect(f.events).toEqual(["reply-link","My Skills","open-skill","reload","return-chat"]);expect(f.state.authoredSkillInspection).toMatchObject({readbackMatched:true,editorSurface:"studio"}); }); it.each([undefined, null, ""])("rejects missing company prefix %s before any UI navigation", async prefix => { const f=fixture();Object.assign(f.ctx.input.fixtures.company,{issuePrefix:prefix}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index 7bf3ccee0df..0b260d5d636 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -7,7 +7,7 @@ import { factualContextQueryAttachment } from "../chat-query-attachments.js"; import type { CompanySkill, CompanySkillDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js"; import type { EffectiveSkillPolicy, SkillPolicyDecision } from "../../../packages/shared/src/validators/skill-policy.js"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; -import { checks, observation, project, type InternalObservation } from "./internal-support.js"; +import { checks, internalResultLink, observation, project, type InternalObservation } from "./internal-support.js"; import { integratorDefinition, groundedStatement, inspectResult, resultHref, observeInternal } from "./integrator-support.js"; export const websiteChecklist = ["Navigate every interactive control using only the keyboard.", "Verify every form field has a visible label.", "Check text contrast meets WCAG AA.", "Test the landing page at a 375 pixel viewport without horizontal scrolling."]; @@ -129,7 +129,7 @@ export function gradeIntegratorSkills(value: unknown) { && preservesWebsiteChecklist(markdown) && markdown === skill.markdown, "The current saved SKILL.md revision preserves keyboard-only controls, visible form labels, WCAG AA text contrast and a 375px viewport without horizontal overflow; paraphrase is permitted."); const inspection = o.state.authoredSkillInspection as AuthoredSkillInspection | undefined; c.add("inspectable-authored-skill", !!skill && inspection?.companyId === o.companyId && inspection.skillId === skill.id && inspection.versionId === skill.currentVersionId && inspection.libraryVisible === true && inspection.editorVisible === true && (inspection.editorSurface === "detail" || inspection.editorSurface === "studio") && inspection.editable === true && inspection.checklistVisible === true && inspection.reloaded === true && inspection.readbackMatched === true, - "The actual saved skill is discoverable through ordinary Skills/My Skills, opens the actual editable skill content, and survives reload with the same saved version. No specific reply-link format is required."); + "The reply links the actual saved skill; following that link opens it, it is listed in My Skills, opens the actual editable skill content, and survives reload with the same saved version."); c.add("creation-not-global-assignment", authorSkillAssignmentScoped(o, skill, version) && !claimsGlobalSkillAssignment(o.reply), "Creation leaves the skill unassigned or explicitly binds only its acting author in the same real run, with current saved-version and assignment audit evidence and truthful targeted reporting; no global policy change, other-agent assignment or false global activation claim is admitted."); } else if (o.scenarioId === "skills.explicit-restriction") { const before = o.state.skill as CompanySkillDetail; const current = o.skills.find(row => row.id === before?.id); @@ -163,13 +163,20 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content; if (!observed || observed.companyId !== ctx.input.fixtures.company.id || observed.leadId !== ctx.input.fixtures.agent.id || authored.length !== 1 || !skill || skill.companyId !== observed.companyId || !skill.editable || !skill.currentVersionId || version?.authorAgentId !== observed.leadId || version.companyId !== observed.companyId || version.companySkillId !== skill.id || typeof markdown !== "string" || markdown !== skill.markdown || !preservesWebsiteChecklist(markdown)) throw new Error("No exact authored saved checklist version is available to inspect"); const source = ctx.input.page.url(); - await ctx.input.page.getByRole("link", { name: "Skills", exact: true }).first().click({ timeout: remaining() }); + const companyPath = `/${encodeURIComponent(prefix)}`; + const skillPaths = [skillStudioRoute(skill.id), skillRoute(skill, observed.skills)]; + const studioRoute = `${companyPath}${skillPaths[0]}`; + const detailRoute = `${companyPath}${skillPaths[1]}`; + // Skills is no longer a primary sidebar link, so the person opens the new skill from the + // link the reply rendered (unprefixed links redirect into the current company). + const href = o.links.find(link => { try { const path = new URL(link, source).pathname; return skillPaths.some(route => path === route || path === `${companyPath}${route}`); } catch { return false; } }); + if (!href) throw new Error("The reply did not link the saved skill, so the person cannot open it from the chat"); + await internalResultLink(ctx, o, href).first().click({ timeout: remaining() }); + await expect(ctx.input.page).toHaveURL(url => url.pathname === studioRoute || url.pathname === detailRoute, { timeout: remaining() }); + // From the skill, the Skills section's own My Skills link shows it in the person's library. await ctx.input.page.getByRole("link", { name: "My Skills", exact: true }).first().click({ timeout: remaining() }); const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByText(skill.name, { exact: true }) }).first(); await expect(card).toBeVisible({ timeout: remaining() }); await card.click({ timeout: remaining() }); - const companyPath = `/${encodeURIComponent(prefix)}`; - const studioRoute = `${companyPath}${skillStudioRoute(skill.id)}`; - const detailRoute = `${companyPath}${skillRoute(skill, observed.skills)}`; const openSavedEditor = async () => { await expect(ctx.input.page).toHaveURL(url => url.origin === new URL(source).origin && (url.pathname === studioRoute || url.pathname === detailRoute), { timeout: remaining() }); @@ -201,7 +208,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => { const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill); base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"]; - return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v6", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), + return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v7", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"), async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); }, async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); }, } : base;