mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
test(evals): open an authored skill from its Skill created card when the reply has no link
The create_skill tool shows a Skill created card whose side panel opens the skill in Skill Studio. That card is the product's own path, so the person uses it when the reply has no skill link. Fail only when neither path exists. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
44e54d6185
commit
b3c3e1ef1c
2 files changed
+21
-9
No files matched your search
@@ -13,10 +13,12 @@ function fixture(){
|
||||
const skill={id:"skill",companyId:"company",name:"Website review checklist",slug:"website-review-checklist",editable:true,currentVersionId:"revision",markdown,usedByAgents:[]};
|
||||
const state:Record<string,unknown>={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:["/skills/studio/skill"],renderedLinkEvidence:[{threadTestId:"task-chat-thread",href:"/skills/studio/skill",source:"reply",conversationId:"conversation",anchorCommentId:"query",commentId:"reply"}],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation;
|
||||
observed.base=o;
|
||||
const ui={library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,foreignRoute:false,directStudio:false,delayedHydration:false,inEdit:false,filesSelected:false};const events:string[]=[];
|
||||
const ui={card:true,library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,foreignRoute:false,directStudio:false,delayedHydration:false,inEdit:false,filesSelected:false};const events:string[]=[];
|
||||
const editor={get visible(){return ui.studio && (ui.inEdit || ui.directStudio);},get editable(){return ui.editable;},locator:()=>({evaluateAll:async()=>ui.delayedHydration&&editorReads++===0?[]:(ui.changedAfterReload&&reloads?"Lost checklist":ui.body).split("\n")})};
|
||||
const page={url:()=>current,getByRole:(role:string,options?:{name?:string})=>{
|
||||
if(role==="textbox")return editor;
|
||||
if(role==="article")return{getByRole:()=>({first:()=>({click:async()=>{if(options?.name!==`Skill created: ${skill.name}`||!ui.card)throw Error("No Skill created card");events.push("skill-card");}})})};
|
||||
if(role==="button" && options?.name==="Open in Skill Studio")return{click:async()=>{if(!events.includes("skill-card"))throw Error("Skill panel is closed");events.push("open-in-studio");current="https://fixture.test/ACORN/skills/studio/skill";}};
|
||||
if(role==="tab" && options?.name==="Files")return{click:async()=>{ui.filesSelected=true;events.push("Files");}};
|
||||
if(role==="button" && options?.name==="Edit")return{click:async()=>{if(!ui.editable||!ui.filesSelected)throw Error("No Edit control outside Files");ui.inEdit=true;events.push("Edit");}};
|
||||
if(role==="link")return{first:()=>({click:async()=>{events.push(options!.name!);current=options?.name==="Skills"?"https://fixture.test/ACORN/skills":"https://fixture.test/ACORN/skills/studio";}})};
|
||||
@@ -64,10 +66,16 @@ it("rejects invented inspection receipts and keeps other skill cases on their or
|
||||
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1");
|
||||
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v7");
|
||||
});
|
||||
it.each(["no link","link to another skill"])("rejects a reply with %s before any UI navigation",async fault=>{
|
||||
const f=fixture(),href=fault==="no link"?null:"/skills/studio/other";
|
||||
it("opens the skill from its Skill created card when the reply has no skill link",async()=>{
|
||||
const f=fixture();Object.assign(f.o,{links:[],renderedLinkEvidence:[]});
|
||||
await inspectAuthoredSkill(f.ctx,f.state,f.o);
|
||||
expect(f.events.slice(0,4)).toEqual(["skill-card","open-in-studio","My Skills","open-skill"]);
|
||||
expect(f.state.authoredSkillInspection).toMatchObject({readbackMatched:true});
|
||||
});
|
||||
it.each(["no link","link to another skill"])("rejects %s and no Skill created card before inspection succeeds",async fault=>{
|
||||
const f=fixture(),href=fault==="no link"?null:"/skills/studio/other";f.ui.card=false;
|
||||
Object.assign(f.o,{links:href?[href]:[],renderedLinkEvidence:href?[{threadTestId:"task-chat-thread",href,source:"reply",conversationId:"conversation",anchorCommentId:"query",commentId:"reply"}]:[]});
|
||||
await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("did not link the saved skill");expect(f.events).toEqual([]);expect(f.state.authoredSkillInspection).toBeUndefined();
|
||||
await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("No Skill created card");expect(f.state.authoredSkillInspection).toBeUndefined();
|
||||
});
|
||||
it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{
|
||||
const f=fixture();Object.assign(f.ctx,{observationDeadlineAt:Date.now()-1});
|
||||
|
||||
@@ -129,7 +129,7 @@ export function gradeIntegratorSkills(value: unknown) {
|
||||
&& preservesWebsiteChecklist(markdown) && markdown === skill.markdown, "The current saved SKILL.md revision preserves keyboard-only controls, visible form labels, WCAG AA text contrast and a 375px viewport without horizontal overflow; paraphrase is permitted.");
|
||||
const inspection = o.state.authoredSkillInspection as AuthoredSkillInspection | undefined;
|
||||
c.add("inspectable-authored-skill", !!skill && inspection?.companyId === o.companyId && inspection.skillId === skill.id && inspection.versionId === skill.currentVersionId && inspection.libraryVisible === true && inspection.editorVisible === true && (inspection.editorSurface === "detail" || inspection.editorSurface === "studio") && inspection.editable === true && inspection.checklistVisible === true && inspection.reloaded === true && inspection.readbackMatched === true,
|
||||
"The reply links the actual saved skill; following that link opens it, it is listed in My Skills, opens the actual editable skill content, and survives reload with the same saved version.");
|
||||
"The person opens the actual saved skill from the chat (the reply's link or the Skill created card), finds it in My Skills, opens the actual editable skill content, and it survives reload with the same saved version.");
|
||||
c.add("creation-not-global-assignment", authorSkillAssignmentScoped(o, skill, version) && !claimsGlobalSkillAssignment(o.reply), "Creation leaves the skill unassigned or explicitly binds only its acting author in the same real run, with current saved-version and assignment audit evidence and truthful targeted reporting; no global policy change, other-agent assignment or false global activation claim is admitted.");
|
||||
} else if (o.scenarioId === "skills.explicit-restriction") {
|
||||
const before = o.state.skill as CompanySkillDetail; const current = o.skills.find(row => row.id === before?.id);
|
||||
@@ -167,11 +167,15 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor
|
||||
const skillPaths = [skillStudioRoute(skill.id), skillRoute(skill, observed.skills)];
|
||||
const studioRoute = `${companyPath}${skillPaths[0]}`;
|
||||
const detailRoute = `${companyPath}${skillPaths[1]}`;
|
||||
// Skills is no longer a primary sidebar link, so the person opens the new skill from the
|
||||
// link the reply rendered (unprefixed links redirect into the current company).
|
||||
// Skills is no longer a primary sidebar link, so the person opens the new skill from the chat:
|
||||
// the link the reply rendered (unprefixed links redirect into the current company), or else the
|
||||
// "Skill created" card the tool shows, whose side panel opens it in Skill Studio.
|
||||
const href = o.links.find(link => { try { const path = new URL(link, source).pathname; return skillPaths.some(route => path === route || path === `${companyPath}${route}`); } catch { return false; } });
|
||||
if (!href) throw new Error("The reply did not link the saved skill, so the person cannot open it from the chat");
|
||||
await internalResultLink(ctx, o, href).first().click({ timeout: remaining() });
|
||||
if (href) await internalResultLink(ctx, o, href).first().click({ timeout: remaining() });
|
||||
else {
|
||||
await ctx.input.page.getByRole("article", { name: `Skill created: ${skill.name}`, exact: true }).getByRole("button").first().click({ timeout: remaining() });
|
||||
await ctx.input.page.getByRole("button", { name: "Open in Skill Studio", exact: true }).click({ timeout: remaining() });
|
||||
}
|
||||
await expect(ctx.input.page).toHaveURL(url => url.pathname === studioRoute || url.pathname === detailRoute, { timeout: remaining() });
|
||||
// From the skill, the Skills section's own My Skills link shows it in the person's library.
|
||||
await ctx.input.page.getByRole("link", { name: "My Skills", exact: true }).first().click({ timeout: remaining() });
|
||||
|
||||
Reference in new issue
Block a user