Jannik/PaperClipAI
1
0 Fork 0
mirror of https://github.com/paperclipai/paperclip.git synced 2026-11-10 12:37:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

test(evals): open an authored skill from the reply link, then find it in My Skills

Browse Source 
Skills is no longer a primary sidebar link. The person follows the skill link the reply rendered, then uses the Skills section's My Skills link to confirm it is in the library. A reply without a link to the saved skill now fails the case. Bump skill-authoring oracle to v7.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
DottaandClaude Opus 5.5 committed 2026-11-08 23:28:52 -06:00
1 parent 9415389f0f
commit 8680b56b89
3 files changed
+25 -13
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/runner-e2e/chat-catalog-descriptors.json
+1 -1
View File
@@ -5120,7 +5120,7 @@
"../../ui/src/pages/CompanySkills.tsx",
"../../ui/src/lib/company-skill-routes.ts"
],
"oracleVersion": "paperclip.chat-catalog.skill-authoring.v6",
"oracleVersion": "paperclip.chat-catalog.skill-authoring.v7",
"expectedRunCount": 1,
"expectedHumanMessageCount": 1,
"turnTimeoutMs": 120000,
tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts
+10 -5
View File
@@ -11,7 +11,7 @@ function fixture(){
const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0;
const version={id:"revision",companyId:"company",companySkillId:"skill",revisionNumber:1,authorAgentId:"ceo",authorUserId:null,fileInventory:[{path:"SKILL.md",kind:"skill",content:markdown}]};
const skill={id:"skill",companyId:"company",name:"Website review checklist",slug:"website-review-checklist",editable:true,currentVersionId:"revision",markdown,usedByAgents:[]};
const state:Record<string,unknown>={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:[],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation;
const state:Record<string,unknown>={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:["/skills/studio/skill"],renderedLinkEvidence:[{threadTestId:"task-chat-thread",href:"/skills/studio/skill",source:"reply",conversationId:"conversation",anchorCommentId:"query",commentId:"reply"}],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation;
observed.base=o;
const ui={library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,foreignRoute:false,directStudio:false,delayedHydration:false,inEdit:false,filesSelected:false};const events:string[]=[];
const editor={get visible(){return ui.studio && (ui.inEdit || ui.directStudio);},get editable(){return ui.editable;},locator:()=>({evaluateAll:async()=>ui.delayedHydration&&editorReads++===0?[]:(ui.changedAfterReload&&reloads?"Lost checklist":ui.body).split("\n")})};
@@ -21,7 +21,7 @@ function fixture(){
if(role==="button" && options?.name==="Edit")return{click:async()=>{if(!ui.editable||!ui.filesSelected)throw Error("No Edit control outside Files");ui.inEdit=true;events.push("Edit");}};
if(role==="link")return{first:()=>({click:async()=>{events.push(options!.name!);current=options?.name==="Skills"?"https://fixture.test/ACORN/skills":"https://fixture.test/ACORN/skills/studio";}})};
return{filter:()=>({first:()=>({get visible(){return ui.library;},click:async()=>{events.push("open-skill");current=ui.foreignRoute?"https://fixture.test/FOREIGN/skills/website-review-checklist":ui.wrongRoute?"https://fixture.test/ACORN/skills/other":ui.directStudio?"https://fixture.test/ACORN/skills/studio/skill":"https://fixture.test/ACORN/skills/website-review-checklist";}})})};
},getByText:()=>({first:()=>({get visible(){return ui.studio;}})}),getByTestId:()=>({get visible(){return current===source;}}),reload:async()=>{reloads++;ui.inEdit=false;ui.filesSelected=false;events.push("reload");},goto:async(url:string)=>{current=url;events.push("return-chat");}};
},getByText:()=>({first:()=>({get visible(){return ui.studio;}})}),locator:()=>({}),getByTestId:()=>({get visible(){return current===source;},filter:()=>({locator:(selector:string)=>({first:()=>({click:async()=>{if(!selector.includes('"/skills/studio/skill"'))throw Error("Not the rendered skill link");events.push("reply-link");current="https://fixture.test/ACORN/skills/studio/skill";}})})})}),reload:async()=>{reloads++;ui.inEdit=false;ui.filesSelected=false;events.push("reload");},goto:async(url:string)=>{current=url;events.push("return-chat");}};
const after=structuredClone(skill),get=vi.fn(async(route:string)=>{
if(route.endsWith("/skills"))return[{id:"skill"}];
if(route.endsWith("/skills/skill"))return after;
@@ -36,7 +36,7 @@ function fixture(){
it("the actual authoring act performs nonmutating canonical-library/Edit inspection and keeps a single native/human turn",async()=>{
const f=fixture(),d=integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!;
await d.act!(f.ctx,f.state);
expect(f.events).toEqual(["Skills","My Skills","open-skill","Files","Edit","reload","Files","Edit","return-chat"]);
expect(f.events).toEqual(["reply-link","My Skills","open-skill","Files","Edit","reload","Files","Edit","return-chat"]);
expect(f.state.authoredSkillInspection).toMatchObject({skillId:"skill",versionId:"revision",editorSurface:"detail",editorVisible:true,readbackMatched:true});
expect(gradeIntegratorSkills({...f.o,state:f.state}).passed).toBe(true);
expect(d.expectedRunCount).toBe(1);expect(d.expectedHumanMessageCount).toBe(1);
@@ -62,7 +62,12 @@ it("rejects invented inspection receipts and keeps other skill cases on their or
f.state.authoredSkillInspection={companyId:"company",skillId:"other",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true};
expect(gradeIntegratorSkills(f.o).passed).toBe(false);
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1");
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v6");
expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!.oracleVersion).toBe("paperclip.chat-catalog.skill-authoring.v7");
});
it.each(["no link","link to another skill"])("rejects a reply with %s before any UI navigation",async fault=>{
const f=fixture(),href=fault==="no link"?null:"/skills/studio/other";
Object.assign(f.o,{links:href?[href]:[],renderedLinkEvidence:href?[{threadTestId:"task-chat-thread",href,source:"reply",conversationId:"conversation",anchorCommentId:"query",commentId:"reply"}]:[]});
await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("did not link the saved skill");expect(f.events).toEqual([]);expect(f.state.authoredSkillInspection).toBeUndefined();
});
it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{
const f=fixture();Object.assign(f.ctx,{observationDeadlineAt:Date.now()-1});
@@ -84,7 +89,7 @@ it("inspects real rich-text blocks without applying Markdown marker assumptions
expect(preservesVisibleWebsiteChecklist([...websiteChecklist.slice(0,3),"375 pixel viewport"])).toBe(false);
});
it("retains exact direct Studio card navigation when the UI uses it", async()=>{ const f=fixture();f.ui.directStudio=true;await inspectAuthoredSkill(f.ctx,f.state,f.o);expect(f.events).toEqual(["Skills","My Skills","open-skill","reload","return-chat"]);expect(f.state.authoredSkillInspection).toMatchObject({readbackMatched:true,editorSurface:"studio"}); });
it("retains exact direct Studio card navigation when the UI uses it", async()=>{ const f=fixture();f.ui.directStudio=true;await inspectAuthoredSkill(f.ctx,f.state,f.o);expect(f.events).toEqual(["reply-link","My Skills","open-skill","reload","return-chat"]);expect(f.state.authoredSkillInspection).toMatchObject({readbackMatched:true,editorSurface:"studio"}); });
it.each([undefined, null, ""])("rejects missing company prefix %s before any UI navigation", async prefix => {
const f=fixture();Object.assign(f.ctx.input.fixtures.company,{issuePrefix:prefix});
tests/runner-e2e/chat-catalog-domains/integrator-skills.ts
+14 -7
View File
@@ -7,7 +7,7 @@ import { factualContextQueryAttachment } from "../chat-query-attachments.js";
import type { CompanySkill, CompanySkillDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js";
import type { EffectiveSkillPolicy, SkillPolicyDecision } from "../../../packages/shared/src/validators/skill-policy.js";
import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js";
import { checks, observation, project, type InternalObservation } from "./internal-support.js";
import { checks, internalResultLink, observation, project, type InternalObservation } from "./internal-support.js";
import { integratorDefinition, groundedStatement, inspectResult, resultHref, observeInternal } from "./integrator-support.js";
export const websiteChecklist = ["Navigate every interactive control using only the keyboard.", "Verify every form field has a visible label.", "Check text contrast meets WCAG AA.", "Test the landing page at a 375 pixel viewport without horizontal scrolling."];
@@ -129,7 +129,7 @@ export function gradeIntegratorSkills(value: unknown) {
&& preservesWebsiteChecklist(markdown) && markdown === skill.markdown, "The current saved SKILL.md revision preserves keyboard-only controls, visible form labels, WCAG AA text contrast and a 375px viewport without horizontal overflow; paraphrase is permitted.");
const inspection = o.state.authoredSkillInspection as AuthoredSkillInspection | undefined;
c.add("inspectable-authored-skill", !!skill && inspection?.companyId === o.companyId && inspection.skillId === skill.id && inspection.versionId === skill.currentVersionId && inspection.libraryVisible === true && inspection.editorVisible === true && (inspection.editorSurface === "detail" || inspection.editorSurface === "studio") && inspection.editable === true && inspection.checklistVisible === true && inspection.reloaded === true && inspection.readbackMatched === true,
"The actual saved skill is discoverable through ordinary Skills/My Skills, opens the actual editable skill content, and survives reload with the same saved version. No specific reply-link format is required.");
"The reply links the actual saved skill; following that link opens it, it is listed in My Skills, opens the actual editable skill content, and survives reload with the same saved version.");
c.add("creation-not-global-assignment", authorSkillAssignmentScoped(o, skill, version) && !claimsGlobalSkillAssignment(o.reply), "Creation leaves the skill unassigned or explicitly binds only its acting author in the same real run, with current saved-version and assignment audit evidence and truthful targeted reporting; no global policy change, other-agent assignment or false global activation claim is admitted.");
} else if (o.scenarioId === "skills.explicit-restriction") {
const before = o.state.skill as CompanySkillDetail; const current = o.skills.find(row => row.id === before?.id);
@@ -163,13 +163,20 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor
const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content;
if (!observed || observed.companyId !== ctx.input.fixtures.company.id || observed.leadId !== ctx.input.fixtures.agent.id || authored.length !== 1 || !skill || skill.companyId !== observed.companyId || !skill.editable || !skill.currentVersionId || version?.authorAgentId !== observed.leadId || version.companyId !== observed.companyId || version.companySkillId !== skill.id || typeof markdown !== "string" || markdown !== skill.markdown || !preservesWebsiteChecklist(markdown)) throw new Error("No exact authored saved checklist version is available to inspect");
const source = ctx.input.page.url();
await ctx.input.page.getByRole("link", { name: "Skills", exact: true }).first().click({ timeout: remaining() });
const companyPath = `/${encodeURIComponent(prefix)}`;
const skillPaths = [skillStudioRoute(skill.id), skillRoute(skill, observed.skills)];
const studioRoute = `${companyPath}${skillPaths[0]}`;
const detailRoute = `${companyPath}${skillPaths[1]}`;
// Skills is no longer a primary sidebar link, so the person opens the new skill from the
// link the reply rendered (unprefixed links redirect into the current company).
const href = o.links.find(link => { try { const path = new URL(link, source).pathname; return skillPaths.some(route => path === route || path === `${companyPath}${route}`); } catch { return false; } });
if (!href) throw new Error("The reply did not link the saved skill, so the person cannot open it from the chat");
await internalResultLink(ctx, o, href).first().click({ timeout: remaining() });
await expect(ctx.input.page).toHaveURL(url => url.pathname === studioRoute || url.pathname === detailRoute, { timeout: remaining() });
// From the skill, the Skills section's own My Skills link shows it in the person's library.
await ctx.input.page.getByRole("link", { name: "My Skills", exact: true }).first().click({ timeout: remaining() });
const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByText(skill.name, { exact: true }) }).first();
await expect(card).toBeVisible({ timeout: remaining() }); await card.click({ timeout: remaining() });
const companyPath = `/${encodeURIComponent(prefix)}`;
const studioRoute = `${companyPath}${skillStudioRoute(skill.id)}`;
const detailRoute = `${companyPath}${skillRoute(skill, observed.skills)}`;
const openSavedEditor = async () => {
await expect(ctx.input.page).toHaveURL(url => url.origin === new URL(source).origin
&& (url.pathname === studioRoute || url.pathname === detailRoute), { timeout: remaining() });
@@ -201,7 +208,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor
const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => {
const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill);
base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"];
return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v6", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"),
return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v7", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts", "../../ui/src/components/MarkdownEditor.tsx", "../../ui/src/pages/SkillStudio.tsx", "../../ui/src/pages/CompanySkills.tsx", "../../ui/src/lib/company-skill-routes.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"),
async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); },
async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); },
} : base;