test(evals): inspect authored skills through normal Studio navigation

For the reusable-skill request, prove ordinary library discovery, editable Studio checklist content and reload with the same saved version instead of prescribing a chat-link format. Keep current-version/content/attribution/assignment checks and all other skill contracts unchanged; record the explicit authoring v2 conditions.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/integrator-skill-authoring-v2.test.ts

@@ -0,0 +1,71 @@
+import { afterEach, expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+import type { InternalObservation } from "./internal-support.js";
+vi.mock("@playwright/test",()=>({expect:Object.assign((node:any)=>({toBeVisible:async()=>{if(!node.visible)throw Error("Missing real UI item");},toBeEditable:async()=>{if(!node.editable)throw Error("Noneditable real UI");},toHaveURL:async(predicate:(url:URL)=>boolean)=>{if(!predicate(new URL(node.url())))throw Error("Wrong actual Studio identity");}}),{poll:(read:()=>Promise<boolean>)=>({toBe:async(expected:boolean)=>{for(let i=0;i<3;i++)if(await read()===expected)return;throw Error("UI content did not hydrate to the saved checklist");}})})}));
+const observed=vi.hoisted(()=>({base:undefined as unknown}));
+vi.mock("./integrator-support.js",async original=>({...await original<typeof import("./integrator-support.js")>(),observeInternal:vi.fn(async(_ctx:unknown,state:unknown)=>({...observed.base as object,state}))}));
+import { gradeIntegratorSkills, inspectAuthoredSkill, integratorSkillsCases, websiteChecklist } from "./integrator-skills.js";
+afterEach(()=>vi.restoreAllMocks());
+function fixture(){
+ const markdown=websiteChecklist.join("\n"),source="https://fixture.test/ACORN/chats/ceo";let current=source,reloads=0,editorReads=0;
+ const version={id:"revision",companyId:"company",companySkillId:"skill",revisionNumber:1,authorAgentId:"ceo",authorUserId:null,fileInventory:[{path:"SKILL.md",kind:"skill",content:markdown}]};
+ const skill={id:"skill",companyId:"company",name:"Website review checklist",editable:true,currentVersionId:"revision",markdown,usedByAgents:[]};
+ const state:Record<string,unknown>={},o={scenarioId:"skills.author-skill",origin:"https://fixture.test",companyId:"company",leadId:"ceo",state,skills:[skill],versions:{skill:[version]},policy:{},editDecision:{allowed:true},tasks:[],projects:[],goals:[],comments:[],documents:[],revisions:[],humanQueries:["Turn our website review checklist into a reusable skill."],runs:[{id:"native",status:"succeeded",agentId:"ceo",companyId:"company"}],activity:[{action:"company.skill_created",entityId:"skill",agentId:"ceo"}],links:[],reply:"Created the reusable company skill, saved in the library and not assigned to any agent."} as unknown as InternalObservation;
+ observed.base=o;
+ const ui={library:true,studio:true,editable:true,body:markdown,changedAfterReload:false,wrongRoute:false,delayedHydration:false};const events:string[]=[];
+ const editor={get visible(){return ui.studio;},get editable(){return ui.editable;},innerText:async()=>ui.delayedHydration&&editorReads++===0?"":ui.changedAfterReload&&reloads?"Lost checklist":ui.body};
+ const page={url:()=>current,getByRole:(role:string,options?:{name?:string})=>{
+  if(role==="textbox")return editor;
+  if(role==="link")return{first:()=>({click:async()=>{events.push(options!.name!);current=options?.name==="Skills"?"https://fixture.test/ACORN/skills":"https://fixture.test/ACORN/skills/studio";}})};
+  return{filter:()=>({first:()=>({get visible(){return ui.library;},click:async()=>{events.push("open-skill");current=ui.wrongRoute?"https://fixture.test/ACORN/skills/studio/other":"https://fixture.test/ACORN/skills/studio/skill";}})})};
+ },getByText:()=>({first:()=>({get visible(){return ui.studio;}})}),getByTestId:()=>({get visible(){return current===source;}}),reload:async()=>{reloads++;events.push("reload");},goto:async(url:string)=>{current=url;events.push("return-chat");}};
+ const after=structuredClone(skill),get=vi.fn(async(route:string)=>{
+  if(route.endsWith("/skills"))return[{id:"skill"}];
+  if(route.endsWith("/skills/skill"))return after;
+  if(route.endsWith("/versions"))return[{...version,fileInventory:[{path:"SKILL.md",kind:"skill"}]}];
+  if(route.endsWith("/versions/revision"))return version;
+  if(route.endsWith("/skill-policy"))return{};
+  throw Error("Unexpected public inspection read");
+ });const post=vi.fn(async()=>({allowed:true})),evidence=vi.fn();
+ const ctx={scenario:{id:"skills.author-skill"},observationDeadlineAt:Date.now()+270000,input:{fixtures:{company:{id:"company",issuePrefix:"ACORN"},agent:{id:"ceo"}},api:{baseURL:o.origin,get,post},page,evidence},refreshIssue:async()=>{},issue:()=>({conversationState:"waiting"}),allRuns:async()=>o.runs} as unknown as ChatCatalogContext;
+ return{ctx,state,o,skill,version,ui,events,after,get,post,evidence};
+}
+it("the actual authoring act performs readonly normal library/Studio inspection and keeps a single native/human turn",async()=>{
+ const f=fixture(),d=integratorSkillsCases.find(x=>x.scenarioId==="skills.author-skill")!;
+ await d.act!(f.ctx,f.state);
+ expect(f.events).toEqual(["Skills","My Skills","open-skill","reload","return-chat"]);
+ expect(f.state.authoredSkillInspection).toMatchObject({skillId:"skill",versionId:"revision",readbackMatched:true});
+ expect(gradeIntegratorSkills({...f.o,state:f.state}).passed).toBe(true);
+ expect(d.expectedRunCount).toBe(1);expect(d.expectedHumanMessageCount).toBe(1);
+ expect(f.post).toHaveBeenCalledExactlyOnceWith("/api/companies/company/skill-policy/evaluate",expect.anything());
+});
+it.each(["missing library","missing Studio","readonly","wrong UI route","wrong UI content","lost after reload","wrong saved version","wrong author","foreign company","no saved version"])("rejects %s before recording inspection success",async invalid=>{
+ const f=fixture();
+ if(invalid==="missing library")f.ui.library=false;
+ if(invalid==="missing Studio")f.ui.studio=false;
+ if(invalid==="readonly")f.ui.editable=false;
+ if(invalid==="wrong UI route")f.ui.wrongRoute=true;
+ if(invalid==="wrong UI content")f.ui.body="Review the site.";
+ if(invalid==="lost after reload")f.ui.changedAfterReload=true;
+ if(invalid==="wrong saved version")f.after.currentVersionId="other";
+ if(invalid==="wrong author")f.version.authorAgentId="other";
+ if(invalid==="foreign company")f.skill.companyId="foreign";
+ if(invalid==="no saved version")f.skill.currentVersionId="missing";
+ await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow();expect(f.state.authoredSkillInspection).toBeUndefined();
+});
+it("rejects invented inspection receipts and keeps other skill cases on their original contract",()=>{
+ const f=fixture();expect(gradeIntegratorSkills(f.o).passed).toBe(false);
+ f.state.authoredSkillInspection={companyId:"company",skillId:"other",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true};
+ expect(gradeIntegratorSkills(f.o).passed).toBe(false);
+ expect(integratorSkillsCases.find(x=>x.scenarioId==="skills.explicit-restriction")!.oracleVersion).toBe("paperclip.chat-catalog.integrator.v1");
+});
+it("cannot extend an exhausted original case budget to inspect a finished artifact",async()=>{
+ const f=fixture();f.ctx.observationDeadlineAt=Date.now()-1;
+ await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("budget");expect(f.events).toEqual([]);
+});
+
+it("waits for actual saved editor content instead of rejecting its visible empty hydration shell",async()=>{
+ const f=fixture();f.ui.delayedHydration=true;
+ await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).resolves.toBeUndefined();
+ expect(f.state.authoredSkillInspection).toMatchObject({checklistVisible:true,reloaded:true});
+});

tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts

@@ -3,10 +3,10 @@ import { gradeIntegratorSkills, integratorSkillsCases, websiteChecklist, preserv
 
 function valid(id: string) {
   const markdown = websiteChecklist.join("\n");
-  const version = { id: "revision", companySkillId: "skill", authorAgentId: "ceo", fileInventory: [{ path: "SKILL.md", content: markdown }] };
+  const version = { id: "revision", companyId: "company", companySkillId: "skill", authorAgentId: "ceo", fileInventory: [{ path: "SKILL.md", content: markdown }] };
   const skill = { id: "skill", companyId: "company", name: "Accessibility review", editable: true, markdown, currentVersionId: "revision", usedByAgents: [] };
   const denied = id === "skills.explicit-restriction";
-  return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: denied ? { skill: structuredClone(skill), versions: [structuredClone(version)], policyRevision: 3 } : {},
+  return { scenarioId: id, origin: "https://fixture.test", companyId: "company", leadId: "ceo", state: denied ? { skill: structuredClone(skill), versions: [structuredClone(version)], policyRevision: 3 } : { authoredSkillInspection: {companyId:"company",skillId:"skill",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true} },
     tasks: [], projects: [], goals: [], comments: [], documents: [], revisions: [], humanQueries: [], runs: [{ id: "run", agentId: "ceo", status: "succeeded" }],
     activity: denied ? [] : [{ id: "create", entityId: "skill", action: "company.skill_created", agentId: "ceo" }],
     reply: denied ? "Company policy denies my skill edits. Ask an administrator to request a change." : "Created the reusable review skill; it is available in Studio. This does not enable it for all agents.",
@@ -26,7 +26,7 @@ describe("authored skill outcomes", () => {
     expect(gradeIntegratorSkills({ ...o, activity: [] }).passed).toBe(false);
     const missing = structuredClone(o); missing.skills[0]!.markdown = "Review the website."; missing.versions.skill[0]!.fileInventory[0]!.content = "Review the website.";
     expect(gradeIntegratorSkills(missing).passed).toBe(false);
-    expect(gradeIntegratorSkills({ ...o, links: ["https://other.test/ACORN/skills/studio/skill"] }).passed).toBe(false);
+    const notInspected = structuredClone(o); delete (notInspected.state as {authoredSkillInspection?:unknown}).authoredSkillInspection; expect(gradeIntegratorSkills(notInspected).passed).toBe(false);
     expect(gradeIntegratorSkills({ ...o, reply: "Created and enabled the skill for every agent." }).passed).toBe(false);
   });
   it("rejects a policy denial explained as success, a fictional source limitation, or a stealth fork", () => {
@@ -62,3 +62,9 @@ it("preserves a numbered viewport check and its actual multiline inspection deta
   expect(preservesWebsiteChecklist(markdown.replace("375 pixel", "768 pixel"))).toBe(false);
   expect(preservesWebsiteChecklist(markdown.replace("Confirm whether", "Ignore whether"))).toBe(false);
 });
+
+it("authoring v2 accepts ordinary Skills/Studio inspection without a prescribed reply link",()=>{
+ const o=valid("skills.author-skill");o.links=[];
+ Object.assign(o.state,{authoredSkillInspection:{companyId:"company",skillId:"skill",versionId:"revision",libraryVisible:true,studioVisible:true,editable:true,checklistVisible:true,reloaded:true,readbackMatched:true}});
+ expect(gradeIntegratorSkills(o).passed).toBe(true);
+});

tests/runner-e2e/chat-catalog-domains/integrator-skills.ts

@@ -1,5 +1,7 @@
 import { readSkillVersionHistory } from "./integrator-skill-versions.js";
 import { saveSkillConversationContext } from "./integrator-skill-context.js";
+import { remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
+import { expect } from "@playwright/test";
 import { factualContextQueryAttachment } from "../chat-query-attachments.js";
 import type { CompanySkill, CompanySkillDetail, CompanySkillListItem, CompanySkillVersion } from "../../../packages/shared/src/types/company-skill.js";
 import type { EffectiveSkillPolicy, SkillPolicyDecision } from "../../../packages/shared/src/validators/skill-policy.js";
@@ -51,7 +53,9 @@ export function gradeIntegratorSkills(value: unknown) {
     const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content;
     c.add("actual-checklist-in-current-revision", !!skill && !!version && version.authorAgentId === o.leadId && typeof markdown === "string"
       && preservesWebsiteChecklist(markdown) && markdown === skill.markdown, "The current saved SKILL.md revision preserves keyboard-only controls, visible form labels, WCAG AA text contrast and a 375px viewport without horizontal overflow; paraphrase is permitted.");
-    c.add("inspectable-skill-link", !!skill && !!resultHref(o, skill.id, "studio"), "Chat links the saved editable skill in Studio.");
+    const inspection = o.state.authoredSkillInspection as AuthoredSkillInspection | undefined;
+    c.add("inspectable-authored-skill", !!skill && inspection?.companyId === o.companyId && inspection.skillId === skill.id && inspection.versionId === skill.currentVersionId && inspection.libraryVisible === true && inspection.studioVisible === true && inspection.editable === true && inspection.checklistVisible === true && inspection.reloaded === true && inspection.readbackMatched === true,
+      "The actual saved skill is discoverable through ordinary Skills/My Skills, opens editable Studio content, and survives reload with the same saved version. No specific reply-link format is required.");
     c.add("creation-not-global-assignment", !!skill && skill.usedByAgents.length === 0 && !/\b(?:enabled|assigned|activated)\s+(?:it|the skill|this skill)?\s*(?:for\s+)?(?:all|every)\b/i.test(o.reply), "Creating a reusable skill does not silently enable it for all agents or claim that happened.");
   } else if (o.scenarioId === "skills.explicit-restriction") {
     const before = o.state.skill as CompanySkillDetail; const current = o.skills.find(row => row.id === before?.id);
@@ -73,10 +77,47 @@ async function verifySkill(ctx: ChatCatalogContext, state: Record<string, unknow
   const href = resultHref(o, skill.id, "studio");
   await inspectResult(ctx, `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`, skill.name, href);
 }
+interface AuthoredSkillInspection { companyId: string; skillId: string; versionId: string; libraryVisible: true; studioVisible: true; editable: true; checklistVisible: true; reloaded: true; readbackMatched: true }
+export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Record<string, unknown>, o: InternalObservation): Promise<void> {
+  const remaining = () => remainingIdentityObservationBudget(ctx);
+  remaining();
+  const observed = skillsObservation(o);
+  const authored = observed?.skills.filter(skill => observed.activity.some(event => event.action === "company.skill_created" && event.entityId === skill.id && event.agentId === observed.leadId)) ?? [];
+  const skill = authored[0], version = skill && observed?.versions[skill.id]?.find(row => row.id === skill.currentVersionId);
+  const markdown = version?.fileInventory.find(file => file.path === "SKILL.md")?.content;
+  if (!observed || observed.companyId !== ctx.input.fixtures.company.id || observed.leadId !== ctx.input.fixtures.agent.id || authored.length !== 1 || !skill || skill.companyId !== observed.companyId || !skill.editable || !skill.currentVersionId || version?.authorAgentId !== observed.leadId || version.companyId !== observed.companyId || version.companySkillId !== skill.id || typeof markdown !== "string" || markdown !== skill.markdown || !preservesWebsiteChecklist(markdown)) throw new Error("No exact authored saved checklist version is available to inspect");
+  const source = ctx.input.page.url();
+  await ctx.input.page.getByRole("link", { name: "Skills", exact: true }).first().click({ timeout: remaining() });
+  await ctx.input.page.getByRole("link", { name: "My Skills", exact: true }).first().click({ timeout: remaining() });
+  const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByText(skill.name, { exact: true }) }).first();
+  await expect(card).toBeVisible({ timeout: remaining() }); await card.click({ timeout: remaining() });
+  const route = `/${ctx.input.fixtures.company.issuePrefix}/skills/studio/${skill.id}`;
+  await expect(ctx.input.page).toHaveURL(url => url.pathname === route, { timeout: remaining() });
+  await expect(ctx.input.page.getByText(skill.name, { exact: true }).first()).toBeVisible({ timeout: remaining() });
+  const editor = ctx.input.page.getByRole("textbox", { name: "editable markdown", exact: true });
+  await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() });
+  await expect.poll(async () => preservesWebsiteChecklist(await editor.innerText()), { timeout: remaining() }).toBe(true);
+  const body = await editor.innerText();
+  await ctx.input.page.reload({ waitUntil: "domcontentloaded", timeout: remaining() });
+  await expect(ctx.input.page).toHaveURL(url => url.pathname === route, { timeout: remaining() });
+  await expect(editor).toBeVisible({ timeout: remaining() }); await expect(editor).toBeEditable({ timeout: remaining() });
+  await expect.poll(async () => await editor.innerText() === body, { timeout: remaining() }).toBe(true);
+  const after = await ctx.input.api.get<CompanySkillDetail>(`${root(ctx)}/skills/${encodeURIComponent(skill.id)}`, { timeout: remaining() });
+  if (after.id !== skill.id || after.companyId !== observed.companyId || !after.editable || after.currentVersionId !== skill.currentVersionId || after.markdown !== markdown) throw new Error("Authored skill identity, saved version or bytes changed during inspection");
+  remaining();
+  state.authoredSkillInspection = { companyId: observed.companyId, skillId: skill.id, versionId: skill.currentVersionId, libraryVisible: true, studioVisible: true, editable: true, checklistVisible: true, reloaded: true, readbackMatched: true } satisfies AuthoredSkillInspection;
+  await ctx.input.evidence("chat-catalog-authored-skill-ui.json", state.authoredSkillInspection);
+  await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() });
+  await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible({ timeout: remaining() });
+  remaining();
+}
 const define = (id: string, setup: ChatCatalogCase["setup"]): ChatCatalogCase => {
   const base = integratorDefinition("skills", id, setup, observeSkills, gradeIntegratorSkills, verifySkill);
   base.sourceFiles = [...base.sourceFiles, "chat-catalog-domains/integrator-skill-versions.ts"];
-  return id === "skills.author-skill" ? { ...base, authentication: "human-session", queryAttachments: factualContextQueryAttachment("skill-context.md") } : base;
+  return id === "skills.author-skill" ? { ...base, authentication: "human-session", oracleVersion: "paperclip.chat-catalog.skill-authoring.v2", sourceFiles: [...base.sourceFiles, "chat-catalog-domains/identity-support.ts"], queryAttachments: factualContextQueryAttachment("skill-context.md"),
+    async act(ctx, state) { remainingIdentityObservationBudget(ctx); await waitForReply(ctx, ctx.observationDeadlineAt); await inspectAuthoredSkill(ctx, state, await observeSkills(ctx, state)); },
+    async verifyResult(ctx, state, value) { const o = observation(value); if (!o) throw new Error("Missing authored skill inspection evidence"); await inspectAuthoredSkill(ctx, state, o); },
+  } : base;
 };
 export const integratorSkillsCases: readonly ChatCatalogCase[] = [
   define("skills.author-skill", async ctx => {

tests/runner-e2e/chat-scenarios/skills.yaml

@@ -26,8 +26,9 @@ scenarios:
       Turn our website review checklist into a reusable skill.
     expected_outcome: >-
       A writable company skill captures the checklist with clear instructions and a saved
-      revision. Chat links to the skill so the person can inspect or edit it. The agent
-      preserves the actual checklist instead of merely posting a suggestion or pretending
+      revision. The person can discover it through ordinary Skills/My Skills navigation, open
+      its editable Studio content and reload the same saved revision. No particular chat-link
+      format is required. The agent preserves the actual checklist instead of merely posting a suggestion or pretending
       creation automatically enables the skill for every agent.
   - id: skills.fork-read-only
     capabilities: