fix(evals): require authored skill company prefix

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-04 16:59:19 -06:00
1 parent f0717eda1c
commit 06e7ac0bd7
2 files changed
+9 -1

No files matched your search

@@ -83,3 +83,9 @@ it("inspects real rich-text blocks without applying Markdown marker assumptions
});
it("retains exact direct Studio card navigation when the UI uses it", async()=>{ const f=fixture();f.ui.directStudio=true;await inspectAuthoredSkill(f.ctx,f.state,f.o);expect(f.events).toEqual(["Skills","My Skills","open-skill","reload","return-chat"]);expect(f.state.authoredSkillInspection).toMatchObject({readbackMatched:true,editorSurface:"studio"}); });
it.each([undefined, null, ""])("rejects missing company prefix %s before any UI navigation", async prefix => {
const f=fixture();Object.assign(f.ctx.input.fixtures.company,{issuePrefix:prefix});
await expect(inspectAuthoredSkill(f.ctx,f.state,f.o)).rejects.toThrow("company prefix");
expect(f.events).toEqual([]);expect(f.state.authoredSkillInspection).toBeUndefined();
});
@@ -102,6 +102,8 @@ interface AuthoredSkillInspection { companyId: string; skillId: string; versionI
export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Record<string, unknown>, o: InternalObservation): Promise<void> {
const remaining = () => remainingIdentityObservationBudget(ctx);
remaining();
const prefix = ctx.input.fixtures.company.issuePrefix;
if (typeof prefix !== "string" || !prefix.trim()) throw new Error("Authored skill inspection requires its actual company prefix");
const observed = skillsObservation(o);
const authored = observed?.skills.filter(skill => observed.activity.some(event => event.action === "company.skill_created" && event.entityId === skill.id && event.agentId === observed.leadId)) ?? [];
const skill = authored[0], version = skill && observed?.versions[skill.id]?.find(row => row.id === skill.currentVersionId);
@@ -112,7 +114,7 @@ export async function inspectAuthoredSkill(ctx: ChatCatalogContext, state: Recor
await ctx.input.page.getByRole("link", { name: "My Skills", exact: true }).first().click({ timeout: remaining() });
const card = ctx.input.page.getByRole("button").filter({ has: ctx.input.page.getByText(skill.name, { exact: true }) }).first();
await expect(card).toBeVisible({ timeout: remaining() }); await card.click({ timeout: remaining() });
const companyPath = `/${encodeURIComponent(ctx.input.fixtures.company.issuePrefix)}`;
const companyPath = `/${encodeURIComponent(prefix)}`;
const studioRoute = `${companyPath}${skillStudioRoute(skill.id)}`;
const detailRoute = `${companyPath}${skillRoute(skill, observed.skills)}`;
const openSavedEditor = async () => {