From 04da1afa3402ab68342ff080cb9207bcfc9537a5 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 5 Nov 2026 16:43:02 -0600 Subject: [PATCH] test(evals): bind global assignment negation to its verb Co-Authored-By: Paperclip --- .../chat-catalog-domains/integrator-skills.test.ts | 13 +++++++++++++ .../chat-catalog-domains/integrator-skills.ts | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts index 295b93dc7d5..fb1328d73c9 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.test.ts @@ -159,3 +159,16 @@ it("rejects global policy changes even without any final agent assignment", () = const activity = [...o.activity, { id: "policy", action: "company.skill_policy_reset", entityId: "company", agentId: "ceo" }]; expect(gradeIntegratorSkills({ ...o, activity }).passed).toBe(false); }); + + +it("rejects a correlative not-only claim of global enablement", () => { + const o = valid("skills.author-skill"); + o.reply = "Not only created it but enabled it for all agents."; + expect(gradeIntegratorSkills(o).passed).toBe(false); +}); +it("preserves a negation that directly governs the assignment verb", () => { + for (const denial of ["I have not enabled it for all agents.", "Did not enable it for all agents."]) { + const o = authoredSelfUse(); o.reply += " " + denial; + expect(gradeIntegratorSkills(o).passed).toBe(true); + } +}); diff --git a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts index 1cefc298c09..73fb70f26a2 100644 --- a/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts +++ b/tests/runner-e2e/chat-catalog-domains/integrator-skills.ts @@ -104,7 +104,7 @@ function authorSkillAssignmentScoped(o: SkillsObservation, skill: CompanySkillDe } function claimsGlobalSkillAssignment(reply: string): boolean { const claims = /\b(?:enabled|assigned|activated|attached)\s+(?:it|the skill|this skill)?\s*(?:(?:for|to)\s+)?(?:all|every|everyone)\b/gi; - return [...reply.matchAll(claims)].some(match => !/\b(?:not|never|didn't|haven't|hasn't)\b[^,;.!?]{0,35}$/i.test(reply.slice(0, match.index))); + return [...reply.matchAll(claims)].some(match => !/\b(?:not|never|didn't|haven't|hasn't)\s+(?:(?:been|ever|yet|automatically)\s+){0,2}$/i.test(reply.slice(0, match.index))); } export function gradeIntegratorSkills(value: unknown) { const c = checks(); const o = skillsObservation(value);