From ba8418fc4df87e359ff9ac22a0d18ce822fc1034 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:38:23 -0600 Subject: [PATCH] fix(evals): bind adjacent team skill recovery pronouns Co-Authored-By: Paperclip --- .../external-team-conditional.test.ts | 16 ++++++++++++++++ .../external-team-conditional.ts | 11 ++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index 7e6dc1e5464..e7c0c5bace3 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -222,3 +222,19 @@ it("does not treat a historical negative alone as an asserted initial recovery", o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was previously not installed."; expect(gradeTeamConditions(o, "initial").passed).toBe(false); }); + +it.each(["It", "This skill", "That skill"])("accepts immediately bound %s recovery without repeating the skill name", subject => { + const { o } = recovered(); + o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist hit a slug conflict. ${subject} is now installed under a separate slug.`; + expect(gradeTeamConditions(o, "initial").passed).toBe(true); +}); +it("rejects an immediately bound pronoun claiming the recovered skill is still missing", () => { + const { o } = recovered(); + o.phaseReplies[1]!.body = "Launch Checklist is installed. It is still missing and not installed."; + expect(gradeTeamConditions(o).passed).toBe(false); +}); +it.each(["Launch Checklist hit a conflict. Another skill was discussed. It is now installed.", "It is now installed. Launch Checklist hit a conflict.", "Launch Checklist hit a conflict.\n\nIt is now installed."])("does not credit interrupted or unbound pronoun recovery: %s", body => { + const { o } = recovered(); + o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. ${body}`; + expect(gradeTeamConditions(o, "initial").passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index cc21fa10622..496d68ed2c6 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -176,7 +176,16 @@ export function verifiedTeamSkillRecovery(state: Record, compan } function namedTeamSkillClauses(reply: string, skill: Pick): string[] { const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase()); - return reply.split(/[.!?;\n]+/).filter(clause => names.some(name => clause.toLowerCase().includes(name))); + const bound: string[] = []; + for (const paragraph of reply.split(/\n\s*\n/)) { + let previousNamesTarget = false; + for (const clause of paragraph.split(/[.!?;\n]+/).map(text => text.trim()).filter(Boolean)) { + const named = names.some(name => clause.toLowerCase().includes(name)); + if (named || previousNamesTarget && /^(?:it|(?:this|that) skill)\b/i.test(clause)) bound.push(clause); + previousNamesTarget = named; + } + } + return bound; } function truthfulTeamRecovery(reply: string, skill: Pick): boolean { return namedTeamSkillClauses(reply, skill).some(clause => [...clause.matchAll(/\b(?:installed|available|resolved|recovered)\b/gi)].some(match =>