From 552140495c16ee283f09cf595235adcc8a32a493 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 4 Nov 2026 17:40:26 -0600 Subject: [PATCH] fix(evals): retain bound skill contradiction chains Co-Authored-By: Paperclip --- .../external-team-conditional.test.ts | 10 ++++++++++ .../chat-catalog-domains/external-team-conditional.ts | 9 +++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts index e7c0c5bace3..a099a410f10 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts @@ -238,3 +238,13 @@ it.each(["Launch Checklist hit a conflict. Another skill was discussed. It is no o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. ${body}`; expect(gradeTeamConditions(o, "initial").passed).toBe(false); }); + +it("invalidates contradictory current failure in a contiguous bound pronoun chain in either phase", () => { + const outcomes = [0, 1].map(phase => { + const { o } = recovered(); + const contradiction = "Launch Checklist hit a conflict. It is now installed. It is still not installed."; + o.phaseReplies[phase]!.body = phase === 0 ? `The existing CTO was preserved and the new role renamed CTO 2. ${contradiction}` : contradiction; + return gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed; + }); + expect(outcomes).toEqual([false, false]); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts index 496d68ed2c6..b592ccdddcf 100644 --- a/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts +++ b/tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts @@ -174,15 +174,16 @@ export function verifiedTeamSkillRecovery(state: Record, compan && (selection.versionId === null || selection.versionId === version.id); }); } -function namedTeamSkillClauses(reply: string, skill: Pick): string[] { +function namedTeamSkillClauses(reply: string, skill: Pick, continuePronouns = false): string[] { const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase()); const bound: string[] = []; for (const paragraph of reply.split(/\n\s*\n/)) { let previousNamesTarget = false; for (const clause of paragraph.split(/[.!?;\n]+/).map(text => text.trim()).filter(Boolean)) { const named = names.some(name => clause.toLowerCase().includes(name)); - if (named || previousNamesTarget && /^(?:it|(?:this|that) skill)\b/i.test(clause)) bound.push(clause); - previousNamesTarget = named; + const pronoun: boolean = previousNamesTarget && /^(?:it|(?:this|that) skill)\b/i.test(clause); + if (named || pronoun) bound.push(clause); + previousNamesTarget = named || continuePronouns && pronoun; } } return bound; @@ -193,7 +194,7 @@ function truthfulTeamRecovery(reply: string, skill: Pick): boolean { - return namedTeamSkillClauses(reply, skill).some(clause => + return namedTeamSkillClauses(reply, skill, true).some(clause => [...clause.matchAll(/\b(?:not installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => { const before = clause.slice(0, match.index); return !/\b(?:was|were|had been)\s+(?:(?:previously|initially|formerly|earlier|still)\s+){0,2}$/i.test(before)