fix(evals): distinguish historical team recovery claims

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/external-team-conditional.test.ts

@@ -202,3 +202,23 @@ it("keeps historical failure disclosure and a generic reconciliation reply valid
   o.phaseReplies[1]!.body = "Launch Checklist was blocked before; it is now fixed. Launch Checklist is not missing. Launch Checklist is installed despite a blocked local slug. The same team already exists.";
   expect(gradeTeamConditions(o).passed).toBe(true);
 });
+
+it.each([0, 1])("accepts reported historical noninstallation followed by supported current recovery in phase %s", phase => {
+  const { o } = recovered();
+  const history = "Launch Checklist was previously not installed, but is now installed under a separate slug.";
+  o.phaseReplies[phase]!.body = phase === 0 ? `The existing CTO was preserved and the new role renamed CTO 2. ${history}` : history;
+  expect(gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed).toBe(true);
+});
+it("rejects an explicit current noninstallation claim after historical failure in either phase", () => {
+  for (const phase of [0, 1]) {
+    const { o } = recovered();
+    const wrong = "Launch Checklist was previously not installed and is still not installed.";
+    o.phaseReplies[phase]!.body = phase === 0 ? `The existing CTO was preserved and the new role renamed CTO 2. ${wrong}` : wrong;
+    expect(gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed).toBe(false);
+  }
+});
+it("does not treat a historical negative alone as an asserted initial recovery", () => {
+  const { o } = recovered();
+  o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was previously not installed.";
+  expect(gradeTeamConditions(o, "initial").passed).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/external-team-conditional.ts

@@ -174,17 +174,20 @@ export function verifiedTeamSkillRecovery(state: Record<string, unknown>, compan
       && (selection.versionId === null || selection.versionId === version.id);
   });
 }
+function namedTeamSkillClauses(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): string[] {
+  const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase());
+  return reply.split(/[.!?;\n]+/).filter(clause => names.some(name => clause.toLowerCase().includes(name)));
+}
 function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
-  return [skill.slug, skill.slug.replaceAll("-", " "), skill.name].some(name => reply.toLowerCase().includes(name.toLowerCase()))
-    && /\b(?:installed|available|resolved|recovered)\b/i.test(reply)
-    && !/\b(?:not installed|still (?:blocked|missing|unavailable)|remains (?:blocked|missing|unavailable))\b/i.test(reply);
+  return namedTeamSkillClauses(reply, skill).some(clause => [...clause.matchAll(/\b(?:installed|available|resolved|recovered)\b/gi)].some(match =>
+    !/\b(?:not|never|isn.t|aren.t|wasn.t|weren.t)\s+(?:(?:be|been|yet|currently|still)\s+){0,2}$/i.test(clause.slice(0, match.index))))
+    && !claimsRecoveredSkillMissing(reply, skill);
 }
 function claimsRecoveredSkillMissing(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
-  const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase());
-  return reply.split(/[.!?;\n]+/).some(clause => names.some(name => clause.toLowerCase().includes(name))
-    && [...clause.matchAll(/\b(?:not installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => {
+  return namedTeamSkillClauses(reply, skill).some(clause =>
+    [...clause.matchAll(/\b(?:not installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => {
       const before = clause.slice(0, match.index);
-      return !/\b(?:was|were|had been)\s+(?:still\s+)?$/i.test(before)
+      return !/\b(?:was|were|had been)\s+(?:(?:previously|initially|formerly|earlier|still)\s+){0,2}$/i.test(before)
         && (match[0].toLowerCase() === "not installed" || /\b(?:is|are|remains?|still)\s+(?:currently\s+)?$/i.test(before));
     }));
 }