fix(evals): reject contradictory missing-skill recovery claims

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
DottaandPaperclip committed 2026-11-04 19:14:18 -06:00
1 parent a400fd8a61
commit ca648d97ae
2 files changed
+12 -1

No files matched your search

@@ -260,3 +260,11 @@ it("does not credit resolved or all-ready claims as an unresolved occupancy repo
expect(truthfulTeamPartial(body, skill)).toBe(false);
}
});
it("rejects a later same-skill current recovery claim while saved state still shows the required catalog skill missing", () => {
const o = valid();
o.phaseReplies[0]!.body = "The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. Launch Checklist is now installed under another slug.";
expect(gradeTeamConditions(o, "initial").passed).toBe(false);
o.phaseReplies[0]!.body = "The existing CTO was preserved and the imported role is CTO 2. Launch Checklist's slug is occupied. The other three required skills installed.";
expect(gradeTeamConditions(o, "initial").passed).toBe(true);
});
@@ -266,7 +266,10 @@ export function truthfulTeamPartial(
)
);
});
return reported && !falseFull;
const subjects = [skill.slug, skill.slug.replaceAll("-", " "), skill.name]
.map(name => name.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")).join("|");
const currentRecovery = new RegExp("(?:\\b(?:" + subjects + ")[*_`]*|^(?:it|(?:this|that) skill))\\s+(?:skill\\s+)?(?:is|has been)\\s+(?:(?:now|already|currently)\\s+)?(?:installed|available|resolved|recovered)\\b", "i");
return reported && !falseFull && !namedTeamSkillClauses(reply, skill, true).some(clause => currentRecovery.test(clause));
}
export function gradeTeamConditions(
o: ExternalObservation,