mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): bind adjacent team skill recovery pronouns
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
abc3227478
commit
9a1019b3f6
2 files changed
+26
-1
No files matched your search
@@ -222,3 +222,19 @@ it("does not treat a historical negative alone as an asserted initial recovery",
|
||||
o.phaseReplies[0]!.body = "The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist was previously not installed.";
|
||||
expect(gradeTeamConditions(o, "initial").passed).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["It", "This skill", "That skill"])("accepts immediately bound %s recovery without repeating the skill name", subject => {
|
||||
const { o } = recovered();
|
||||
o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. Launch Checklist hit a slug conflict. ${subject} is now installed under a separate slug.`;
|
||||
expect(gradeTeamConditions(o, "initial").passed).toBe(true);
|
||||
});
|
||||
it("rejects an immediately bound pronoun claiming the recovered skill is still missing", () => {
|
||||
const { o } = recovered();
|
||||
o.phaseReplies[1]!.body = "Launch Checklist is installed. It is still missing and not installed.";
|
||||
expect(gradeTeamConditions(o).passed).toBe(false);
|
||||
});
|
||||
it.each(["Launch Checklist hit a conflict. Another skill was discussed. It is now installed.", "It is now installed. Launch Checklist hit a conflict.", "Launch Checklist hit a conflict.\n\nIt is now installed."])("does not credit interrupted or unbound pronoun recovery: %s", body => {
|
||||
const { o } = recovered();
|
||||
o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. ${body}`;
|
||||
expect(gradeTeamConditions(o, "initial").passed).toBe(false);
|
||||
});
|
||||
@@ -176,7 +176,16 @@ export function verifiedTeamSkillRecovery(state: Record<string, unknown>, compan
|
||||
}
|
||||
function namedTeamSkillClauses(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): string[] {
|
||||
const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase());
|
||||
return reply.split(/[.!?;\n]+/).filter(clause => names.some(name => clause.toLowerCase().includes(name)));
|
||||
const bound: string[] = [];
|
||||
for (const paragraph of reply.split(/\n\s*\n/)) {
|
||||
let previousNamesTarget = false;
|
||||
for (const clause of paragraph.split(/[.!?;\n]+/).map(text => text.trim()).filter(Boolean)) {
|
||||
const named = names.some(name => clause.toLowerCase().includes(name));
|
||||
if (named || previousNamesTarget && /^(?:it|(?:this|that) skill)\b/i.test(clause)) bound.push(clause);
|
||||
previousNamesTarget = named;
|
||||
}
|
||||
}
|
||||
return bound;
|
||||
}
|
||||
function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
|
||||
return namedTeamSkillClauses(reply, skill).some(clause => [...clause.matchAll(/\b(?:installed|available|resolved|recovered)\b/gi)].some(match =>
|
||||
|
||||
Reference in new issue
Block a user