mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-10 12:37:55 +01:00
fix(evals): retain bound skill contradiction chains
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
ba8418fc4d
commit
552140495c
2 files changed
+15
-4
No files matched your search
@@ -238,3 +238,13 @@ it.each(["Launch Checklist hit a conflict. Another skill was discussed. It is no
|
||||
o.phaseReplies[0]!.body = `The existing CTO was preserved and the new role renamed CTO 2. ${body}`;
|
||||
expect(gradeTeamConditions(o, "initial").passed).toBe(false);
|
||||
});
|
||||
|
||||
it("invalidates contradictory current failure in a contiguous bound pronoun chain in either phase", () => {
|
||||
const outcomes = [0, 1].map(phase => {
|
||||
const { o } = recovered();
|
||||
const contradiction = "Launch Checklist hit a conflict. It is now installed. It is still not installed.";
|
||||
o.phaseReplies[phase]!.body = phase === 0 ? `The existing CTO was preserved and the new role renamed CTO 2. ${contradiction}` : contradiction;
|
||||
return gradeTeamConditions(o, phase === 0 ? "initial" : "complete").passed;
|
||||
});
|
||||
expect(outcomes).toEqual([false, false]);
|
||||
});
|
||||
@@ -174,15 +174,16 @@ export function verifiedTeamSkillRecovery(state: Record<string, unknown>, compan
|
||||
&& (selection.versionId === null || selection.versionId === version.id);
|
||||
});
|
||||
}
|
||||
function namedTeamSkillClauses(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): string[] {
|
||||
function namedTeamSkillClauses(reply: string, skill: Pick<CatalogSkill, "slug" | "name">, continuePronouns = false): string[] {
|
||||
const names = [skill.slug, skill.slug.replaceAll("-", " "), skill.name].map(name => name.toLowerCase());
|
||||
const bound: string[] = [];
|
||||
for (const paragraph of reply.split(/\n\s*\n/)) {
|
||||
let previousNamesTarget = false;
|
||||
for (const clause of paragraph.split(/[.!?;\n]+/).map(text => text.trim()).filter(Boolean)) {
|
||||
const named = names.some(name => clause.toLowerCase().includes(name));
|
||||
if (named || previousNamesTarget && /^(?:it|(?:this|that) skill)\b/i.test(clause)) bound.push(clause);
|
||||
previousNamesTarget = named;
|
||||
const pronoun: boolean = previousNamesTarget && /^(?:it|(?:this|that) skill)\b/i.test(clause);
|
||||
if (named || pronoun) bound.push(clause);
|
||||
previousNamesTarget = named || continuePronouns && pronoun;
|
||||
}
|
||||
}
|
||||
return bound;
|
||||
@@ -193,7 +194,7 @@ function truthfulTeamRecovery(reply: string, skill: Pick<CatalogSkill, "slug" |
|
||||
&& !claimsRecoveredSkillMissing(reply, skill);
|
||||
}
|
||||
function claimsRecoveredSkillMissing(reply: string, skill: Pick<CatalogSkill, "slug" | "name">): boolean {
|
||||
return namedTeamSkillClauses(reply, skill).some(clause =>
|
||||
return namedTeamSkillClauses(reply, skill, true).some(clause =>
|
||||
[...clause.matchAll(/\b(?:not installed|uninstalled|missing|unavailable|blocked)\b/gi)].some(match => {
|
||||
const before = clause.slice(0, match.index);
|
||||
return !/\b(?:was|were|had been)\s+(?:(?:previously|initially|formerly|earlier|still)\s+){0,2}$/i.test(before)
|
||||
|
||||
Reference in new issue
Block a user