mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-07 07:55:47 +01:00
test(evals): distinguish capability-view advice from target selection
This commit is contained in:
1 parent
ba87644fcf
commit
50d9ecad91
2 files changed
+8
-1
No files matched your search
@@ -47,3 +47,10 @@ it("rejects a recommendation of the incompatible target despite otherwise truthf
|
||||
for (const text of ["Select the unsupported computer.", "Use an incompatible target.", "The unsupported environment is the recommended choice."]) expect(recommendsIncompatibleTarget(text)).toBe(true);
|
||||
for (const text of ["Do not choose the incompatible target.", "Never select an unsupported computer.", "If a target is incompatible, choose another computer.", "Selecting an incompatible computer would fail; ask the operator to inspect supported capabilities."]) expect(recommendsIncompatibleTarget(text)).toBe(false);
|
||||
});
|
||||
|
||||
it("distinguishes using a capability view to reject targets from choosing those targets", () => {
|
||||
const o = valid(); o.reply = "An operator can use the capability view to rule out unsupported targets. Claude Code support does not establish runtime health; test before use.";
|
||||
expect(gradeCapabilityChoice(o).passed).toBe(true);
|
||||
o.reply = "Use the unsupported computer for Claude Code. The operator capability settings show support; runtime health still needs testing.";
|
||||
expect(gradeCapabilityChoice(o).passed).toBe(false);
|
||||
});
|
||||
@@ -15,7 +15,7 @@ export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, s
|
||||
export function recommendsIncompatibleTarget(reply: string) {
|
||||
// Match a recommendation whose object is the incompatible target, not a
|
||||
// warning or a hypothetical that tells the operator to choose another one.
|
||||
const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\b[^.!?;,\n]{0,100}\b(?:incompatible|unsupported)\b/gi;
|
||||
const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\s+(?:(?:the|an?|this|that|our|your|existing|known)\s+){0,3}(?:incompatible|unsupported)\s+(?:(?:execution|runtime|sandbox|cloud|local|SSH)\s+)?(?:target|computer|environment|one)\b/gi;
|
||||
return [...reply.matchAll(recommendation)].some(match => !/\b(?:do not|don't|cannot|can't|never|not to|avoid|won't|would not)\s*(?:\w+\s+){0,3}$/i.test(reply.slice(Math.max(0, match.index! - 45), match.index!)))
|
||||
|| /\b(?:incompatible|unsupported)\s+(?:target|computer|environment)\s+(?:is|would be)\s+(?:the\s+)?(?:recommended|preferred|correct|right|best)\s+(?:choice|target|computer|environment)\b/i.test(reply);
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user