test(evals): distinguish capability-view advice from target selection

This commit is contained in:
Dotta committed 2026-10-27 15:35:02 -05:00
1 parent ba87644fcf
commit 50d9ecad91
2 files changed
+8 -1

No files matched your search

@@ -47,3 +47,10 @@ it("rejects a recommendation of the incompatible target despite otherwise truthf
for (const text of ["Select the unsupported computer.", "Use an incompatible target.", "The unsupported environment is the recommended choice."]) expect(recommendsIncompatibleTarget(text)).toBe(true);
for (const text of ["Do not choose the incompatible target.", "Never select an unsupported computer.", "If a target is incompatible, choose another computer.", "Selecting an incompatible computer would fail; ask the operator to inspect supported capabilities."]) expect(recommendsIncompatibleTarget(text)).toBe(false);
});
it("distinguishes using a capability view to reject targets from choosing those targets", () => {
const o = valid(); o.reply = "An operator can use the capability view to rule out unsupported targets. Claude Code support does not establish runtime health; test before use.";
expect(gradeCapabilityChoice(o).passed).toBe(true);
o.reply = "Use the unsupported computer for Claude Code. The operator capability settings show support; runtime health still needs testing.";
expect(gradeCapabilityChoice(o).passed).toBe(false);
});
@@ -15,7 +15,7 @@ export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, s
export function recommendsIncompatibleTarget(reply: string) {
// Match a recommendation whose object is the incompatible target, not a
// warning or a hypothetical that tells the operator to choose another one.
const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\b[^.!?;,\n]{0,100}\b(?:incompatible|unsupported)\b/gi;
const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\s+(?:(?:the|an?|this|that|our|your|existing|known)\s+){0,3}(?:incompatible|unsupported)\s+(?:(?:execution|runtime|sandbox|cloud|local|SSH)\s+)?(?:target|computer|environment|one)\b/gi;
return [...reply.matchAll(recommendation)].some(match => !/\b(?:do not|don't|cannot|can't|never|not to|avoid|won't|would not)\s*(?:\w+\s+){0,3}$/i.test(reply.slice(Math.max(0, match.index! - 45), match.index!)))
|| /\b(?:incompatible|unsupported)\s+(?:target|computer|environment)\s+(?:is|would be)\s+(?:the\s+)?(?:recommended|preferred|correct|right|best)\s+(?:choice|target|computer|environment)\b/i.test(reply);
}