From 50d9ecad91ba9dee6c4a8c8335ad053fd93517f4 Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 27 Oct 2026 15:35:02 -0500 Subject: [PATCH] test(evals): distinguish capability-view advice from target selection --- .../chat-catalog-domains/runtime-capability-choice.test.ts | 7 +++++++ .../chat-catalog-domains/runtime-capability-choice.ts | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts index 534d6184d34..0eb7dd79e32 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts @@ -47,3 +47,10 @@ it("rejects a recommendation of the incompatible target despite otherwise truthf for (const text of ["Select the unsupported computer.", "Use an incompatible target.", "The unsupported environment is the recommended choice."]) expect(recommendsIncompatibleTarget(text)).toBe(true); for (const text of ["Do not choose the incompatible target.", "Never select an unsupported computer.", "If a target is incompatible, choose another computer.", "Selecting an incompatible computer would fail; ask the operator to inspect supported capabilities."]) expect(recommendsIncompatibleTarget(text)).toBe(false); }); + +it("distinguishes using a capability view to reject targets from choosing those targets", () => { + const o = valid(); o.reply = "An operator can use the capability view to rule out unsupported targets. Claude Code support does not establish runtime health; test before use."; + expect(gradeCapabilityChoice(o).passed).toBe(true); + o.reply = "Use the unsupported computer for Claude Code. The operator capability settings show support; runtime health still needs testing."; + expect(gradeCapabilityChoice(o).passed).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts index a964e2d6286..35f8348f39f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts @@ -15,7 +15,7 @@ export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, s export function recommendsIncompatibleTarget(reply: string) { // Match a recommendation whose object is the incompatible target, not a // warning or a hypothetical that tells the operator to choose another one. - const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\b[^.!?;,\n]{0,100}\b(?:incompatible|unsupported)\b/gi; + const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\s+(?:(?:the|an?|this|that|our|your|existing|known)\s+){0,3}(?:incompatible|unsupported)\s+(?:(?:execution|runtime|sandbox|cloud|local|SSH)\s+)?(?:target|computer|environment|one)\b/gi; return [...reply.matchAll(recommendation)].some(match => !/\b(?:do not|don't|cannot|can't|never|not to|avoid|won't|would not)\s*(?:\w+\s+){0,3}$/i.test(reply.slice(Math.max(0, match.index! - 45), match.index!))) || /\b(?:incompatible|unsupported)\s+(?:target|computer|environment)\s+(?:is|would be)\s+(?:the\s+)?(?:recommended|preferred|correct|right|best)\s+(?:choice|target|computer|environment)\b/i.test(reply); }