test(evals): reject incompatible target recommendations

tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts

@@ -1,6 +1,6 @@
 import { expect, it } from "vitest";
 import { getEnvironmentCapabilities } from "../../../packages/shared/src/environment-support.js";
-import { advertisedClaudeTargets, capabilityChoiceReply, gradeCapabilityChoice, runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js";
+import { advertisedClaudeTargets, capabilityChoiceReply, recommendsIncompatibleTarget, gradeCapabilityChoice, runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js";
 import { runtimeComputerSshCases } from "./runtime-computers-ssh.js";
 import { runtimePrivateRequirements } from "../chat-runtime-private-requirements.js";
 function valid() {
@@ -40,3 +40,10 @@ it("removes only the choice case private contract and preserves both genuine SSH
     expect(runtimePrivateRequirements).toHaveProperty(id);
   }
 });
+
+it("rejects a recommendation of the incompatible target despite otherwise truthful capability advice", () => {
+  const o = valid(); o.reply = "Choose the incompatible target for Morgan with Claude Code. The operator settings explain capability support; runtime health still needs testing.";
+  expect(gradeCapabilityChoice(o).passed).toBe(false);
+  for (const text of ["Select the unsupported computer.", "Use an incompatible target.", "The unsupported environment is the recommended choice."]) expect(recommendsIncompatibleTarget(text)).toBe(true);
+  for (const text of ["Do not choose the incompatible target.", "Never select an unsupported computer.", "If a target is incompatible, choose another computer.", "Selecting an incompatible computer would fail; ask the operator to inspect supported capabilities."]) expect(recommendsIncompatibleTarget(text)).toBe(false);
+});

tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts

@@ -12,10 +12,17 @@ export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, s
     && unsupported.driver === "sandbox" && unsupported.config.provider === "fake"
     && claude.sandboxProviders.fake === "unsupported" && capabilities.sandboxProviders.fake?.status === "unsupported";
 }
+export function recommendsIncompatibleTarget(reply: string) {
+  // Match a recommendation whose object is the incompatible target, not a
+  // warning or a hypothetical that tells the operator to choose another one.
+  const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\b[^.!?;,\n]{0,100}\b(?:incompatible|unsupported)\b/gi;
+  return [...reply.matchAll(recommendation)].some(match => !/\b(?:do not|don't|cannot|can't|never|not to|avoid|won't|would not)\s*(?:\w+\s+){0,3}$/i.test(reply.slice(Math.max(0, match.index! - 45), match.index!)))
+    || /\b(?:incompatible|unsupported)\s+(?:target|computer|environment)\s+(?:is|would be)\s+(?:the\s+)?(?:recommended|preferred|correct|right|best)\s+(?:choice|target|computer|environment)\b/i.test(reply);
+}
 export function capabilityChoiceReply(reply: string) {
   const healthClaim = /\b(?:(?:is|are|already|fully|now)\s+(?:fully\s+)?(?:ready|healthy|qualified|validated)|tested successfully|health (?:test|probe|validation) (?:passed|succeeded))\b/gi;
   const falseHealth = [...reply.matchAll(healthClaim)].some(match => !/\b(?:not|never)\s*$|\b(?:not|never)\s+(?:yet|already|fully)\s*$/i.test(reply.slice(Math.max(0, match.index! - 30), match.index!)));
-  return !falseHealth && /Claude Code/i.test(reply) && /capabilit|compatib|support/i.test(reply)
+  return !falseHealth && !recommendsIncompatibleTarget(reply) && /Claude Code/i.test(reply) && /capabilit|compatib|support/i.test(reply)
     && /health|probe|test|qualif|validat/i.test(reply) && /operator|administrator|settings|human|board/i.test(reply)
     && !/I (?:inspected|read|checked) (?:the )?(?:hidden|board.only|restricted) (?:environment )?inventory/i.test(reply);
 }