diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts index e066d00371e..534d6184d34 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts @@ -1,6 +1,6 @@ import { expect, it } from "vitest"; import { getEnvironmentCapabilities } from "../../../packages/shared/src/environment-support.js"; -import { advertisedClaudeTargets, capabilityChoiceReply, gradeCapabilityChoice, runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js"; +import { advertisedClaudeTargets, capabilityChoiceReply, recommendsIncompatibleTarget, gradeCapabilityChoice, runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js"; import { runtimeComputerSshCases } from "./runtime-computers-ssh.js"; import { runtimePrivateRequirements } from "../chat-runtime-private-requirements.js"; function valid() { @@ -40,3 +40,10 @@ it("removes only the choice case private contract and preserves both genuine SSH expect(runtimePrivateRequirements).toHaveProperty(id); } }); + +it("rejects a recommendation of the incompatible target despite otherwise truthful capability advice", () => { + const o = valid(); o.reply = "Choose the incompatible target for Morgan with Claude Code. The operator settings explain capability support; runtime health still needs testing."; + expect(gradeCapabilityChoice(o).passed).toBe(false); + for (const text of ["Select the unsupported computer.", "Use an incompatible target.", "The unsupported environment is the recommended choice."]) expect(recommendsIncompatibleTarget(text)).toBe(true); + for (const text of ["Do not choose the incompatible target.", "Never select an unsupported computer.", "If a target is incompatible, choose another computer.", "Selecting an incompatible computer would fail; ask the operator to inspect supported capabilities."]) expect(recommendsIncompatibleTarget(text)).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts index ee50485e6f9..a964e2d6286 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts @@ -12,10 +12,17 @@ export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, s && unsupported.driver === "sandbox" && unsupported.config.provider === "fake" && claude.sandboxProviders.fake === "unsupported" && capabilities.sandboxProviders.fake?.status === "unsupported"; } +export function recommendsIncompatibleTarget(reply: string) { + // Match a recommendation whose object is the incompatible target, not a + // warning or a hypothetical that tells the operator to choose another one. + const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\b[^.!?;,\n]{0,100}\b(?:incompatible|unsupported)\b/gi; + return [...reply.matchAll(recommendation)].some(match => !/\b(?:do not|don't|cannot|can't|never|not to|avoid|won't|would not)\s*(?:\w+\s+){0,3}$/i.test(reply.slice(Math.max(0, match.index! - 45), match.index!))) + || /\b(?:incompatible|unsupported)\s+(?:target|computer|environment)\s+(?:is|would be)\s+(?:the\s+)?(?:recommended|preferred|correct|right|best)\s+(?:choice|target|computer|environment)\b/i.test(reply); +} export function capabilityChoiceReply(reply: string) { const healthClaim = /\b(?:(?:is|are|already|fully|now)\s+(?:fully\s+)?(?:ready|healthy|qualified|validated)|tested successfully|health (?:test|probe|validation) (?:passed|succeeded))\b/gi; const falseHealth = [...reply.matchAll(healthClaim)].some(match => !/\b(?:not|never)\s*$|\b(?:not|never)\s+(?:yet|already|fully)\s*$/i.test(reply.slice(Math.max(0, match.index! - 30), match.index!))); - return !falseHealth && /Claude Code/i.test(reply) && /capabilit|compatib|support/i.test(reply) + return !falseHealth && !recommendsIncompatibleTarget(reply) && /Claude Code/i.test(reply) && /capabilit|compatib|support/i.test(reply) && /health|probe|test|qualif|validat/i.test(reply) && /operator|administrator|settings|human|board/i.test(reply) && !/I (?:inspected|read|checked) (?:the )?(?:hidden|board.only|restricted) (?:environment )?inventory/i.test(reply); }