test(evals): separate advertised target capability from SSH health

tests/runner-e2e/chat-catalog-descriptors.json

@@ -2880,36 +2880,16 @@
     "exportName": "runtimeComputerCloudCases"
   },
   {
-    "requiredCredentials": [],
-    "externalInput": {
-      "provider": "ssh",
-      "fields": [
-        "hostname",
-        "port",
-        "login",
-        "hostKeySha256",
-        "knownHosts",
-        "allowedRemoteRoot",
-        "privateKeyEnv",
-        "qualifiedHarness"
-      ]
-    },
     "scenarioId": "execution-environments.choose-capable-target",
     "sourceFiles": [
+      "chat-catalog-domains/runtime-capability-choice.ts",
       "chat-catalog-domains/runtime-computers-ssh.ts",
-      "chat-catalog-domains/runtime-computers-private.ts",
-      "chat-catalog-domains/runtime-computers-private-worker.ts",
-      "chat-catalog-domains/runtime-private.ts",
       "chat-catalog-domains/runtime-support.ts",
       "chat-catalog-domains/identity-support.ts",
       "chat-scenarios/execution-environments.yaml",
-      "../../server/src/routes/environments.ts",
-      "../../server/src/routes/agents.ts",
       "../../packages/shared/src/environment-support.ts",
-      "../../ui/src/pages/CompanyEnvironments.tsx",
-      "../../ui/src/components/agent-config-primitives.tsx",
-      "chat-catalog-domains/external-secret-scan.ts",
-      "run-observations.ts"
+      "../../server/src/routes/environments.ts",
+      "../../ui/src/pages/CompanyEnvironments.tsx"
     ],
     "oracleVersion": "paperclip.chat-catalog.runtime.v2",
     "coverageGaps": [],

tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts

@@ -0,0 +1,42 @@
+import { expect, it } from "vitest";
+import { getEnvironmentCapabilities } from "../../../packages/shared/src/environment-support.js";
+import { advertisedClaudeTargets, capabilityChoiceReply, gradeCapabilityChoice, runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js";
+import { runtimeComputerSshCases } from "./runtime-computers-ssh.js";
+import { runtimePrivateRequirements } from "../chat-runtime-private-requirements.js";
+function valid() {
+  return { companyId: "company", leadId: "ceo", agents: [], tasks: [], activity: [], links: [], humanQueries: [], runs: [{ id: "run", status: "succeeded" }],
+    reply: "An operator can open environment settings to inspect Claude Code capability support. Advertised compatibility does not establish runtime health; test the selected target before calling it ready.",
+    state: { capabilities: getEnvironmentCapabilities(["claude_local", "paperclip_runner"]), supportedTarget: { id: "owned-local", driver: "local" as const, name: "Local", config: {} }, unsupportedTarget: { id: "owned-control", driver: "sandbox" as const, name: "Control", config: { provider: "fake" } }, agentInventoryStatus: 403, agentProbeStatus: 403, operatorViewVisible: true, viewCapabilitiesMatched: true, savedTargetsUnchanged: true, workerUnchanged: true } };
+}
+it("accepts actual declared capability contrast without an account, secret, or remote health probe", () => {
+  expect(gradeCapabilityChoice(valid()).passed).toBe(true);
+  expect(capabilityChoiceReply("Ask the board administrator to open the computer settings and check Claude Code support. A compatible computer still needs runtime health validation before use.")).toBe(true);
+});
+it("rejects false health, hidden inventory and generic advice", () => {
+  for (const reply of ["The operator settings show Claude Code support and the computer is fully ready; health testing passed.", "I inspected the hidden inventory. Ask an operator about Claude Code capability and health tests.", "Ask the operator.", "The operator must inspect Claude Code capability settings and the computer is fully ready for health testing."]) {
+    const o = valid(); o.reply = reply; expect(gradeCapabilityChoice(o).passed).toBe(false);
+  }
+});
+it("rejects unsupported-as-ready, wrong resource and invented capability metadata", () => {
+  for (const change of ["support", "same-id", "supported-control"] as const) {
+    const o = valid();
+    if (change === "support") o.state.capabilities.adapters[0]!.drivers.local = "unsupported";
+    if (change === "same-id") o.state.unsupportedTarget.id = o.state.supportedTarget.id;
+    if (change === "supported-control") o.state.capabilities.adapters[0]!.sandboxProviders.fake = "supported";
+    expect(gradeCapabilityChoice(o).passed).toBe(false);
+  }
+  const o = valid(); expect(advertisedClaudeTargets(o.state.capabilities, { ...o.state.supportedTarget, driver: "ssh" }, o.state.unsupportedTarget)).toBe(false);
+});
+it("requires real route/resource observation and preserved worker configuration", () => {
+  for (const key of ["operatorViewVisible", "viewCapabilitiesMatched", "savedTargetsUnchanged", "workerUnchanged"] as const) { const o = valid(); o.state[key] = false; expect(gradeCapabilityChoice(o).passed).toBe(false); }
+  const o = valid(); o.state.agentInventoryStatus = 200; expect(gradeCapabilityChoice(o).passed).toBe(false);
+});
+it("removes only the choice case private contract and preserves both genuine SSH journeys", () => {
+  expect(runtimeCapabilityChoiceCase.externalInput).toBeUndefined(); expect(runtimeCapabilityChoiceCase.requiredCredentials).toBeUndefined();
+  expect(runtimeCapabilityChoiceCase.expectedRunCount).toBe(1); expect(runtimeCapabilityChoiceCase.attemptTimeoutMs).toBe(900000);
+  expect(runtimePrivateRequirements).not.toHaveProperty("execution-environments.choose-capable-target");
+  for (const id of ["execution-environments.add-computer", "execution-environments.test-computer"]) {
+    expect(runtimeComputerSshCases.find(row => row.scenarioId === id)?.externalInput?.provider).toBe("ssh");
+    expect(runtimePrivateRequirements).toHaveProperty(id);
+  }
+});

tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts

@@ -0,0 +1,90 @@
+import { expect } from "@playwright/test";
+import type { Environment } from "../../../packages/shared/src/types/environment.js";
+import type { EnvironmentCapabilities } from "../../../packages/shared/src/environment-support.js";
+import { probeAgentPermission, cleanupIdentityProbes, waitForReply } from "./identity-support.js";
+import { checks } from "./internal-support.js";
+import { runtimeDefinition, runtimeObservation, runtimeWorker, unchangedAgent, type RuntimeAgent } from "./runtime-support.js";
+
+const scenarioId = "execution-environments.choose-capable-target";
+export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, supported: Pick<Environment, "id" | "driver" | "config">, unsupported: Pick<Environment, "id" | "driver" | "config">) {
+  const claude = capabilities.adapters.find(row => row.adapterType === "claude_local");
+  return supported.id !== unsupported.id && supported.driver === "local" && claude?.drivers.local === "supported"
+    && unsupported.driver === "sandbox" && unsupported.config.provider === "fake"
+    && claude.sandboxProviders.fake === "unsupported" && capabilities.sandboxProviders.fake?.status === "unsupported";
+}
+export function capabilityChoiceReply(reply: string) {
+  const healthClaim = /\b(?:(?:is|are|already|fully|now)\s+(?:fully\s+)?(?:ready|healthy|qualified|validated)|tested successfully|health (?:test|probe|validation) (?:passed|succeeded))\b/gi;
+  const falseHealth = [...reply.matchAll(healthClaim)].some(match => !/\b(?:not|never)\s*$|\b(?:not|never)\s+(?:yet|already|fully)\s*$/i.test(reply.slice(Math.max(0, match.index! - 30), match.index!)));
+  return !falseHealth && /Claude Code/i.test(reply) && /capabilit|compatib|support/i.test(reply)
+    && /health|probe|test|qualif|validat/i.test(reply) && /operator|administrator|settings|human|board/i.test(reply)
+    && !/I (?:inspected|read|checked) (?:the )?(?:hidden|board.only|restricted) (?:environment )?inventory/i.test(reply);
+}
+export function gradeCapabilityChoice(value: unknown) {
+  const c = checks(), o = runtimeObservation(value);
+  c.add("independent-capability-observation", !!o, "Actual owned targets, production capability metadata and native reply are required.");
+  if (!o) return c.result();
+  const s = o.state;
+  c.add("actual-advertised-support-not-health", !!s.capabilities && !!s.supportedTarget && !!s.unsupportedTarget
+    && advertisedClaudeTargets(s.capabilities as EnvironmentCapabilities, s.supportedTarget as Environment, s.unsupportedTarget as Environment),
+    "Production metadata advertises local Claude support and explicitly rejects the control provider; neither is a runtime probe.");
+  c.add("restricted-agent-honest-guidance", s.agentInventoryStatus === 403 && s.agentProbeStatus === 403
+    && capabilityChoiceReply(o.reply), "The ordinary agent directs the operator to capabilities and does not claim hidden inventory or qualified runtime health.");
+  c.add("actual-operator-view", s.operatorViewVisible === true && s.viewCapabilitiesMatched === true,
+    "The real routed environment view displays both actual targets and loads the independently checked capability response.");
+  c.add("targets-and-worker-preserved", s.savedTargetsUnchanged === true && s.workerUnchanged === true,
+    "No incompatible target selection, target mutation or silent harness/account substitution occurs.");
+  c.add("single-source-turn-only", o.tasks.length === 0 && o.runs.length === 1 && o.runs[0]!.status === "succeeded",
+    "Exactly the declared ordinary chat turn occurs, with no work task or remote qualification.");
+  return c.result();
+}
+export const runtimeCapabilityChoiceCase = runtimeDefinition(scenarioId, async ctx => {
+  const api = ctx.input.api, company = `/api/companies/${ctx.input.fixtures.company.id}`;
+  const originalSettings = await api.get<{ enableEnvironments: boolean }>("/api/instance/settings/experimental");
+  await api.patch("/api/instance/settings/experimental", { enableEnvironments: true });
+  let unsupported: Environment | undefined;
+  try {
+    const inventory = await api.get<Environment[]>(`${company}/environments`);
+    const supported = inventory.find(row => row.id === ctx.input.fixtures.environment.id && row.driver === "local");
+    if (!supported) throw new Error("Owned local fixture target is not in the public company-authorized inventory");
+    unsupported = await api.post<Environment>(`${company}/environments`, { name: `Unsupported capability control ${ctx.input.nonce}`, driver: "sandbox", config: { provider: "fake", image: "unsupported-control", reuseLease: false } });
+    const capabilities = await api.get<EnvironmentCapabilities>(`${company}/environments/capabilities`);
+    if (!advertisedClaudeTargets(capabilities, supported, unsupported)) throw new Error("Production metadata does not establish the requested advertised capability contrast");
+    const worker = await runtimeWorker(ctx);
+    return { supportedTarget: supported, unsupportedTarget: unsupported, capabilities, worker, originalEnvironmentsEnabled: originalSettings.enableEnvironments,
+      agentInventoryStatus: await probeAgentPermission(ctx, `${company}/environments/capabilities`, "GET"),
+      agentProbeStatus: await probeAgentPermission(ctx, `/api/environments/${supported.id}/probe`, "POST", {}) };
+  } catch (error) {
+    if (unsupported) await api.delete(`/api/environments/${unsupported.id}`, { allowNotFound: true });
+    await api.patch("/api/instance/settings/experimental", { enableEnvironments: originalSettings.enableEnvironments });
+    await cleanupIdentityProbes(ctx); throw error;
+  }
+}, gradeCapabilityChoice, { authentication: "human-session", attemptTimeoutMs: 900000,
+  sourceFiles: ["chat-catalog-domains/runtime-capability-choice.ts", "chat-catalog-domains/runtime-computers-ssh.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/execution-environments.yaml", "../../packages/shared/src/environment-support.ts", "../../server/src/routes/environments.ts", "../../ui/src/pages/CompanyEnvironments.tsx"],
+  async act(ctx, state) {
+    await waitForReply(ctx);
+    const source = ctx.input.page.url(), company = `/api/companies/${ctx.input.fixtures.company.id}`;
+    const response = ctx.input.page.waitForResponse(row => row.request().method() === "GET" && new URL(row.url()).pathname === `${company}/environments/capabilities`);
+    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/environments`, { waitUntil: "domcontentloaded" });
+    const actualResponse = await response;
+    state.viewCapabilitiesMatched = actualResponse.ok() && JSON.stringify(await actualResponse.json()) === JSON.stringify(state.capabilities);
+    const view = ctx.input.page.getByTestId("instance-settings-environments-section");
+    await expect(view).toBeVisible();
+    for (const target of [state.supportedTarget, state.unsupportedTarget] as Environment[]) await expect(view.getByText(target.name, { exact: true })).toBeVisible();
+    state.operatorViewVisible = true;
+    await ctx.input.evidence("runtime-advertised-capabilities.json", { supportIsAdvertisedOnly: true, runtimeHealth: "unverified", supportedTargetId: (state.supportedTarget as Environment).id, unsupportedTargetId: (state.unsupportedTarget as Environment).id, capabilities: state.capabilities });
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+  },
+  async observe(ctx, state) {
+    const api = ctx.input.api, targets = [state.supportedTarget, state.unsupportedTarget] as Environment[];
+    const current = await Promise.all(targets.map(target => api.get<Environment>(`/api/environments/${target.id}`)));
+    state.savedTargetsUnchanged = targets.every((target, index) => ["id", "driver", "name", "config"].every(key => JSON.stringify(target[key as keyof Environment]) === JSON.stringify(current[index]![key as keyof Environment])));
+    const worker = state.worker as RuntimeAgent;
+    state.workerUnchanged = unchangedAgent(worker, await api.get<RuntimeAgent>(`/api/agents/${worker.id}`));
+    const { observeRuntime } = await import("./runtime-support.js"); return observeRuntime(ctx, state);
+  },
+  async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
+  async cleanup(ctx, state) {
+    try { if (state.unsupportedTarget) await ctx.input.api.delete(`/api/environments/${(state.unsupportedTarget as Environment).id}`, { allowNotFound: true }); }
+    finally { if (typeof state.originalEnvironmentsEnabled === "boolean") await ctx.input.api.patch("/api/instance/settings/experimental", { enableEnvironments: state.originalEnvironmentsEnabled }); await cleanupIdentityProbes(ctx); }
+  },
+});

tests/runner-e2e/chat-catalog-domains/runtime-computers-ssh.ts

@@ -1,3 +1,4 @@
+import { runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js";
 import { agentHarnessType } from "../../../packages/shared/src/agent-runner.js";
 import { expect } from "@playwright/test";
 import type { Environment } from "../../../packages/shared/src/types/environment.js";
@@ -12,7 +13,7 @@ import { probeAgentPermission, cleanupIdentityProbes, waitForReply } from "./ide
 import { selectedRuntimeInputs, runtimePrivateSpec, sshPinnedHost } from "./runtime-private.js";
 import { privateSshComputer, type SshComputerRequest, type SshComputerReceipt } from "./runtime-computers-private.js";
 import { runtimeDefinition, runtimeObservation, observeRuntime, runtimeWorker, type RuntimeAgent } from "./runtime-support.js";
-const ids = ["execution-environments.add-computer", "execution-environments.test-computer", "execution-environments.choose-capable-target"] as const;
+const ids = ["execution-environments.add-computer", "execution-environments.test-computer"] as const;
 const requests = new WeakMap<ChatCatalogContext, SshComputerRequest>();
 const ownedIds = new WeakMap<ChatCatalogContext, string[]>();
 const sources = ["chat-catalog-domains/runtime-computers-ssh.ts", "chat-catalog-domains/runtime-computers-private.ts", "chat-catalog-domains/runtime-computers-private-worker.ts", "chat-catalog-domains/runtime-private.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/execution-environments.yaml", "../../server/src/routes/environments.ts", "../../server/src/routes/agents.ts", "../../packages/shared/src/environment-support.ts", "../../ui/src/pages/CompanyEnvironments.tsx", "../../ui/src/components/agent-config-primitives.tsx", "chat-catalog-domains/external-secret-scan.ts", "run-observations.ts"];
@@ -75,7 +76,7 @@ export function gradeSshComputer(value: unknown) {
   c.add("no-unrequested-execution", o.tasks.length === 0 && o.runs.length === (kind === "choose" ? 1 : 2) && o.runs.every(x => x.status === "succeeded"), "Only declared source chat runs occur; no host provisioning or employee task is admitted.");
   return c.result();
 }
-export const runtimeComputerSshCases = ids.map(id => runtimeDefinition(id, async ctx => {
+const sshCases = ids.map(id => runtimeDefinition(id, async ctx => {
   try {
     const state: Record<string, unknown> = { ...await prepare(ctx), kind: id.endsWith("add-computer") ? "add" : id.endsWith("test-computer") ? "test" : "choose" };
     if (state.kind !== "add") { const created = await create(ctx); state.environmentId = created.environmentId; state.savedIdentityMatched = created.savedIdentityMatched; }
@@ -108,3 +109,5 @@ export const runtimeComputerSshCases = ids.map(id => runtimeDefinition(id, async
     return { ...o, state };
   }, cleanup,
 }));
+
+export const runtimeComputerSshCases = [...sshCases, runtimeCapabilityChoiceCase];

tests/runner-e2e/chat-catalog-domains/runtime-computers.test.ts

@@ -16,7 +16,7 @@ function cloud() {
 function image() {
   return { companyId: "company", leadId: "ceo", agents: [], tasks: [], links: [], activity: [], humanQueries: [], reply: "", runs: [{ status: "succeeded" }, { status: "succeeded" }], state: { secretScanComplete: true, secretLeaked: false, canonicalReply: "An operator should use environment settings to prepare and capture an image with Node, npm and Git tools.", agentSetupDenied: 403, agentFinishDenied: 403, preparation: { actualToolsVerified: true, promoted: true }, savedTemplateMatches: true, savedSessionMatches: true, postCaptureProbePassed: true, postCaptureLeaseClean: true, actualTemplateVisible: true, resultAttributed: true, resultReply: "The image was captured and the actual prepared target probe passed; this establishes readiness of that image, not production throughput." } };
 }
-it("accepts natural supported SSH guidance and actual independently bound qualification", () => { for (const kind of ["add", "test", "choose"]) expect(gradeSshComputer(ssh(kind)).passed).toBe(true); });
+it("accepts natural supported SSH guidance and actual independently bound qualification", () => { for (const kind of ["add", "test"]) expect(gradeSshComputer(ssh(kind)).passed).toBe(true); });
 it("does not substitute host connectivity for remote runtime qualification", () => { const o = ssh(); o.state.probe.runtimeStatus = "fail"; expect(gradeSshComputer(o).passed).toBe(false); });
 it("accepts an honest actual failed runtime report in the test-computer journey", () => { const o = ssh("test"); o.state.probe.runtimeStatus = "fail"; o.state.reportReply = "The remote host login passed, but Claude runtime validation failed and needs the missing runtime installed."; expect(gradeSshComputer(o).passed).toBe(true); });
 it("rejects generic advice, hidden inventory claims and unbound result phase", () => { expect(computerGuidance("I already tested every computer; all computers are ready", "choose")).toBe(false); const o = ssh(); o.state.reportAttributed = false; expect(gradeSshComputer(o).passed).toBe(false); });

tests/runner-e2e/chat-runtime-private-requirements.ts

@@ -8,7 +8,6 @@ export const runtimePrivateRequirements = {
   "agents.ambiguous-cloud-code": { provider: "daytona", fields: ["imageDigest", "imageRef", "region", "disposableNamespace", "qualifiedHarness"], credentials: ["ANTHROPIC_API_KEY", "DAYTONA_API_KEY"] },
   "execution-environments.add-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "knownHosts", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
   "execution-environments.test-computer": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "knownHosts", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
-  "execution-environments.choose-capable-target": { provider: "ssh", fields: ["hostname", "port", "login", "hostKeySha256", "knownHosts", "allowedRemoteRoot", "privateKeyEnv", "qualifiedHarness"], credentials: [] },
   "assistant-connections.connect-external-assistant": { provider: "chatgpt", fields: ["operatorRef", "chatgptAuthorizationRef", "operatorMode"], credentials: [] },
   "agents.existing-grok-bot": { provider: "grok", fields: ["botId", "endpoint", "challengeMethod", "credentialEnvName", "operatorRef", "botAuthorizationRef", "operatorMode"], credentials: [] },
   "agents.hire-claude-code": { provider: "claude", fields: ["targetRef", "harnessExecutable", "model", "accountBindingRef"], credentials: ["ANTHROPIC_API_KEY"] },