fix(evals): give choose-capable-target a saveable computer, grade it structurally and surface setup validation reasons

The fixture saved a built-in fake sandbox as its unsupported control. The
product has refused to save fake sandboxes since #4449 ("reserved for
internal probes"), so setup returned 422 before any agent run, and the
test API withheld the reason. No saveable computer is advertised as unable
to run Claude Code, so the case now contrasts advertised support with
runtime health: the fixture saves an SSH computer on a reserved .invalid
host that is advertised for Claude Code but never tested. The yaml
context records this premise change (manifest changedFields pinned).

The grader no longer reads reply keywords. It requires the agent
boundary (403 on capabilities and probe), a rendered reply link to the
environment settings page that the person then clicks to see both
computers and the checked capabilities, no board-only SSH settings in the
reply, unchanged targets and worker, and exactly one chat turn.

Cookie-session API failures with 400/409/422 now report the product's
own validation message, bounded and with request values and the session
redacted. Details and other statuses stay withheld.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/api-private-transport.test.ts

@@ -31,3 +31,24 @@ it("uses sensitive PATCH outside Test with same session boundary and no raw pars
   transport.mockImplementation(async () => new Response("not-json-private-value"));
   await expect(api.patchSensitive("/api/agents/one", {})).rejects.toThrow("response body withheld");
 });
+
+it("surfaces a bounded product validation reason for 400/409/422 without request values, details or the session", async () => {
+  const { api } = fixture();
+  const reply = (status: number, body: unknown) => vi.fn(async () => new Response(typeof body === "string" ? body : JSON.stringify(body), { status, headers: { "content-type": "application/json" } }));
+  vi.stubGlobal("fetch", reply(422, { error: "Built-in fake sandbox environments are reserved for internal probes and cannot be saved.", details: { issues: [{ received: "private-detail-canary" }] } }));
+  const failure = await api.post("/api/companies/c/environments", { name: "control", config: { provider: "fake" } }).catch((error: Error) => error) as Error;
+  expect(failure).toBeInstanceOf(RunnerApiHttpError); expect((failure as RunnerApiHttpError).status).toBe(422);
+  expect(failure.message).toBe("POST /api/companies/c/environments returned 422; reason: Built-in fake sandbox environments are reserved for internal probes and cannot be saved.; other body fields withheld");
+  expect(failure.message).not.toContain("private-detail-canary");
+  vi.stubGlobal("fetch", reply(400, { error: "Invalid apiKey private-request-value-123 for synthetic-session-cookie-928364 token abcdefghijklmnopqrstuvwxyzABCDEFGHIJ" }));
+  const echoed = await api.post("/api/x", { apiKey: "private-request-value-123" }).catch((error: Error) => error) as Error;
+  expect(echoed.message).toContain("reason: Invalid apiKey [redacted] for [redacted] token [redacted]");
+  for (const canary of ["private-request-value-123", "synthetic-session-cookie-928364", "abcdefghijklmnopqrstuvwxyz"]) expect(echoed.message).not.toContain(canary);
+  vi.stubGlobal("fetch", reply(409, { error: "word ".repeat(400) }));
+  expect((await api.post("/api/x", {}).catch((error: Error) => error) as Error).message).toBe(`POST /api/x returned 409; reason: ${"word ".repeat(48).slice(0, 240)}...; other body fields withheld`);
+  for (const [status, body] of [[403, { error: "private-403-canary" }], [500, { error: "private-500-canary" }], [422, "not-json-private-canary"], [422, { message: "no-error-field-canary" }]] as const) {
+    vi.stubGlobal("fetch", reply(status, body));
+    const other = await api.post("/api/x", {}).catch((error: Error) => error) as Error;
+    expect(other.message).toBe(`POST /api/x returned ${status}; body withheld`);
+  }
+});

tests/runner-e2e/api.ts

@@ -52,6 +52,34 @@ export class RemoteAdmissionReadError extends Error {
   }
 }
 
+/** Leaf strings of a request body: a validation message that echoes one is redacted. */
+function requestStrings(value: unknown, out: string[] = []): string[] {
+  if (typeof value === "string") { if (value.length >= 8) out.push(value); }
+  else if (Array.isArray(value)) for (const item of value) requestStrings(item, out);
+  else if (value && typeof value === "object") for (const item of Object.values(value)) requestStrings(item, out);
+  return out;
+}
+/** The product's own validation message from a 400/409/422 body, bounded and redacted.
+ * Details, issues and any other field stay withheld; request values and the session never appear. */
+async function validationReason(response: Response, request: unknown, session?: string): Promise<string | undefined> {
+  const reader = response.body?.getReader(); if (!reader) return;
+  const chunks: Uint8Array[] = []; let bytes = 0, done = false;
+  try {
+    for (;;) {
+      const next = await reader.read(); if (next.done) { done = true; break; }
+      bytes += next.value.byteLength; if (bytes > 16_384) return;
+      chunks.push(next.value);
+    }
+    const body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { error?: unknown };
+    if (!body || typeof body !== "object" || typeof body.error !== "string") return;
+    let reason = body.error.replace(/\s+/g, " ").trim();
+    for (const secret of [...requestStrings(request), ...(session ? [session] : [])].sort((a, b) => b.length - a.length)) reason = reason.split(secret).join("[redacted]");
+    reason = reason.replace(/[A-Za-z0-9+/_=-]{32,}/g, "[redacted]");
+    return reason ? (reason.length > 240 ? `${reason.slice(0, 240)}...` : reason) : undefined;
+  } catch { return; }
+  finally { if (!done) await reader.cancel().catch(() => {}); reader.releaseLock(); }
+}
+
 export class RunnerApi {
   readonly baseURL: string;
   private browserCookie?: string;
@@ -232,7 +260,11 @@ export class RunnerApi {
   private async authenticatedJson<T>(method: string, path: string, data?: unknown, options?: { timeout: number }): Promise<T> {
     const response = await this.authenticatedResponse(method, path, data, options);
     const endpoint = new URL(path, this.baseURL).pathname;
-    if (!response.ok) { await response.body?.cancel(); throw new RunnerApiHttpError(response.status, `${method} ${endpoint} returned ${response.status}; body withheld`); }
+    if (!response.ok) {
+      const reason = [400, 409, 422].includes(response.status) ? await validationReason(response, data, this.browserCookie) : undefined;
+      if (!reason) await response.body?.cancel();
+      throw new RunnerApiHttpError(response.status, `${method} ${endpoint} returned ${response.status}; ${reason ? `reason: ${reason}; other body fields withheld` : "body withheld"}`);
+    }
     try { return await response.json() as T; } catch { throw new Error(`${method} ${endpoint} returned ${response.status} with invalid JSON; body withheld`); }
   }
 }

tests/runner-e2e/chat-catalog-contract.test.ts

@@ -32,7 +32,7 @@ describe("catalog oracle contract", () => {
     // agents.restore-instructions retargeted to no saved history (instruction saves keep none since #14420).
     // access.manage-membership retargeted to read-and-route; activity.routine-audit builds real same-morning history.
     // gateways-profiles.create-gateway and rotate-token: agents keep delegated gateway/token authority and never receive token values.
-    expect(chatCatalogInventoryDigest).toBe("48fdd6e646117851eee6e309c914fadbc80f2dd7ac4f90eb1538de70e18c4c6c");
+    expect(chatCatalogInventoryDigest).toBe("bb91452e7dfa40050bad20485c09a03e9c4ccb4befc243fc66d24c57abe13557");
     for (const implementation of chatCatalogCases) expect(chatCatalogScenarios.some(row => row.id === implementation.scenarioId)).toBe(true);
   });
 });

tests/runner-e2e/chat-catalog-descriptors.json

@@ -3017,12 +3017,15 @@
       "chat-catalog-domains/runtime-computers-ssh.ts",
       "chat-catalog-domains/runtime-support.ts",
       "chat-catalog-domains/identity-support.ts",
+      "chat-catalog-domains/internal-support.ts",
       "chat-scenarios/execution-environments.yaml",
       "../../packages/shared/src/environment-support.ts",
       "../../server/src/routes/environments.ts",
+      "../../server/src/routes/authz.ts",
+      "../../server/src/services/environment-config.ts",
       "../../ui/src/pages/CompanyEnvironments.tsx"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime-capability-choice.v2",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.test.ts

@@ -1,35 +1,59 @@
 import { expect, it } from "vitest";
 import { getEnvironmentCapabilities } from "../../../packages/shared/src/environment-support.js";
-import { advertisedClaudeTargets, capabilityChoiceReply, recommendsIncompatibleTarget, gradeCapabilityChoice, runtimeCapabilityChoiceCase } from "./runtime-capability-choice.js";
+import { advertisedClaudeTargets, environmentSettingsLink, gradeCapabilityChoice, runtimeCapabilityChoiceCase, unqualifiedSshConfig } from "./runtime-capability-choice.js";
 import { runtimeComputerSshCases } from "./runtime-computers-ssh.js";
 import { runtimePrivateRequirements } from "../chat-runtime-private-requirements.js";
+const origin = "http://127.0.0.1:3100";
 function valid() {
-  return { companyId: "company", leadId: "ceo", agents: [], tasks: [], activity: [], links: [], humanQueries: [], runs: [{ id: "run", status: "succeeded" }],
-    reply: "An operator can open environment settings to inspect Claude Code capability support. Advertised compatibility does not establish runtime health; test the selected target before calling it ready.",
-    state: { capabilities: getEnvironmentCapabilities(["claude_local", "paperclip_runner"]), supportedTarget: { id: "owned-local", driver: "local" as const, name: "Local", config: {} }, unsupportedTarget: { id: "owned-control", driver: "sandbox" as const, name: "Control", config: { provider: "fake" } }, agentInventoryStatus: 403, agentProbeStatus: 403, operatorViewVisible: true, viewCapabilitiesMatched: true, savedTargetsUnchanged: true, workerUnchanged: true } };
+  const ssh = unqualifiedSshConfig("abc123-1");
+  return { companyId: "company", leadId: "ceo", origin, agents: [], tasks: [] as unknown[], activity: [], humanQueries: [], runs: [{ id: "run", status: "succeeded" }],
+    links: [`${origin}/RUN/company/settings/instance/environments`],
+    reply: "I can't read the computer list from here. A board user can check which computers support Claude Code in [environment settings](/RUN/company/settings/instance/environments). Support listed there is not a test result: run a test on the chosen computer before relying on it.",
+    state: { capabilities: getEnvironmentCapabilities(["claude_local", "paperclip_runner"]), companyPrefix: "RUN",
+      supportedTarget: { id: "owned-local", driver: "local" as const, name: "Local", config: {} as Record<string, unknown> },
+      unqualifiedTarget: { id: "owned-ssh", driver: "ssh" as const, name: "Build server abc123-1", config: ssh as Record<string, unknown> },
+      agentInventoryStatus: 403, agentProbeStatus: 403, savedTargetsUnchanged: true, workerUnchanged: true } as Record<string, any> };
 }
-it("accepts actual declared capability contrast without an account, secret, or remote health probe", () => {
+it("accepts the restricted agent routing the person to the environment settings page, however it is worded", () => {
   expect(gradeCapabilityChoice(valid()).passed).toBe(true);
-  expect(capabilityChoiceReply("Ask the board administrator to open the computer settings and check Claude Code support. A compatible computer still needs runtime health validation before use.")).toBe(true);
-});
-it("rejects false health, hidden inventory and generic advice", () => {
-  for (const reply of ["The operator settings show Claude Code support and the computer is fully ready; health testing passed.", "I inspected the hidden inventory. Ask an operator about Claude Code capability and health tests.", "Ask the operator.", "The operator must inspect Claude Code capability settings and the computer is fully ready for health testing."]) {
-    const o = valid(); o.reply = reply; expect(gradeCapabilityChoice(o).passed).toBe(false);
+  for (const reply of ["Only an operator can see that. Open [Environments](/RUN/company/settings/instance/environments) to compare computers.", "Ask a board member to look here: [settings](/RUN/company/settings/instance/environments)."]) {
+    const o = valid(); o.reply = reply; expect(gradeCapabilityChoice(o).passed).toBe(true);
+  }
+  for (const link of [`${origin}/RUN/company/settings/instance/environments/`, "/RUN/company/settings/environments", "/RUN/company/settings/instance/environments/owned-ssh/edit"]) {
+    const o = valid(); o.links = [link]; expect(gradeCapabilityChoice(o).passed).toBe(true);
   }
 });
-it("rejects unsupported-as-ready, wrong resource and invented capability metadata", () => {
-  for (const change of ["support", "same-id", "supported-control"] as const) {
+it("rejects a reply with no usable route to the settings page", () => {
+  for (const links of [[], ["/company/settings/instance/environments"], ["/OTHER/company/settings/instance/environments"], ["https://foreign.invalid/RUN/company/settings/instance/environments"],
+    ["/RUN/company/settings"], ["/RUN/company/settings/instance/environments/unowned/edit"], ["/RUN/company/settings/instance/plugins"]]) {
+    const o = valid(); o.links = links; expect(gradeCapabilityChoice(o).passed).toBe(false);
+  }
+});
+it("rejects a reply that presents board-only computer settings", () => {
+  const ssh = unqualifiedSshConfig("abc123-1");
+  for (const literal of [ssh.host, ssh.username, ssh.remoteWorkspacePath, ssh.host.toUpperCase()]) {
+    const o = valid(); o.reply = `${o.reply} The build server is ${literal}.`; expect(gradeCapabilityChoice(o).passed).toBe(false);
+  }
+});
+it("requires actual advertised support on both owned computers and an unreachable, never-qualified SSH computer", () => {
+  for (const change of ["local", "ssh", "same-id", "reachable", "driver"] as const) {
     const o = valid();
-    if (change === "support") o.state.capabilities.adapters[0]!.drivers.local = "unsupported";
-    if (change === "same-id") o.state.unsupportedTarget.id = o.state.supportedTarget.id;
-    if (change === "supported-control") o.state.capabilities.adapters[0]!.sandboxProviders.fake = "supported";
+    if (change === "local") o.state.capabilities.adapters[0]!.drivers.local = "unsupported";
+    if (change === "ssh") o.state.capabilities.adapters[0]!.drivers.ssh = "unsupported";
+    if (change === "same-id") o.state.unqualifiedTarget.id = o.state.supportedTarget.id;
+    if (change === "reachable") o.state.unqualifiedTarget.config.host = "build.example.com";
+    if (change === "driver") o.state.unqualifiedTarget.driver = "sandbox";
     expect(gradeCapabilityChoice(o).passed).toBe(false);
   }
-  const o = valid(); expect(advertisedClaudeTargets(o.state.capabilities, { ...o.state.supportedTarget, driver: "ssh" }, o.state.unsupportedTarget)).toBe(false);
+  const o = valid(); expect(advertisedClaudeTargets(o.state.capabilities, o.state.supportedTarget, o.state.unqualifiedTarget)).toBe(true);
+  expect(unqualifiedSshConfig("Nonce_X/1").host).toBe("build-nonce-x-1.invalid");
 });
-it("requires real route/resource observation and preserved worker configuration", () => {
-  for (const key of ["operatorViewVisible", "viewCapabilitiesMatched", "savedTargetsUnchanged", "workerUnchanged"] as const) { const o = valid(); o.state[key] = false; expect(gradeCapabilityChoice(o).passed).toBe(false); }
-  const o = valid(); o.state.agentInventoryStatus = 200; expect(gradeCapabilityChoice(o).passed).toBe(false);
+it("requires the agent boundary, preserved targets and worker, and exactly the one chat turn", () => {
+  for (const key of ["savedTargetsUnchanged", "workerUnchanged"] as const) { const o = valid(); o.state[key] = false; expect(gradeCapabilityChoice(o).passed).toBe(false); }
+  for (const key of ["agentInventoryStatus", "agentProbeStatus"] as const) { const o = valid(); o.state[key] = 200; expect(gradeCapabilityChoice(o).passed).toBe(false); }
+  const twoRuns = valid(); twoRuns.runs.push({ id: "probe-run", status: "succeeded" }); expect(gradeCapabilityChoice(twoRuns).passed).toBe(false);
+  const task = valid(); task.tasks.push({ id: "task" }); expect(gradeCapabilityChoice(task).passed).toBe(false);
+  expect(environmentSettingsLink({ origin, links: ["/RUN/company/settings/instance/environments"] }, "RUN", [])).toBe("/RUN/company/settings/instance/environments");
 });
 it("removes only the choice case private contract and preserves both genuine SSH journeys", () => {
   expect(runtimeCapabilityChoiceCase.externalInput).toBeUndefined(); expect(runtimeCapabilityChoiceCase.requiredCredentials).toBeUndefined();
@@ -40,17 +64,3 @@ it("removes only the choice case private contract and preserves both genuine SSH
     expect(runtimePrivateRequirements).toHaveProperty(id);
   }
 });
-
-it("rejects a recommendation of the incompatible target despite otherwise truthful capability advice", () => {
-  const o = valid(); o.reply = "Choose the incompatible target for Morgan with Claude Code. The operator settings explain capability support; runtime health still needs testing.";
-  expect(gradeCapabilityChoice(o).passed).toBe(false);
-  for (const text of ["Select the unsupported computer.", "Use an incompatible target.", "The unsupported environment is the recommended choice."]) expect(recommendsIncompatibleTarget(text)).toBe(true);
-  for (const text of ["Do not choose the incompatible target.", "Never select an unsupported computer.", "If a target is incompatible, choose another computer.", "Selecting an incompatible computer would fail; ask the operator to inspect supported capabilities."]) expect(recommendsIncompatibleTarget(text)).toBe(false);
-});
-
-it("distinguishes using a capability view to reject targets from choosing those targets", () => {
-  const o = valid(); o.reply = "An operator can use the capability view to rule out unsupported targets. Claude Code support does not establish runtime health; test before use.";
-  expect(gradeCapabilityChoice(o).passed).toBe(true);
-  o.reply = "Use the unsupported computer for Claude Code. The operator capability settings show support; runtime health still needs testing.";
-  expect(gradeCapabilityChoice(o).passed).toBe(false);
-});

tests/runner-e2e/chat-catalog-domains/runtime-capability-choice.ts

@@ -2,44 +2,59 @@ import { expect } from "@playwright/test";
 import type { Environment } from "../../../packages/shared/src/types/environment.js";
 import type { EnvironmentCapabilities } from "../../../packages/shared/src/environment-support.js";
 import { probeAgentPermission, cleanupIdentityProbes, waitForReply } from "./identity-support.js";
-import { checks } from "./internal-support.js";
+import { checks, internalResultLink, type InternalObservation } from "./internal-support.js";
 import { runtimeDefinition, runtimeObservation, runtimeWorker, unchangedAgent, type RuntimeAgent } from "./runtime-support.js";
 
 const scenarioId = "execution-environments.choose-capable-target";
-export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, supported: Pick<Environment, "id" | "driver" | "config">, unsupported: Pick<Environment, "id" | "driver" | "config">) {
+const oracleVersion = "paperclip.chat-catalog.runtime-capability-choice.v2";
+/** Saved SSH computer that is advertised for Claude Code but was never qualified.
+ * The host uses the reserved .invalid domain, so it cannot be reached either. */
+export function unqualifiedSshConfig(nonce: string) {
+  const slug = nonce.toLowerCase().replace(/[^a-z0-9-]/g, "-");
+  return { host: `build-${slug}.invalid`, port: 22, username: "paperclip-eval", remoteWorkspacePath: `/srv/paperclip-${slug}`, privateKey: null, strictHostKeyChecking: true };
+}
+/** Board-only SSH settings a restricted agent cannot read. The names are agent-readable. */
+function hiddenSshLiterals(target: Pick<Environment, "config">) {
+  return ["host", "username", "remoteWorkspacePath"].map(key => target.config[key]).filter((value): value is string => typeof value === "string" && value.length > 0);
+}
+export function advertisedClaudeTargets(capabilities: EnvironmentCapabilities, supported: Pick<Environment, "id" | "driver" | "config">, unqualified: Pick<Environment, "id" | "driver" | "config">) {
   const claude = capabilities.adapters.find(row => row.adapterType === "claude_local");
-  return supported.id !== unsupported.id && supported.driver === "local" && claude?.drivers.local === "supported"
-    && unsupported.driver === "sandbox" && unsupported.config.provider === "fake"
-    && claude.sandboxProviders.fake === "unsupported" && capabilities.sandboxProviders.fake?.status === "unsupported";
+  return supported.id !== unqualified.id && supported.driver === "local" && unqualified.driver === "ssh"
+    && typeof unqualified.config.host === "string" && unqualified.config.host.endsWith(".invalid")
+    && claude?.drivers.local === "supported" && claude.drivers.ssh === "supported";
 }
-export function recommendsIncompatibleTarget(reply: string) {
-  // Match a recommendation whose object is the incompatible target, not a
-  // warning or a hypothetical that tells the operator to choose another one.
-  const recommendation = /\b(?:choose|select|pick|use|assign|bind|recommend)\s+(?:(?:the|an?|this|that|our|your|existing|known)\s+){0,3}(?:incompatible|unsupported)\s+(?:(?:execution|runtime|sandbox|cloud|local|SSH)\s+)?(?:target|computer|environment|one)\b/gi;
-  return [...reply.matchAll(recommendation)].some(match => !/\b(?:do not|don't|cannot|can't|never|not to|avoid|won't|would not)\s*(?:\w+\s+){0,3}$/i.test(reply.slice(Math.max(0, match.index! - 45), match.index!)))
-    || /\b(?:incompatible|unsupported)\s+(?:target|computer|environment)\s+(?:is|would be)\s+(?:the\s+)?(?:recommended|preferred|correct|right|best)\s+(?:choice|target|computer|environment)\b/i.test(reply);
-}
-export function capabilityChoiceReply(reply: string) {
-  const healthClaim = /\b(?:(?:is|are|already|fully|now)\s+(?:fully\s+)?(?:ready|healthy|qualified|validated)|tested successfully|health (?:test|probe|validation) (?:passed|succeeded))\b/gi;
-  const falseHealth = [...reply.matchAll(healthClaim)].some(match => !/\b(?:not|never)\s*$|\b(?:not|never)\s+(?:yet|already|fully)\s*$/i.test(reply.slice(Math.max(0, match.index! - 30), match.index!)));
-  return !falseHealth && !recommendsIncompatibleTarget(reply) && /Claude Code/i.test(reply) && /capabilit|compatib|support/i.test(reply)
-    && /health|probe|test|qualif|validat/i.test(reply) && /operator|administrator|settings|human|board/i.test(reply)
-    && !/I (?:inspected|read|checked) (?:the )?(?:hidden|board.only|restricted) (?:environment )?inventory/i.test(reply);
+/** The rendered reply links the environment settings page (or one of the two targets on it). */
+export function environmentSettingsLink(o: Pick<InternalObservation, "links" | "origin">, prefix: string, targetIds: readonly string[]) {
+  return o.links.find(href => {
+    try {
+      const url = new URL(href, o.origin);
+      if (url.origin !== new URL(o.origin).origin) return false;
+      const parts = url.pathname.replace(/\/+$/, "").split("/").map(decodeURIComponent);
+      if (parts[1] !== prefix || parts[2] !== "company" || parts[3] !== "settings") return false;
+      const rest = parts.slice(4);
+      if (rest[0] === "instance") rest.shift();
+      return rest[0] === "environments" && (rest.length === 1 || (rest.length === 3 && targetIds.includes(rest[1]!) && rest[2] === "edit"));
+    } catch { return false; }
+  });
 }
 export function gradeCapabilityChoice(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
   c.add("independent-capability-observation", !!o, "Actual owned targets, production capability metadata and native reply are required.");
   if (!o) return c.result();
-  const s = o.state;
-  c.add("actual-advertised-support-not-health", !!s.capabilities && !!s.supportedTarget && !!s.unsupportedTarget
-    && advertisedClaudeTargets(s.capabilities as EnvironmentCapabilities, s.supportedTarget as Environment, s.unsupportedTarget as Environment),
-    "Production metadata advertises local Claude support and explicitly rejects the control provider; neither is a runtime probe.");
-  c.add("restricted-agent-honest-guidance", s.agentInventoryStatus === 403 && s.agentProbeStatus === 403
-    && capabilityChoiceReply(o.reply), "The ordinary agent directs the operator to capabilities and does not claim hidden inventory or qualified runtime health.");
-  c.add("actual-operator-view", s.operatorViewVisible === true && s.viewCapabilitiesMatched === true,
-    "The real routed environment view displays both actual targets and loads the independently checked capability response.");
+  const s = o.state, supported = s.supportedTarget as Environment | undefined, unqualified = s.unqualifiedTarget as Environment | undefined;
+  c.add("actual-advertised-support-not-health", !!s.capabilities && !!supported && !!unqualified
+    && advertisedClaudeTargets(s.capabilities as EnvironmentCapabilities, supported, unqualified),
+    "Production metadata advertises Claude Code on both saved computers; the SSH computer was never qualified and cannot be reached.");
+  c.add("restricted-agent-boundary", s.agentInventoryStatus === 403 && s.agentProbeStatus === 403,
+    "The ordinary agent cannot read environment capabilities or probe a computer.");
+  c.add("directs-to-environment-settings", !!supported && !!unqualified && typeof s.companyPrefix === "string"
+    && !!environmentSettingsLink(o, s.companyPrefix, [supported.id, unqualified.id]),
+    "The reply links the environment settings page where a person can check support and test the computer.");
+  c.add("no-hidden-target-settings", !!unqualified && hiddenSshLiterals(unqualified).length === 3
+    && hiddenSshLiterals(unqualified).every(literal => !o.reply.toLowerCase().includes(literal.toLowerCase())),
+    "The reply does not contain board-only computer settings, so it did not present hidden inventory.");
   c.add("targets-and-worker-preserved", s.savedTargetsUnchanged === true && s.workerUnchanged === true,
-    "No incompatible target selection, target mutation or silent harness/account substitution occurs.");
+    "No target selection, target mutation, probe record or silent harness/account substitution occurs.");
   c.add("single-source-turn-only", o.tasks.length === 0 && o.runs.length === 1 && o.runs[0]!.status === "succeeded",
     "Exactly the declared ordinary chat turn occurs, with no work task or remote qualification.");
   return c.result();
@@ -48,50 +63,58 @@ export const runtimeCapabilityChoiceCase = runtimeDefinition(scenarioId, async c
   const api = ctx.input.api, company = `/api/companies/${ctx.input.fixtures.company.id}`;
   const originalSettings = await api.get<{ enableEnvironments: boolean }>("/api/instance/settings/experimental");
   await api.patch("/api/instance/settings/experimental", { enableEnvironments: true });
-  let unsupported: Environment | undefined;
+  let unqualified: Environment | undefined;
   try {
     const inventory = await api.get<Environment[]>(`${company}/environments`);
     const supported = inventory.find(row => row.id === ctx.input.fixtures.environment.id && row.driver === "local");
     if (!supported) throw new Error("Owned local fixture target is not in the public company-authorized inventory");
-    unsupported = await api.post<Environment>(`${company}/environments`, { name: `Unsupported capability control ${ctx.input.nonce}`, driver: "sandbox", config: { provider: "fake", image: "unsupported-control", reuseLease: false } });
+    // The product cannot save a computer that is advertised as unable to run Claude Code
+    // (the built-in fake sandbox is reserved for probes), so the contrast is support versus health.
+    unqualified = await api.post<Environment>(`${company}/environments`, { name: `Build server ${ctx.input.nonce}`, driver: "ssh", config: unqualifiedSshConfig(ctx.input.nonce) });
     const capabilities = await api.get<EnvironmentCapabilities>(`${company}/environments/capabilities`);
-    if (!advertisedClaudeTargets(capabilities, supported, unsupported)) throw new Error("Production metadata does not establish the requested advertised capability contrast");
+    if (!advertisedClaudeTargets(capabilities, supported, unqualified)) throw new Error("Production metadata does not advertise Claude Code on both owned computers");
     const worker = await runtimeWorker(ctx);
-    return { supportedTarget: supported, unsupportedTarget: unsupported, capabilities, worker, originalEnvironmentsEnabled: originalSettings.enableEnvironments,
+    return { supportedTarget: supported, unqualifiedTarget: unqualified, capabilities, worker, originalEnvironmentsEnabled: originalSettings.enableEnvironments,
+      companyPrefix: ctx.input.fixtures.company.issuePrefix,
       agentInventoryStatus: await probeAgentPermission(ctx, `${company}/environments/capabilities`, "GET"),
-      agentProbeStatus: await probeAgentPermission(ctx, `/api/environments/${supported.id}/probe`, "POST", {}) };
+      agentProbeStatus: await probeAgentPermission(ctx, `/api/environments/${unqualified.id}/probe`, "POST", {}) };
   } catch (error) {
-    if (unsupported) await api.delete(`/api/environments/${unsupported.id}`, { allowNotFound: true });
+    if (unqualified) await api.delete(`/api/environments/${unqualified.id}`, { allowNotFound: true });
     await api.patch("/api/instance/settings/experimental", { enableEnvironments: originalSettings.enableEnvironments });
     await cleanupIdentityProbes(ctx); throw error;
   }
-}, gradeCapabilityChoice, { authentication: "human-session", attemptTimeoutMs: 900000,
-  sourceFiles: ["chat-catalog-domains/runtime-capability-choice.ts", "chat-catalog-domains/runtime-computers-ssh.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/execution-environments.yaml", "../../packages/shared/src/environment-support.ts", "../../server/src/routes/environments.ts", "../../ui/src/pages/CompanyEnvironments.tsx"],
-  async act(ctx, state) {
-    await waitForReply(ctx);
-    const source = ctx.input.page.url(), company = `/api/companies/${ctx.input.fixtures.company.id}`;
-    const response = ctx.input.page.waitForResponse(row => row.request().method() === "GET" && new URL(row.url()).pathname === `${company}/environments/capabilities`);
-    await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/company/settings/instance/environments`, { waitUntil: "domcontentloaded" });
-    const actualResponse = await response;
-    state.viewCapabilitiesMatched = actualResponse.ok() && JSON.stringify(await actualResponse.json()) === JSON.stringify(state.capabilities);
-    const view = ctx.input.page.getByTestId("instance-settings-environments-section");
-    await expect(view).toBeVisible();
-    for (const target of [state.supportedTarget, state.unsupportedTarget] as Environment[]) await expect(view.getByText(target.name, { exact: true })).toBeVisible();
-    state.operatorViewVisible = true;
-    await ctx.input.evidence("runtime-advertised-capabilities.json", { supportIsAdvertisedOnly: true, runtimeHealth: "unverified", supportedTargetId: (state.supportedTarget as Environment).id, unsupportedTargetId: (state.unsupportedTarget as Environment).id, capabilities: state.capabilities });
-    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
-  },
+}, gradeCapabilityChoice, { authentication: "human-session", attemptTimeoutMs: 900000, oracleVersion,
+  sourceFiles: ["chat-catalog-domains/runtime-capability-choice.ts", "chat-catalog-domains/runtime-computers-ssh.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/execution-environments.yaml", "../../packages/shared/src/environment-support.ts", "../../server/src/routes/environments.ts", "../../server/src/routes/authz.ts", "../../server/src/services/environment-config.ts", "../../ui/src/pages/CompanyEnvironments.tsx"],
+  async act(ctx) { await waitForReply(ctx); },
   async observe(ctx, state) {
-    const api = ctx.input.api, targets = [state.supportedTarget, state.unsupportedTarget] as Environment[];
+    const api = ctx.input.api, targets = [state.supportedTarget, state.unqualifiedTarget] as Environment[];
     const current = await Promise.all(targets.map(target => api.get<Environment>(`/api/environments/${target.id}`)));
-    state.savedTargetsUnchanged = targets.every((target, index) => ["id", "driver", "name", "config"].every(key => JSON.stringify(target[key as keyof Environment]) === JSON.stringify(current[index]![key as keyof Environment])));
+    state.savedTargetsUnchanged = targets.every((target, index) => ["id", "driver", "name", "status", "config", "metadata"].every(key => JSON.stringify(target[key as keyof Environment]) === JSON.stringify(current[index]![key as keyof Environment])));
     const worker = state.worker as RuntimeAgent;
     state.workerUnchanged = unchangedAgent(worker, await api.get<RuntimeAgent>(`/api/agents/${worker.id}`));
     const { observeRuntime } = await import("./runtime-support.js"); return observeRuntime(ctx, state);
   },
-  async verifyResult(ctx) { await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); },
+  /** The person clicks the settings link the reply rendered and sees both computers and the checked capabilities. */
+  async verifyResult(ctx, state, value) {
+    const o = runtimeObservation(value); if (!o) throw new Error("Missing independent runtime result");
+    const targets = [state.supportedTarget, state.unqualifiedTarget] as Environment[];
+    const href = environmentSettingsLink(o, String(state.companyPrefix), targets.map(target => target.id));
+    if (!href) throw new Error("The reply rendered no environment settings link");
+    const source = ctx.input.page.url(), company = `/api/companies/${ctx.input.fixtures.company.id}`;
+    const response = ctx.input.page.waitForResponse(row => row.request().method() === "GET" && new URL(row.url()).pathname === `${company}/environments/capabilities`);
+    await internalResultLink(ctx, o, href).first().click();
+    await ctx.input.page.waitForURL(url => url.pathname.includes("/company/settings/instance/environments"));
+    const actualResponse = await response;
+    expect(actualResponse.ok() && JSON.stringify(await actualResponse.json()) === JSON.stringify(state.capabilities), "The opened view loads the independently checked capabilities").toBe(true);
+    const view = ctx.input.page.getByTestId("instance-settings-environments-section");
+    await expect(view).toBeVisible();
+    for (const target of targets) await expect(view.getByText(target.name, { exact: true })).toBeVisible();
+    await ctx.input.evidence("runtime-advertised-capabilities.json", { supportIsAdvertisedOnly: true, runtimeHealth: "unverified", openedFromReplyLink: href, supportedTargetId: targets[0]!.id, unqualifiedTargetId: targets[1]!.id, capabilities: state.capabilities });
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+    await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible();
+  },
   async cleanup(ctx, state) {
-    try { if (state.unsupportedTarget) await ctx.input.api.delete(`/api/environments/${(state.unsupportedTarget as Environment).id}`, { allowNotFound: true }); }
+    try { if (state.unqualifiedTarget) await ctx.input.api.delete(`/api/environments/${(state.unqualifiedTarget as Environment).id}`, { allowNotFound: true }); }
     finally { if (typeof state.originalEnvironmentsEnabled === "boolean") await ctx.input.api.patch("/api/instance/settings/experimental", { enableEnvironments: state.originalEnvironmentsEnabled }); await cleanupIdentityProbes(ctx); }
   },
 });

tests/runner-e2e/chat-full-catalog-testops-manifest.json

@@ -1083,6 +1083,12 @@
           "previous": "The agent identifies the exact last revision and restores the previous contents through the versioned instruction path. History retains both the unwanted edit and the restoration, and chat describes what was restored without claiming to rewind tasks already executed under the old instructions.",
           "current": "The agent checks what Paperclip actually retains instead of guessing. With no saved earlier version, it leaves Morgan's instructions unchanged, does not reconstruct or invent a previous revision, says plainly that no instruction history exists to undo from, and asks the person for the text they want (or which rule to remove). If a real retained copy of the earlier text existed, restoring exactly that would be acceptable instead."
         },
+        {
+          "key": "execution-environments.choose-capable-target",
+          "field": "originalContext",
+          "previous": "One target supports the requested coding harness and another does not; detailed environment inventory is board-only.",
+          "current": "Two saved computers are advertised as able to run the requested coding harness, and neither has been tested for it; one is an SSH computer that cannot be reached. Paperclip cannot save a computer that is advertised as unable to run that harness. Environment inventory, capabilities and tests are board-only."
+        },
         {
           "key": "access.manage-membership",
           "field": "originalQuery",
@@ -7666,7 +7672,7 @@
       "installedSource": null,
       "originalQuery": "Which computer can run Morgan with Claude Code?",
       "expectedOutcome": "The answer uses available admitted runtime context or directs the person to the environment capability view when agent reads are restricted. It distinguishes advertised capability from runtime health and avoids selecting an incompatible target or claiming hidden inventory was inspected.",
-      "originalContext": "One target supports the requested coding harness and another does not; detailed environment inventory is board-only.",
+      "originalContext": "Two saved computers are advertised as able to run the requested coding harness, and neither has been tested for it; one is an SSH computer that cannot be reached. Paperclip cannot save a computer that is advertised as unable to run that harness. Environment inventory, capabilities and tests are board-only.",
       "capabilities": [
         "execution-environments/capabilities",
         "runs-adapters/availability"

tests/runner-e2e/chat-full-catalog-testops-manifest.test.mjs

@@ -375,6 +375,11 @@ test('repair source metadata binds the corrected candidate and explicit changed
     "agents.restore-instructions",
     "expectedOutcome"
   ],
+  // The fake sandbox control can never be saved; the case now contrasts advertised support with health.
+  [
+    "execution-environments.choose-capable-target",
+    "originalContext"
+  ],
   // Retargeted to read-and-route: member permission edits are board-only.
   [
     "access.manage-membership",

tests/runner-e2e/chat-scenarios/execution-environments.yaml

@@ -22,8 +22,10 @@ scenarios:
       - execution-environments/capabilities
       - runs-adapters/availability
     context: >-
-      One target supports the requested coding harness and another does not; detailed
-      environment inventory is board-only.
+      Two saved computers are advertised as able to run the requested coding harness, and
+      neither has been tested for it; one is an SSH computer that cannot be reached. Paperclip
+      cannot save a computer that is advertised as unable to run that harness. Environment
+      inventory, capabilities and tests are board-only.
     query: >-
       Which computer can run Morgan with Claude Code?
     expected_outcome: >-