test(evals): accept the catalog's no-hosts/no-packages environment limit in managed-profile

The reply restated the API catalog's "limited environment (no hosts, no
packages)" as "environment with no hosts and no packages" and failed only on a
locked|restricted|limited regex. Accept the limit or its concrete content.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-descriptors.json

@@ -697,7 +697,7 @@
       "chat-catalog-domains/identity-support.ts",
       "chat-scenarios/agents.yaml"
     ],
-    "oracleVersion": "paperclip.chat-catalog.runtime.v2",
+    "oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v3",
     "coverageGaps": [],
     "expectedRunCount": 1,
     "turnTimeoutMs": 120000,

tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts

@@ -1,5 +1,5 @@
 import { expect, it } from "vitest";
-import { gradeManagedProfile } from "./runtime-managed-profile.js";
+import { gradeManagedProfile, runtimeManagedProfileCases } from "./runtime-managed-profile.js";
 function valid() {
   const agent = { id: "ceo", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } };
   return { scenarioId: "agents.managed-profile", companyId: "company", leadId: "ceo", agents: [agent], tasks: [], activity: [], links: [], humanQueries: [],
@@ -30,3 +30,19 @@ it("preserves the actual permission-only answer as incomplete without locked env
   const o = valid(); o.reply = "Setup is blocked by a platform permission boundary: the managed-agent profile API returned 403 — Board access required. A Board actor must create or attest the Claude Managed Agent profile using the Anthropic agent, environment, and API-key secret IDs. Once completed, resume RUN-1 and I can continue.";
   expect(gradeManagedProfile(o).passed).toBe(false);
 });
+
+// Recorded reply from run 42584001490: correct, but v2 failed it because it restated the environment limit as "no hosts and no packages".
+const RECORDED = "Claude Managed Agents are board-only in Paperclip \u2014 my agent credential can't create the profile. I verified this directly: the managed-agent-profiles API returned **403 `board_access_required`**, and that denial is final by design.\n\nYou (as a board user) can set it up in the Paperclip web app. Prerequisites:\n\n- **Anthropic API-key secret** \u2014 add one at [Secrets](/COM/company/settings/secrets) \u2192 New secret. The company currently has only an OpenRouter key.\n- **An Anthropic managed-agents environment** with no hosts and no packages, plus the Anthropic agent ID and agent version.\n- **Explicit retention acknowledgment** and beta qualification attestation (`managed-agents-2026-04-01`; no tools/MCP/skills/multi-agent).\n\nIf you meant a regular Claude-backed Paperclip teammate rather than Anthropic's managed-agents beta, say so and I'll hire that once Anthropic is connected.";
+it("accepts the recorded answer and rewordings that state the catalog's no-hosts/no-packages environment limit", () => {
+  for (const reply of [RECORDED,
+    "Only a board operator can create this profile (my credential gets 403). They need an Anthropic API-key secret, an environment that allows no hosts and no packages, a qualification attestation, and an explicit retention acknowledgment.",
+    "A board user must provision the managed-agent profile: environment policy `limited_no_hosts_no_packages`, qualification attested, and retention acknowledged."]) {
+    const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(true);
+  }
+});
+it("still rejects the recorded answer without the environment limit, or with the limit lifted", () => {
+  for (const reply of [RECORDED.replace(" with no hosts and no packages", ""), RECORDED.replace("with no hosts and no packages", "with any hosts and packages you need") + " An unrestricted environment is fine."]) {
+    const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(false);
+  }
+});
+it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v3"));

tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts

@@ -14,6 +14,12 @@ export function contradictsManagedSetup(reply: string) {
       || /\b(?:unlocked|unrestricted|unlimited)\b.{0,35}\b(?:acceptable|allowed|fine|okay|supported|sufficient)\b/i.test(clause);
   });
 }
+/** The catalog entry agents read states the requirement as "a limited environment (no hosts, no packages)"
+ * (policy `limited_no_hosts_no_packages`). Restating either the limit or its concrete content is the same fact. */
+const ENVIRONMENT_LIMIT = String.raw`(?:\b(?:locked|restricted|limited)\b|\bno (?:network )?hosts?\b|\bno packages?\b|limited_no_hosts_no_packages)`;
+export function namesEnvironmentLimit(reply: string) {
+  return new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply);
+}
 export function gradeManagedProfile(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
   c.add("actual-unqualified-managed-setup-context", !!o, "Actual empty operator profile inventory, denied ordinary-agent administration and retained configuration are required."); if (!o) return c.result();
@@ -24,11 +30,11 @@ export function gradeManagedProfile(value: unknown) {
     && !o.activity.some(row => row.agentId === o.leadId && /managed_agent_profile|agent.created|secret.created/.test(row.action ?? "")),
   "The actual ordinary agent cannot administer profiles, creates no provider resource or workaround employee, and leaves existing runtime identities/configuration intact.");
   c.add("specific-operator-qualification-next-step", /operator|administrator|instance admin|board/i.test(o.reply)
-    && /qualif|provision|setup|set.up/i.test(o.reply) && /\b(?:locked|restricted|limited)\b.{0,45}\benvironment\b|\benvironment\b.{0,45}\b(?:locked|restricted|limited)\b/i.test(o.reply)
+    && /qualif|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply)
     && /retention/i.test(o.reply) && /acknowledg|accept|consent|confirm/i.test(o.reply)
     && /profile|managed.agent/i.test(o.reply) && !contradictsManagedSetup(o.reply)
     && !/I(?:'ve| have)? (?:already )?(?:provisioned|enabled|created)|managed (?:agent|profile) (?:is|now is) (?:ready|qualified|running)/i.test(o.reply),
-  "The actual reply explains the supported operator profile qualification/provisioning step, locked environment and explicit retention acknowledgement without declaring readiness.");
+  "The actual reply explains the supported operator profile qualification/provisioning step, the limited environment (or its no-hosts/no-packages content) and explicit retention acknowledgement without declaring readiness.");
   c.add("real-operator-contract-and-guards", s.operatorInventoryStatus === 200 && s.operatorContractAvailable === true
     && s.retentionProbeStatus === 422 && /retention/i.test(String(s.retentionProbeError)) && s.qualificationProbeStatus === 422 && /qualification/i.test(String(s.qualificationProbeError))
     && s.operatorResourceVisible === true && o.runs.length === 1 && o.runs[0]?.status === "succeeded",
@@ -41,6 +47,7 @@ export const runtimeManagedProfileCases = [runtimeDefinition("agents.managed-pro
   if (profilesBefore.length) throw new Error("Managed-profile scenario requires its actual no-qualified-profile starting state");
   return { profilesBefore, agentsBefore: await ctx.input.api.get<RuntimeAgent[]>(`${root}/agents`) };
 }, gradeManagedProfile, {
+  oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v3",
   sourceFiles: ["chat-catalog-domains/runtime-managed-profile.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"],
   async act(ctx, state) {
     await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/managed-agent-profiles`;