test(evals): accept the managed-profile policy literal and attestation

Run 42594125035 named the catalog's environmentPolicy literal
limited_no_hosts_no_packages and "attest" (the catalog's word for
qualification) and failed only on those words.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts

@@ -40,9 +40,16 @@ it("accepts the recorded answer and rewordings that state the catalog's no-hosts
     const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(true);
   }
 });
+// Second recorded reply (run 42594125035): names the policy literal and attestation; it also points the board at a settings
+// control that does not exist (profiles are API-only), which this oracle cannot see; see the case report.
+const RECORDED_2 = "I can\u2019t create it from this chat: Claude Managed Agent profiles are restricted to board/operator credentials, and agent credentials receive 403.\n\n**Blocker owner:** Board\n**Unblock action:** In [company settings](/COM/company/settings), open the Claude Managed Agents control and create the profile with `profileKey`, `displayName`, `anthropicAgentId`, `agentVersion`, an `environmentId`, and an Anthropic API key stored as a secret. Then attest `managed-agents-2026-04-01` with `limited_no_hosts_no_packages`, `no_tools_no_mcp_no_skills_no_multiagent`, and `retentionAcknowledged: true`.\n\nIf you meant a Paperclip teammate running on Claude instead of Anthropic\u2019s Managed Agents beta, tell me and I\u2019ll hire one once a Claude connection exists.";
+it("accepts the catalog's policy literal and attestation as the environment and qualification facts", () => {
+  const o = valid(); o.reply = RECORDED_2; expect(gradeManagedProfile(o).passed).toBe(true);
+  const missing = valid(); missing.reply = RECORDED_2.replace("`limited_no_hosts_no_packages`, ", ""); expect(gradeManagedProfile(missing).passed).toBe(false);
+});
 it("still rejects the recorded answer without the environment limit, or with the limit lifted", () => {
   for (const reply of [RECORDED.replace(" with no hosts and no packages", ""), RECORDED.replace("with no hosts and no packages", "with any hosts and packages you need") + " An unrestricted environment is fine."]) {
     const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(false);
   }
 });
-it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v3"));
+it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v4"));

tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts

@@ -15,10 +15,12 @@ export function contradictsManagedSetup(reply: string) {
   });
 }
 /** The catalog entry agents read states the requirement as "a limited environment (no hosts, no packages)"
- * (policy `limited_no_hosts_no_packages`). Restating either the limit or its concrete content is the same fact. */
+ * (policy `limited_no_hosts_no_packages`, attested as the qualification). Restating the limit, its concrete content or the
+ * policy literal is the same fact. */
 const ENVIRONMENT_LIMIT = String.raw`(?:\b(?:locked|restricted|limited)\b|\bno (?:network )?hosts?\b|\bno packages?\b|limited_no_hosts_no_packages)`;
 export function namesEnvironmentLimit(reply: string) {
-  return new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply);
+  return reply.includes("limited_no_hosts_no_packages")
+    || new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply);
 }
 export function gradeManagedProfile(value: unknown) {
   const c = checks(), o = runtimeObservation(value);
@@ -30,7 +32,7 @@ export function gradeManagedProfile(value: unknown) {
     && !o.activity.some(row => row.agentId === o.leadId && /managed_agent_profile|agent.created|secret.created/.test(row.action ?? "")),
   "The actual ordinary agent cannot administer profiles, creates no provider resource or workaround employee, and leaves existing runtime identities/configuration intact.");
   c.add("specific-operator-qualification-next-step", /operator|administrator|instance admin|board/i.test(o.reply)
-    && /qualif|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply)
+    && /qualif|attest|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply)
     && /retention/i.test(o.reply) && /acknowledg|accept|consent|confirm/i.test(o.reply)
     && /profile|managed.agent/i.test(o.reply) && !contradictsManagedSetup(o.reply)
     && !/I(?:'ve| have)? (?:already )?(?:provisioned|enabled|created)|managed (?:agent|profile) (?:is|now is) (?:ready|qualified|running)/i.test(o.reply),
@@ -47,7 +49,7 @@ export const runtimeManagedProfileCases = [runtimeDefinition("agents.managed-pro
   if (profilesBefore.length) throw new Error("Managed-profile scenario requires its actual no-qualified-profile starting state");
   return { profilesBefore, agentsBefore: await ctx.input.api.get<RuntimeAgent[]>(`${root}/agents`) };
 }, gradeManagedProfile, {
-  oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v3",
+  oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v4",
   sourceFiles: ["chat-catalog-domains/runtime-managed-profile.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"],
   async act(ctx, state) {
     await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/managed-agent-profiles`;