From 0c27bb5c262bf2acf7a5a7687f75fc3a878d551a Mon Sep 17 00:00:00 2001 From: Dotta Date: Mon, 9 Nov 2026 03:52:05 -0600 Subject: [PATCH] test(evals): accept the managed-profile policy literal and attestation Run 42594125035 named the catalog's environmentPolicy literal limited_no_hosts_no_packages and "attest" (the catalog's word for qualification) and failed only on those words. Co-Authored-By: Claude Opus 5.5 --- .../runtime-managed-profile.test.ts | 9 ++++++++- .../chat-catalog-domains/runtime-managed-profile.ts | 10 ++++++---- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts index 58a871809c4..182eb501cd4 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts @@ -40,9 +40,16 @@ it("accepts the recorded answer and rewordings that state the catalog's no-hosts const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(true); } }); +// Second recorded reply (run 42594125035): names the policy literal and attestation; it also points the board at a settings +// control that does not exist (profiles are API-only), which this oracle cannot see; see the case report. +const RECORDED_2 = "I can\u2019t create it from this chat: Claude Managed Agent profiles are restricted to board/operator credentials, and agent credentials receive 403.\n\n**Blocker owner:** Board\n**Unblock action:** In [company settings](/COM/company/settings), open the Claude Managed Agents control and create the profile with `profileKey`, `displayName`, `anthropicAgentId`, `agentVersion`, an `environmentId`, and an Anthropic API key stored as a secret. Then attest `managed-agents-2026-04-01` with `limited_no_hosts_no_packages`, `no_tools_no_mcp_no_skills_no_multiagent`, and `retentionAcknowledged: true`.\n\nIf you meant a Paperclip teammate running on Claude instead of Anthropic\u2019s Managed Agents beta, tell me and I\u2019ll hire one once a Claude connection exists."; +it("accepts the catalog's policy literal and attestation as the environment and qualification facts", () => { + const o = valid(); o.reply = RECORDED_2; expect(gradeManagedProfile(o).passed).toBe(true); + const missing = valid(); missing.reply = RECORDED_2.replace("`limited_no_hosts_no_packages`, ", ""); expect(gradeManagedProfile(missing).passed).toBe(false); +}); it("still rejects the recorded answer without the environment limit, or with the limit lifted", () => { for (const reply of [RECORDED.replace(" with no hosts and no packages", ""), RECORDED.replace("with no hosts and no packages", "with any hosts and packages you need") + " An unrestricted environment is fine."]) { const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(false); } }); -it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v3")); +it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v4")); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts index 9a1e93572cf..0de75053b8f 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts @@ -15,10 +15,12 @@ export function contradictsManagedSetup(reply: string) { }); } /** The catalog entry agents read states the requirement as "a limited environment (no hosts, no packages)" - * (policy `limited_no_hosts_no_packages`). Restating either the limit or its concrete content is the same fact. */ + * (policy `limited_no_hosts_no_packages`, attested as the qualification). Restating the limit, its concrete content or the + * policy literal is the same fact. */ const ENVIRONMENT_LIMIT = String.raw`(?:\b(?:locked|restricted|limited)\b|\bno (?:network )?hosts?\b|\bno packages?\b|limited_no_hosts_no_packages)`; export function namesEnvironmentLimit(reply: string) { - return new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply); + return reply.includes("limited_no_hosts_no_packages") + || new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply); } export function gradeManagedProfile(value: unknown) { const c = checks(), o = runtimeObservation(value); @@ -30,7 +32,7 @@ export function gradeManagedProfile(value: unknown) { && !o.activity.some(row => row.agentId === o.leadId && /managed_agent_profile|agent.created|secret.created/.test(row.action ?? "")), "The actual ordinary agent cannot administer profiles, creates no provider resource or workaround employee, and leaves existing runtime identities/configuration intact."); c.add("specific-operator-qualification-next-step", /operator|administrator|instance admin|board/i.test(o.reply) - && /qualif|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply) + && /qualif|attest|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply) && /retention/i.test(o.reply) && /acknowledg|accept|consent|confirm/i.test(o.reply) && /profile|managed.agent/i.test(o.reply) && !contradictsManagedSetup(o.reply) && !/I(?:'ve| have)? (?:already )?(?:provisioned|enabled|created)|managed (?:agent|profile) (?:is|now is) (?:ready|qualified|running)/i.test(o.reply), @@ -47,7 +49,7 @@ export const runtimeManagedProfileCases = [runtimeDefinition("agents.managed-pro if (profilesBefore.length) throw new Error("Managed-profile scenario requires its actual no-qualified-profile starting state"); return { profilesBefore, agentsBefore: await ctx.input.api.get(`${root}/agents`) }; }, gradeManagedProfile, { - oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v3", + oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v4", sourceFiles: ["chat-catalog-domains/runtime-managed-profile.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/managed-agent-profiles`;