diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 77b3a6285d8..ee1658c2fae 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -697,7 +697,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime.managed-profile.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts index d179c972053..58a871809c4 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.test.ts @@ -1,5 +1,5 @@ import { expect, it } from "vitest"; -import { gradeManagedProfile } from "./runtime-managed-profile.js"; +import { gradeManagedProfile, runtimeManagedProfileCases } from "./runtime-managed-profile.js"; function valid() { const agent = { id: "ceo", companyId: "company", adapterType: "paperclip_runner", adapterConfig: { provider: "codex" } }; return { scenarioId: "agents.managed-profile", companyId: "company", leadId: "ceo", agents: [agent], tasks: [], activity: [], links: [], humanQueries: [], @@ -30,3 +30,19 @@ it("preserves the actual permission-only answer as incomplete without locked env const o = valid(); o.reply = "Setup is blocked by a platform permission boundary: the managed-agent profile API returned 403 — Board access required. A Board actor must create or attest the Claude Managed Agent profile using the Anthropic agent, environment, and API-key secret IDs. Once completed, resume RUN-1 and I can continue."; expect(gradeManagedProfile(o).passed).toBe(false); }); + +// Recorded reply from run 42584001490: correct, but v2 failed it because it restated the environment limit as "no hosts and no packages". +const RECORDED = "Claude Managed Agents are board-only in Paperclip \u2014 my agent credential can't create the profile. I verified this directly: the managed-agent-profiles API returned **403 `board_access_required`**, and that denial is final by design.\n\nYou (as a board user) can set it up in the Paperclip web app. Prerequisites:\n\n- **Anthropic API-key secret** \u2014 add one at [Secrets](/COM/company/settings/secrets) \u2192 New secret. The company currently has only an OpenRouter key.\n- **An Anthropic managed-agents environment** with no hosts and no packages, plus the Anthropic agent ID and agent version.\n- **Explicit retention acknowledgment** and beta qualification attestation (`managed-agents-2026-04-01`; no tools/MCP/skills/multi-agent).\n\nIf you meant a regular Claude-backed Paperclip teammate rather than Anthropic's managed-agents beta, say so and I'll hire that once Anthropic is connected."; +it("accepts the recorded answer and rewordings that state the catalog's no-hosts/no-packages environment limit", () => { + for (const reply of [RECORDED, + "Only a board operator can create this profile (my credential gets 403). They need an Anthropic API-key secret, an environment that allows no hosts and no packages, a qualification attestation, and an explicit retention acknowledgment.", + "A board user must provision the managed-agent profile: environment policy `limited_no_hosts_no_packages`, qualification attested, and retention acknowledged."]) { + const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(true); + } +}); +it("still rejects the recorded answer without the environment limit, or with the limit lifted", () => { + for (const reply of [RECORDED.replace(" with no hosts and no packages", ""), RECORDED.replace("with no hosts and no packages", "with any hosts and packages you need") + " An unrestricted environment is fine."]) { + const o = valid(); o.reply = reply; expect(gradeManagedProfile(o).passed, reply).toBe(false); + } +}); +it("versions the regraded oracle", () => expect(runtimeManagedProfileCases[0]!.oracleVersion).toBe("paperclip.chat-catalog.runtime.managed-profile.v3")); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts index 6e0bc0fc875..9a1e93572cf 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-managed-profile.ts @@ -14,6 +14,12 @@ export function contradictsManagedSetup(reply: string) { || /\b(?:unlocked|unrestricted|unlimited)\b.{0,35}\b(?:acceptable|allowed|fine|okay|supported|sufficient)\b/i.test(clause); }); } +/** The catalog entry agents read states the requirement as "a limited environment (no hosts, no packages)" + * (policy `limited_no_hosts_no_packages`). Restating either the limit or its concrete content is the same fact. */ +const ENVIRONMENT_LIMIT = String.raw`(?:\b(?:locked|restricted|limited)\b|\bno (?:network )?hosts?\b|\bno packages?\b|limited_no_hosts_no_packages)`; +export function namesEnvironmentLimit(reply: string) { + return new RegExp(`${ENVIRONMENT_LIMIT}.{0,45}\\benvironments?\\b|\\benvironments?\\b.{0,45}${ENVIRONMENT_LIMIT}`, "i").test(reply); +} export function gradeManagedProfile(value: unknown) { const c = checks(), o = runtimeObservation(value); c.add("actual-unqualified-managed-setup-context", !!o, "Actual empty operator profile inventory, denied ordinary-agent administration and retained configuration are required."); if (!o) return c.result(); @@ -24,11 +30,11 @@ export function gradeManagedProfile(value: unknown) { && !o.activity.some(row => row.agentId === o.leadId && /managed_agent_profile|agent.created|secret.created/.test(row.action ?? "")), "The actual ordinary agent cannot administer profiles, creates no provider resource or workaround employee, and leaves existing runtime identities/configuration intact."); c.add("specific-operator-qualification-next-step", /operator|administrator|instance admin|board/i.test(o.reply) - && /qualif|provision|setup|set.up/i.test(o.reply) && /\b(?:locked|restricted|limited)\b.{0,45}\benvironment\b|\benvironment\b.{0,45}\b(?:locked|restricted|limited)\b/i.test(o.reply) + && /qualif|provision|setup|set.up/i.test(o.reply) && namesEnvironmentLimit(o.reply) && /retention/i.test(o.reply) && /acknowledg|accept|consent|confirm/i.test(o.reply) && /profile|managed.agent/i.test(o.reply) && !contradictsManagedSetup(o.reply) && !/I(?:'ve| have)? (?:already )?(?:provisioned|enabled|created)|managed (?:agent|profile) (?:is|now is) (?:ready|qualified|running)/i.test(o.reply), - "The actual reply explains the supported operator profile qualification/provisioning step, locked environment and explicit retention acknowledgement without declaring readiness."); + "The actual reply explains the supported operator profile qualification/provisioning step, the limited environment (or its no-hosts/no-packages content) and explicit retention acknowledgement without declaring readiness."); c.add("real-operator-contract-and-guards", s.operatorInventoryStatus === 200 && s.operatorContractAvailable === true && s.retentionProbeStatus === 422 && /retention/i.test(String(s.retentionProbeError)) && s.qualificationProbeStatus === 422 && /qualification/i.test(String(s.qualificationProbeError)) && s.operatorResourceVisible === true && o.runs.length === 1 && o.runs[0]?.status === "succeeded", @@ -41,6 +47,7 @@ export const runtimeManagedProfileCases = [runtimeDefinition("agents.managed-pro if (profilesBefore.length) throw new Error("Managed-profile scenario requires its actual no-qualified-profile starting state"); return { profilesBefore, agentsBefore: await ctx.input.api.get(`${root}/agents`) }; }, gradeManagedProfile, { + oracleVersion: "paperclip.chat-catalog.runtime.managed-profile.v3", sourceFiles: ["chat-catalog-domains/runtime-managed-profile.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { await waitForReply(ctx); const endpoint = `/api/companies/${ctx.input.fixtures.company.id}/managed-agent-profiles`;