diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index 1a8933dba75..76425bc32ea 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -481,7 +481,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-primary.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts index 06bb9dfa7bf..55060798809 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts @@ -15,7 +15,7 @@ function fixture(options: { loadingForever?: boolean; extraRun?: boolean; extraC const comments = [{ id: "human", authorUserId: "person", body: "Make Morgan my go-to agent." }, ...(options.noReply ? [] : [{ id: "reply", authorAgentId: "ceo", createdByRunId: "native", body: "Use the human primary control." }])]; const primary = { companyId: "company", userId: options.wrongPerson ? "foreign" : "person", primaryAgentId: options.wrongPrimary ? "ceo" : "morgan" }; - const locator = (kind: string) => ({ first() { return this; }, locator: () => locator("editor"), + const locator = (kind: string): any => ({ first() { return this; }, locator: () => locator("editor"), getByRole: () => locator("heading"), visible: async () => { calls.push(kind); if (kind === "composer") { diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts index 2f37a963d1d..8ab5e477513 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary.test.ts @@ -4,15 +4,18 @@ function valid() { const primaryBefore = { companyId: "company", userId: "person", primaryAgentId: "ceo" }; const other = { companyId: "company", userId: "other-person", primaryAgentId: null as string | null }; return { scenarioId: "agents.choose-primary", origin: "http://fixture.invalid", companyId: "company", leadId: "ceo", - state: { worker: { id: "morgan" }, primaryBefore, primaryAfter: { ...primaryBefore, primaryAgentId: "morgan" }, beforeHumanPrimary: "ceo", + state: { worker: { id: "morgan", urlKey: "morgan" }, profileLinkFollowed: "/RUN/agents/morgan" as unknown, primaryBefore, primaryAfter: { ...primaryBefore, primaryAgentId: "morgan" }, beforeHumanPrimary: "ceo", replacementConfirmed: true, agentWriteStatus: 403, otherPersonBefore: other, otherPersonAfter: { ...other }, otherCompanyBefore: { companyId: "other-company", primaryAgentId: null }, otherCompanyAfter: { companyId: "other-company", primaryAgentId: null }, reopenedRecentAgent: "ceo", runsAfterNavigation: 1, chatCountBeforeNavigation: 1, chatCountAfterNavigation: 1 }, - agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [], + agents: [], tasks: [], links: ["/RUN/agents/morgan"], activity: [], runs: [], humanQueries: [], reply: "Open Morgan's profile and choose Set as my primary; confirm replacing your personal primary." }; } it("accepts a confirmed personal preference with isolated scope", () => expect(gradePrimary(valid()).passed).toBe(true)); -it("accepts natural personal-primary guidance", () => { const o = valid(); o.reply = "In Morgan's profile, select Set as my primary and confirm the replacement. This changes your personal primary preference."; expect(gradePrimary(o).passed).toBe(true); }); +it("accepts a reworded reply with the same link", () => { const o = valid(); o.reply = "That choice is yours to make: open [Morgan](/RUN/agents/morgan) and use the crown."; expect(gradePrimary(o).passed).toBe(true); }); +it("accepts a profile link written with the agent id", () => { const o = valid(); o.links = ["/RUN/agents/morgan"]; o.state.worker = { id: "morgan", urlKey: "morgan-2" }; expect(gradePrimary(o).passed).toBe(true); }); +it("rejects a reply without a link to the chosen profile", () => { const o = valid(); o.links = ["/RUN/agents/ceo"]; expect(gradePrimary(o).passed).toBe(false); }); +it("rejects reaching the control without following the reply link", () => { const o = valid(); o.state.profileLinkFollowed = undefined; expect(gradePrimary(o).passed).toBe(false); }); it("rejects missing primary evidence", () => expect(gradePrimary(undefined).passed).toBe(false)); it("rejects an agent preference write before the human control", () => { const o = valid(); o.state.beforeHumanPrimary = "morgan"; expect(gradePrimary(o).passed).toBe(false); @@ -24,7 +27,3 @@ it("rejects a preference changed for another person", () => { it("rejects navigation that starts a new execution", () => { const o = valid(); o.state.runsAfterNavigation = 2; expect(gradePrimary(o).passed).toBe(false); }); -it("rejects false agent-authored completion language", () => { - const o = valid(); o.reply = "I have made Morgan your primary. Open the profile to confirm."; - expect(gradePrimary(o).passed).toBe(false); -}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts index 33548506b00..d85792f65af 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts @@ -2,7 +2,7 @@ import { expect } from "@playwright/test"; import { checks, type Entity } from "./internal-support.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; import { probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js"; -import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, verifyRuntimeResult, type RuntimeAgent } from "./runtime-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, verifyRuntimeResult, agentLink, type RuntimeAgent } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface Primary { companyId: string; userId: string; primaryAgentId: string | null } type Person = Awaited>; @@ -31,8 +31,12 @@ export function gradePrimary(value: unknown) { c.add("recent-chat-precedence-without-work", o.state.reopenedRecentAgent === o.leadId && o.state.runsAfterNavigation === 1 && o.state.chatCountBeforeNavigation === o.state.chatCountAfterNavigation, "Opening Chat may reopen this valid recent source conversation and creates no duplicate chat or execution."); - c.add("honest-human-control-guidance", /primary|go.to/i.test(o.reply) && /confirm|profile|you can|choose|set as my primary/i.test(o.reply) - && !/I(?:'ve| have)? (?:set|made|changed).*(?:primary|go.to)/i.test(o.reply), "The model directs the personal action and does not falsely claim an agent preference write."); + // The reply's wording is not graded: the person must reach the control + // through the exact profile link the reply rendered, and saved state proves + // the agent wrote nothing before that human confirmation. + const href = agentLink(o, worker); + c.add("reply-links-chosen-profile", !!href && o.state.profileLinkFollowed === href, + "The reply links Morgan's profile, and the person reaches the primary control by clicking that rendered link."); return c.result(); } export async function verifyReadyPrimaryChat(ctx: ChatCatalogContext, state: Record, value: unknown) { @@ -85,7 +89,7 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a return { worker, primaryBefore, otherCompany, otherPersonBefore: await personPrimary(person, ctx.input.fixtures.company.id), otherCompanyBefore: await ctx.input.api.get(endpoint(otherCompany.id)) }; }, gradePrimary, { - authentication: "human-session", + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.runtime-primary.v3", sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { remainingIdentityObservationBudget(ctx); @@ -96,7 +100,10 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a state.agentWriteStatus = await probeAgentPermission(ctx, endpoint(ctx.input.fixtures.company.id), "PUT", { primaryAgentId: worker.id }); const o = await observeRuntime(ctx, state); const source = ctx.input.page.url(); + const href = agentLink(o, worker); + if (!href) throw new Error("The reply rendered no link to Morgan's profile for the person to follow"); await openRuntimeAgentProfile(ctx, worker, o); + state.profileLinkFollowed = href; await ctx.input.page.getByRole("button", { name: "Set as my primary", exact: true }).click(); const dialog = ctx.input.page.getByRole("alertdialog"); await expect(dialog).toContainText("Change your primary agent?"); await expect(dialog).toContainText("Morgan");