diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index dd586de7b22..2d4ee0c2c36 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -301,7 +301,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-conference.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts index 317588fa072..3345f616ac1 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts @@ -2,12 +2,14 @@ import { it, expect } from "vitest"; import { gradeConference } from "./runtime-conference.js"; function valid() { return { scenarioId: "agent-chat.conference-room", companyId: "company", leadId: "ceo", reply: "Open the Conference Room for the separate board-level discussion.", - state: { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceVisible: true, + state: { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceLinkClicked: true, conferenceVisible: true, openedCompanyPrefix: "ACORN", companyPrefix: "ACORN", noBrokenRoute: true, runsBeforeOpen: 1, runsAfterOpen: 1, - standingIssueBefore: "chat", standingIssueAfter: "chat" }, agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [] }; + standingIssueBefore: "chat", standingIssueAfter: "chat" } as Record, agents: [], tasks: [], links: ["/ACORN/board-chat"], activity: [], runs: [], humanQueries: [] }; } it("accepts the real gated board surface in company context", () => expect(gradeConference(valid()).passed).toBe(true)); -it("accepts natural Conference Room navigation guidance", () => { const o = valid(); o.reply = "For a separate board discussion, choose Conference Room in this company. It is a distinct discussion surface."; expect(gradeConference(o).passed).toBe(true); }); +it("does not depend on how the reply words the surface", () => { + const o = valid(); o.reply = "Sure, the launch talk can happen over there: [Conference Room](/ACORN/board-chat). A different assistant answers in that room and it starts fresh."; expect(gradeConference(o).passed).toBe(true); +}); it("rejects missing conference evidence", () => expect(gradeConference(undefined).passed).toBe(false)); it("rejects wrong deployment qualification", () => { const o = valid(); o.state.deploymentMode = "authenticated"; expect(gradeConference(o).passed).toBe(false); @@ -15,10 +17,12 @@ it("rejects wrong deployment qualification", () => { it("rejects navigation to another company", () => { const o = valid(); o.state.openedCompanyPrefix = "BIRCH"; expect(gradeConference(o).passed).toBe(false); }); -it("rejects invented standing-chat history parity", () => { - const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false); +it("rejects hidden execution started by opening the surface", () => { + const o = valid(); o.state.runsAfterOpen = 2; expect(gradeConference(o).passed).toBe(false); }); - -it("preserves failure when the actual model only asks launch questions and supplies no board navigation or scope", () => { - const o = valid(); o.reply = ""; expect(gradeConference(o).passed).toBe(false); +it("preserves failure when the actual model only asks launch questions and renders no Conference Room link", () => { + const o = valid(); o.reply = "Ready. What are we launching, and what is the target date?"; o.links = []; + o.state = { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceLinkClicked: false, + companyPrefix: "ACORN", runsBeforeOpen: 1, standingIssueBefore: "chat" }; + expect(gradeConference(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts index 17f3c59af89..54ac7648125 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts @@ -3,16 +3,22 @@ import { expect } from "@playwright/test"; import { checks } from "./internal-support.js"; import { waitForReply } from "./identity-support.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +/** + * The reply must send the person to the real Conference Room: the scripted human clicks the + * link the reply rendered (no typed-URL fallback) and must land on the board composer in the + * same company. How the reply words the surface's scope is free-form and is not graded by + * keywords; read the passing reply to confirm it describes a separate board surface without + * promising this chat's history or execution. + */ export function gradeConference(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("independent-conference-evidence", !!o, "Actual deployment, gate, company route and rendered board surface are required."); if (!o) return c.result(); c.add("supported-local-enabled-context", o.state.deploymentMode === "local_trusted" && o.state.conferenceEnabled === true && o.state.agentChatEnabled === true, "Both actual gates are enabled in the supported local-trusted deployment."); + c.add("reply-links-conference-room", o.state.conferenceLinkClicked === true, + "The reply rendered a link to this company's Conference Room and the person followed it."); c.add("real-company-conference-surface", o.state.conferenceVisible === true && o.state.openedCompanyPrefix === o.state.companyPrefix - && o.state.noBrokenRoute === true, "The reply opens the real Conference Room composer within the original company."); - c.add("separate-board-scope-described", /conference|board/i.test(o.reply) && /separate|different|board.level|board discussion|board.assistant/i.test(o.reply) - && !/(?:same|shared|identical) (?:standing.chat|agent.chat) (?:history|execution)|(?:full|identical) (?:execution|agent.chat) parity/i.test(o.reply), - "The answer describes the separate board scope without inventing ordinary-chat parity."); + && o.state.noBrokenRoute === true, "The reply's link opens the real Conference Room composer within the original company."); c.add("navigation-has-no-provider-action", o.state.runsBeforeOpen === 1 && o.state.runsAfterOpen === 1 && o.state.standingIssueBefore === o.state.standingIssueAfter, "Opening the board surface preserves the ordinary source and starts no hidden execution."); return c.result(); @@ -25,16 +31,22 @@ export const runtimeConferenceCases = [runtimeDefinition("agent-chat.conference- return { deploymentMode: health.deploymentMode, conferenceEnabled: settings.enableConferenceRoomChat, agentChatEnabled: settings.enableAgentChat, companyPrefix: ctx.input.fixtures.company.issuePrefix }; }, gradeConference, { + oracleVersion: "paperclip.chat-catalog.runtime-conference.v3", sourceFiles: ["chat-catalog-domains/runtime-conference.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id; const o = await observeRuntime(ctx, state); - const href = o.links.find(link => { const url = new URL(link, ctx.input.api.baseURL); - return url.origin === ctx.input.api.baseURL && url.pathname === `/${state.companyPrefix}/board-chat`; }); - + // Find the link as rendered in the thread: the UI may rewrite a reply's href. + const anchors = ctx.input.page.locator('[id^="comment-"] a[href]'); + const rendered = await anchors.evaluateAll(nodes => nodes.map(node => (node as HTMLAnchorElement).href)); + const origin = new URL(ctx.input.page.url()).origin; + const index = rendered.findIndex(href => { try { const url = new URL(href); + return url.origin === origin && url.pathname === `/${state.companyPrefix}/board-chat`; } catch { return false; } }); + state.conferenceLinkClicked = index >= 0; + await ctx.input.evidence("runtime-conference-links.json", { companyId: o.companyId, rendered, replyLinks: o.links }); + if (index < 0) return; const source = ctx.input.page.url(); - if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); - else await ctx.input.page.goto(`/${state.companyPrefix}/board-chat`, { waitUntil: "domcontentloaded" }); + await anchors.nth(index).click(); await expect(ctx.input.page.getByPlaceholder("Ask anything about your organization...")).toBeVisible(); await expect(ctx.input.page.getByText("Conference Room", { exact: true }).first()).toBeVisible(); state.conferenceVisible = true;