From a2921fe079fd7b3ebdb27ad11f23d099deaeb9fa Mon Sep 17 00:00:00 2001 From: Dotta Date: Sun, 8 Nov 2026 05:52:48 -0600 Subject: [PATCH] test(evals): grade conference-room by clicking the reply's own Conference Room link The old grader opened //board-chat by typed URL when the reply had no link, so the surface checks passed for a reply that never pointed there. It also graded the reply's scope with keyword regexes, which reject valid wording and pass invented parity phrased differently. Now the scripted human finds the link as rendered in the thread and clicks it; no link fails reply-links-conference-room. The scope wording is not graded by keywords; the passing reply is read to confirm it. Bump the oracle to runtime-conference.v3 in code and descriptor. Co-Authored-By: Claude Opus 5.5 --- .../runner-e2e/chat-catalog-descriptors.json | 2 +- .../runtime-conference.test.ts | 20 ++++++++----- .../runtime-conference.ts | 30 +++++++++++++------ 3 files changed, 34 insertions(+), 18 deletions(-) diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index dd586de7b22..2d4ee0c2c36 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -301,7 +301,7 @@ "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime.v2", + "oracleVersion": "paperclip.chat-catalog.runtime-conference.v3", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts index 317588fa072..3345f616ac1 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.test.ts @@ -2,12 +2,14 @@ import { it, expect } from "vitest"; import { gradeConference } from "./runtime-conference.js"; function valid() { return { scenarioId: "agent-chat.conference-room", companyId: "company", leadId: "ceo", reply: "Open the Conference Room for the separate board-level discussion.", - state: { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceVisible: true, + state: { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceLinkClicked: true, conferenceVisible: true, openedCompanyPrefix: "ACORN", companyPrefix: "ACORN", noBrokenRoute: true, runsBeforeOpen: 1, runsAfterOpen: 1, - standingIssueBefore: "chat", standingIssueAfter: "chat" }, agents: [], tasks: [], links: [], activity: [], runs: [], humanQueries: [] }; + standingIssueBefore: "chat", standingIssueAfter: "chat" } as Record, agents: [], tasks: [], links: ["/ACORN/board-chat"], activity: [], runs: [], humanQueries: [] }; } it("accepts the real gated board surface in company context", () => expect(gradeConference(valid()).passed).toBe(true)); -it("accepts natural Conference Room navigation guidance", () => { const o = valid(); o.reply = "For a separate board discussion, choose Conference Room in this company. It is a distinct discussion surface."; expect(gradeConference(o).passed).toBe(true); }); +it("does not depend on how the reply words the surface", () => { + const o = valid(); o.reply = "Sure, the launch talk can happen over there: [Conference Room](/ACORN/board-chat). A different assistant answers in that room and it starts fresh."; expect(gradeConference(o).passed).toBe(true); +}); it("rejects missing conference evidence", () => expect(gradeConference(undefined).passed).toBe(false)); it("rejects wrong deployment qualification", () => { const o = valid(); o.state.deploymentMode = "authenticated"; expect(gradeConference(o).passed).toBe(false); @@ -15,10 +17,12 @@ it("rejects wrong deployment qualification", () => { it("rejects navigation to another company", () => { const o = valid(); o.state.openedCompanyPrefix = "BIRCH"; expect(gradeConference(o).passed).toBe(false); }); -it("rejects invented standing-chat history parity", () => { - const o = valid(); o.reply += " It uses the same Agent Chat history."; expect(gradeConference(o).passed).toBe(false); +it("rejects hidden execution started by opening the surface", () => { + const o = valid(); o.state.runsAfterOpen = 2; expect(gradeConference(o).passed).toBe(false); }); - -it("preserves failure when the actual model only asks launch questions and supplies no board navigation or scope", () => { - const o = valid(); o.reply = ""; expect(gradeConference(o).passed).toBe(false); +it("preserves failure when the actual model only asks launch questions and renders no Conference Room link", () => { + const o = valid(); o.reply = "Ready. What are we launching, and what is the target date?"; o.links = []; + o.state = { deploymentMode: "local_trusted", conferenceEnabled: true, agentChatEnabled: true, conferenceLinkClicked: false, + companyPrefix: "ACORN", runsBeforeOpen: 1, standingIssueBefore: "chat" }; + expect(gradeConference(o).passed).toBe(false); }); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts index 17f3c59af89..54ac7648125 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-conference.ts @@ -3,16 +3,22 @@ import { expect } from "@playwright/test"; import { checks } from "./internal-support.js"; import { waitForReply } from "./identity-support.js"; import { runtimeDefinition, runtimeObservation, observeRuntime } from "./runtime-support.js"; +/** + * The reply must send the person to the real Conference Room: the scripted human clicks the + * link the reply rendered (no typed-URL fallback) and must land on the board composer in the + * same company. How the reply words the surface's scope is free-form and is not graded by + * keywords; read the passing reply to confirm it describes a separate board surface without + * promising this chat's history or execution. + */ export function gradeConference(value: unknown) { const c = checks(); const o = runtimeObservation(value); c.add("independent-conference-evidence", !!o, "Actual deployment, gate, company route and rendered board surface are required."); if (!o) return c.result(); c.add("supported-local-enabled-context", o.state.deploymentMode === "local_trusted" && o.state.conferenceEnabled === true && o.state.agentChatEnabled === true, "Both actual gates are enabled in the supported local-trusted deployment."); + c.add("reply-links-conference-room", o.state.conferenceLinkClicked === true, + "The reply rendered a link to this company's Conference Room and the person followed it."); c.add("real-company-conference-surface", o.state.conferenceVisible === true && o.state.openedCompanyPrefix === o.state.companyPrefix - && o.state.noBrokenRoute === true, "The reply opens the real Conference Room composer within the original company."); - c.add("separate-board-scope-described", /conference|board/i.test(o.reply) && /separate|different|board.level|board discussion|board.assistant/i.test(o.reply) - && !/(?:same|shared|identical) (?:standing.chat|agent.chat) (?:history|execution)|(?:full|identical) (?:execution|agent.chat) parity/i.test(o.reply), - "The answer describes the separate board scope without inventing ordinary-chat parity."); + && o.state.noBrokenRoute === true, "The reply's link opens the real Conference Room composer within the original company."); c.add("navigation-has-no-provider-action", o.state.runsBeforeOpen === 1 && o.state.runsAfterOpen === 1 && o.state.standingIssueBefore === o.state.standingIssueAfter, "Opening the board surface preserves the ordinary source and starts no hidden execution."); return c.result(); @@ -25,16 +31,22 @@ export const runtimeConferenceCases = [runtimeDefinition("agent-chat.conference- return { deploymentMode: health.deploymentMode, conferenceEnabled: settings.enableConferenceRoomChat, agentChatEnabled: settings.enableAgentChat, companyPrefix: ctx.input.fixtures.company.issuePrefix }; }, gradeConference, { + oracleVersion: "paperclip.chat-catalog.runtime-conference.v3", sourceFiles: ["chat-catalog-domains/runtime-conference.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agent-chat.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); state.runsBeforeOpen = (await ctx.allRuns()).length; state.standingIssueBefore = ctx.issue()?.id; const o = await observeRuntime(ctx, state); - const href = o.links.find(link => { const url = new URL(link, ctx.input.api.baseURL); - return url.origin === ctx.input.api.baseURL && url.pathname === `/${state.companyPrefix}/board-chat`; }); - + // Find the link as rendered in the thread: the UI may rewrite a reply's href. + const anchors = ctx.input.page.locator('[id^="comment-"] a[href]'); + const rendered = await anchors.evaluateAll(nodes => nodes.map(node => (node as HTMLAnchorElement).href)); + const origin = new URL(ctx.input.page.url()).origin; + const index = rendered.findIndex(href => { try { const url = new URL(href); + return url.origin === origin && url.pathname === `/${state.companyPrefix}/board-chat`; } catch { return false; } }); + state.conferenceLinkClicked = index >= 0; + await ctx.input.evidence("runtime-conference-links.json", { companyId: o.companyId, rendered, replyLinks: o.links }); + if (index < 0) return; const source = ctx.input.page.url(); - if (href) await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(href)}]`).first().click(); - else await ctx.input.page.goto(`/${state.companyPrefix}/board-chat`, { waitUntil: "domcontentloaded" }); + await anchors.nth(index).click(); await expect(ctx.input.page.getByPlaceholder("Ask anything about your organization...")).toBeVisible(); await expect(ctx.input.page.getByText("Conference Room", { exact: true }).first()).toBeVisible(); state.conferenceVisible = true;