diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts new file mode 100644 index 00000000000..06bb9dfa7bf --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary-ready-chat.test.ts @@ -0,0 +1,55 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +vi.mock("@playwright/test", () => ({ expect: (target: any) => ({ + toBeVisible: async () => { if (!await target.visible()) throw Error("Actual Chat is still loading or its conversation is absent"); }, + toBeEditable: async () => { if (!target.editable()) throw Error("Chat composer is not ready"); }, + toHaveURL: async (matches: (url: URL) => boolean) => { if (!matches(new URL(target.url()))) throw Error("Wrong selected Chat route"); }, +}) })); +import { runtimePrimaryCases } from "./runtime-primary.js"; +afterEach(() => { vi.restoreAllMocks(); }); +function fixture(options: { loadingForever?: boolean; extraRun?: boolean; extraChat?: boolean; wrongPrimary?: boolean; wrongPerson?: boolean; + wrongChat?: boolean; noReply?: boolean; wrongRequest?: boolean; missingRenderedReply?: boolean; readOnly?: boolean; expired?: boolean; lateReady?: boolean } = {}) { + const deadline = Date.now() + (options.expired ? -1 : 30_000); let current = "https://fixture.test/RUN/chats/ceo", ready = false; + const calls: string[] = []; + const source = { id: "source", companyId: "company", conversationAgentId: "ceo", conversationUserId: "person" }; + const comments = [{ id: "human", authorUserId: "person", body: "Make Morgan my go-to agent." }, + ...(options.noReply ? [] : [{ id: "reply", authorAgentId: "ceo", createdByRunId: "native", body: "Use the human primary control." }])]; + const primary = { companyId: "company", userId: options.wrongPerson ? "foreign" : "person", primaryAgentId: options.wrongPrimary ? "ceo" : "morgan" }; + const locator = (kind: string) => ({ first() { return this; }, locator: () => locator("editor"), + visible: async () => { + calls.push(kind); + if (kind === "composer") { + if (options.loadingForever) return false; ready = true; + if (options.lateReady) vi.spyOn(Date, "now").mockReturnValue(deadline + 1); + } + return kind === "heading" || ready && !(kind.includes("comment-reply") && options.missingRenderedReply); + }, editable: () => ready && !options.readOnly }); + const page = { url: () => current, goto: async (url: string) => { current = new URL(url, "https://fixture.test").href; ready = false; }, + reload: async () => {}, getByRole: () => locator("heading"), getByTestId: () => locator("composer"), locator }; + const api = { baseURL: "https://fixture.test", get: async (path: string) => { + calls.push(path); if (path.endsWith("/primary-agent/me")) return primary; + if (path.endsWith("/chats/ceo")) return { ...source, ...(options.wrongChat ? { id: "different-source" } : {}) }; + if (path.includes("/comments?")) return comments.map(row => options.wrongRequest && row.id === "human" ? { ...row, body: "different query" } : row); + if (path.endsWith("/chats")) return [source, ...(options.extraChat && ready ? [{ ...source, id: "extra" }] : [])]; + throw Error("Unexpected API read"); + } }; + const state = { worker: { id: "morgan", name: "Morgan", companyId: "company" }, primaryBefore: { companyId: "company", userId: "person", primaryAgentId: "ceo" }, + sourceBeforeNavigation: "source", nativeIdsBeforeNavigation: ["native"], chatCountBeforeNavigation: 1 }; + const evidence = vi.fn(async (_name: string, _value: unknown) => {}); + const ctx = { observationDeadlineAt: deadline, scenario: { id: "agents.choose-primary", query: "Make Morgan my go-to agent." }, + input: { api, page, evidence, fixtures: { company: { id: "company", issuePrefix: "RUN" }, agent: { id: "ceo" } } }, + issue: () => source, refreshIssue: async () => {}, allRuns: async () => [{ id: "native", agentId: "ceo", status: "succeeded" }, ...(options.extraRun && ready ? [{ id: "unexpected", agentId: "ceo", status: "running" }] : [])], + } as unknown as ChatCatalogContext; + const observation = { companyId: "company", leadId: "ceo", state, reply: "Use the human primary control.", agents: [{ id: "ceo", name: "CEO" }, state.worker], tasks: [], links: [], activity: [], runs: [], humanQueries: [] }; + return { ctx, state, observation, evidence, calls, ready: () => ready }; +} +it("actual final verifier waits for meaningful selected Chat then rechecks preference and no extra work before capture", async () => { + const f = fixture(); await runtimePrimaryCases[0]!.verifyResult(f.ctx, f.state, f.observation); + expect(f.ready()).toBe(true); expect(f.evidence).toHaveBeenCalledWith("chat-primary-ready-conversation.json", expect.objectContaining({ sourceId: "source", agentId: "ceo", primaryAgentId: "morgan", noExtraWork: true })); + expect(f.calls).toContain('[id="comment-human"]'); expect(f.calls).toContain('[id="comment-reply"]'); +}); +it.each([{ loadingForever: true }, { extraRun: true }, { extraChat: true }, { wrongPrimary: true }, { wrongPerson: true }, { wrongChat: true }, + { noReply: true }, { wrongRequest: true }, { missingRenderedReply: true }, { readOnly: true }, { expired: true }, { lateReady: true }])("actual final verifier rejects incomplete or changed settled state %j", async options => { + const f = fixture(options); await expect(runtimePrimaryCases[0]!.verifyResult(f.ctx, f.state, f.observation)).rejects.toThrow(); + expect(f.evidence.mock.calls.some(row => row[0] === "chat-primary-ready-conversation.json")).toBe(false); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts index 6e6ada13672..94bcc24a689 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts @@ -1,8 +1,8 @@ import { expect } from "@playwright/test"; import { checks, type Entity } from "./internal-support.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; -import { probeAgentPermission, waitForReply } from "./identity-support.js"; -import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, type RuntimeAgent } from "./runtime-support.js"; +import { probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js"; +import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, verifyRuntimeResult, type RuntimeAgent } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface Primary { companyId: string; userId: string; primaryAgentId: string | null } type Person = Awaited>; @@ -35,6 +35,44 @@ export function gradePrimary(value: unknown) { && !/I(?:'ve| have)? (?:set|made|changed).*(?:primary|go.to)/i.test(o.reply), "The model directs the personal action and does not falsely claim an agent preference write."); return c.result(); } +export async function verifyReadyPrimaryChat(ctx: ChatCatalogContext, state: Record, value: unknown) { + const o = runtimeObservation(value); const before = state.primaryBefore as Primary; const worker = state.worker as RuntimeAgent; + const sourceId = state.sourceBeforeNavigation; const nativeIds = state.nativeIdsBeforeNavigation; + if (!o || o.companyId !== ctx.input.fixtures.company.id || o.leadId !== ctx.input.fixtures.agent.id + || typeof sourceId !== "string" || !sourceId || !Array.isArray(nativeIds) || nativeIds.length !== 1 || typeof nativeIds[0] !== "string") + throw new Error("Primary Chat readiness requires its exact original conversation and native run"); + const budget = () => remainingIdentityObservationBudget(ctx); + const lead = o.agents.find(row => row.id === o.leadId); if (!lead) throw new Error("Primary Chat lead is missing"); + await expect(ctx.input.page).toHaveURL(url => { + const parts = url.pathname.split("/").filter(Boolean).map(decodeURIComponent); + return parts.length === 3 && parts[0] === ctx.input.fixtures.company.issuePrefix && parts[1] === "chats" && [lead.id, lead.urlKey].includes(parts[2]); + }, { timeout: budget() }); budget(); + const composer = ctx.input.page.getByTestId("task-chat-composer-input"); + await expect(composer).toBeVisible({ timeout: budget() }); budget(); + await expect(composer.locator('[contenteditable="true"], textarea').first()).toBeEditable({ timeout: budget() }); budget(); + const chat = await ctx.input.api.get(`/api/companies/${o.companyId}/chats/${o.leadId}`, { timeout: budget() }); budget(); + if (chat.id !== sourceId || chat.companyId !== o.companyId || chat.conversationAgentId !== o.leadId || chat.conversationUserId !== before.userId) + throw new Error("Ready Chat reopened a different conversation or person"); + const comments = await ctx.input.api.get(`/api/issues/${sourceId}/comments?order=asc`, { timeout: budget() }); budget(); + const humans = comments.filter(row => !row.authorAgentId); + const reply = comments.find(row => row.authorAgentId === o.leadId && row.createdByRunId === nativeIds[0] && !!row.body?.trim()); + if (humans.length !== 1 || humans[0]!.authorUserId !== before.userId || humans[0]!.body?.trim() !== ctx.scenario.query.trim() || !reply) + throw new Error("Ready Chat lacks its original human request and native response"); + for (const comment of [humans[0]!, reply]) { + await expect(ctx.input.page.locator(`[id=${JSON.stringify(`comment-${comment.id}`)}]`)).toBeVisible({ timeout: budget() }); budget(); + } + await ctx.refreshIssue(); budget(); + const primary = await ctx.input.api.get(endpoint(o.companyId), { timeout: budget() }); budget(); + const chats = await ctx.input.api.get(`/api/companies/${o.companyId}/chats`, { timeout: budget() }); budget(); + const runs = await ctx.allRuns(); budget(); + if (ctx.issue()?.id !== sourceId || primary.companyId !== o.companyId || primary.userId !== before.userId || primary.primaryAgentId !== worker.id + || chats.length !== state.chatCountBeforeNavigation || chats.filter(row => row.id === sourceId).length !== 1 + || runs.length !== 1 || runs[0]?.id !== nativeIds[0] || runs[0].agentId !== o.leadId || runs[0].status !== "succeeded") + throw new Error("Settled Chat changed the primary preference, conversation or native work"); + state.reopenedRecentAgent = o.leadId; state.runsAfterNavigation = runs.length; state.chatCountAfterNavigation = chats.length; + await ctx.input.evidence("chat-primary-ready-conversation.json", { companyId: o.companyId, agentId: o.leadId, sourceId, userId: before.userId, + primaryAgentId: primary.primaryAgentId, originalNativeIds: nativeIds, chatCount: chats.length, composerReady: true, originalConversationVisible: true, noExtraWork: true }); budget(); +} export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", async ctx => { const secrets = ctx.input.privateSecrets; if (!secrets) throw new Error("Primary fixture requires private credential sink"); const browser = ctx.input.page.context().browser(); if (!browser) throw new Error("Real browser required"); @@ -48,7 +86,7 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a otherCompanyBefore: await ctx.input.api.get(endpoint(otherCompany.id)) }; }, gradePrimary, { authentication: "human-session", - sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-scenarios/agents.yaml"], + sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { await waitForReply(ctx); const worker = state.worker as RuntimeAgent; state.beforeHumanPrimary = (await ctx.input.api.get(endpoint(ctx.input.fixtures.company.id))).primaryAgentId; @@ -67,12 +105,12 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a await expect(ctx.input.page.getByTestId("task-chat-composer-input")).toBeVisible(); const chats = `/api/companies/${ctx.input.fixtures.company.id}/chats`; state.chatCountBeforeNavigation = (await ctx.input.api.get(chats)).length; + state.sourceBeforeNavigation = ctx.issue()?.id; + state.nativeIdsBeforeNavigation = (await ctx.allRuns()).map(run => run.id); await ctx.input.page.goto(`/${ctx.input.fixtures.company.issuePrefix}/chats`, { waitUntil: "domcontentloaded" }); const lead = o.agents.find(row => row.id === ctx.input.fixtures.agent.id)!; await ctx.input.page.waitForURL(url => [lead.id, lead.urlKey].includes(decodeURIComponent(url.pathname.split("/").at(-1)!))); - state.reopenedRecentAgent = ctx.input.fixtures.agent.id; - state.runsAfterNavigation = (await ctx.allRuns()).length; - state.chatCountAfterNavigation = (await ctx.input.api.get(chats)).length; + await verifyReadyPrimaryChat(ctx, state, o); }, async observe(ctx, state) { const person = people.get(ctx); if (!person) throw new Error("Secondary authenticated person missing"); @@ -81,6 +119,12 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a state.otherCompanyAfter = await ctx.input.api.get(endpoint((state.otherCompany as Entity).id)); return observeRuntime(ctx, state); }, + async verifyResult(ctx, state, value) { + remainingIdentityObservationBudget(ctx); + await verifyRuntimeResult(ctx, state, value); + remainingIdentityObservationBudget(ctx); + await verifyReadyPrimaryChat(ctx, state, value); + }, async cleanup(ctx, state) { const person = people.get(ctx); people.delete(ctx); await person?.dispose(); await ctx.input.api.delete(`/api/companies/${(state.otherCompany as Entity).id}`, { allowNotFound: true });