From ad1e11fca641613361866d28ab797a140488c25a Mon Sep 17 00:00:00 2001 From: Dotta Date: Tue, 3 Nov 2026 00:47:01 -0600 Subject: [PATCH] test(evals): await canonical hire review link within original deadline Co-Authored-By: Paperclip --- .../identity-hire-review-readiness.test.ts | 70 +++++++++++++++++++ .../identity-hire-revision.ts | 58 +++++++++++---- 2 files changed, 115 insertions(+), 13 deletions(-) create mode 100644 tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts new file mode 100644 index 00000000000..fc66db7a827 --- /dev/null +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts @@ -0,0 +1,70 @@ +import { afterEach, expect, it, vi } from "vitest"; +import type { ChatCatalogContext } from "../chat-catalog-contract.js"; +const clock = vi.hoisted(() => ({ now: 30_000, deadlines: [] as number[] })); +vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value })); +vi.mock("@playwright/test", () => ({ expect: (node: { visible?: () => boolean; text?: () => string }) => ({ + toBeVisible: async () => { if (!node.visible?.()) throw Error("Actual pending proposal is not visible"); }, + toContainText: async (text: string) => { if (!node.text?.().includes(text)) throw Error("Actual revised model is missing"); }, +}) })); +vi.mock("../api.js", () => ({ pollUntil: async (options: { deadlineAt: number; load(): Promise; accept(value: unknown): boolean; reject?(value: unknown): string | undefined }) => { + clock.deadlines.push(options.deadlineAt); + while (Date.now() < options.deadlineAt) { + const value = await options.load(), rejected = options.reject?.(value); + if (rejected) throw Error(rejected); + if (options.accept(value)) return value; + clock.now += 10_000; + } + throw Error("Controlled original observation expired"); +} })); +import { hireRevisionIdentityCases } from "./identity-hire-revision.js"; +afterEach(() => { vi.restoreAllMocks(); document.body.replaceChildren(); }); +type Options = { deadline?: number | "missing"; readyAt?: number; href?: string; wrongComment?: boolean; foreignSource?: boolean; wrongActor?: boolean; wrongNativeComment?: boolean; wrongModel?: boolean; notPending?: boolean; lateDom?: boolean }; +function fixture(options: Options = {}) { + clock.now = 30_000; clock.deadlines.splice(0); vi.spyOn(Date, "now").mockImplementation(() => clock.now); + vi.spyOn(Element.prototype, "getClientRects").mockImplementation(() => [{ width: 10, height: 10 }] as unknown as DOMRectList); + let current = "https://fixture.test/ACORN/chats/ceo", clicks = 0; + const source = "source", approval = "approval", query = "Switch this hiring proposal to a cheaper model and send it back for approval."; + const readyAt = options.readyAt ?? 210_000, href = options.href ?? "/approvals/approval"; + const run = { id: "native", companyId: "company", agentId: options.wrongActor ? "other" : "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: source, commentId: options.wrongNativeComment ? "other" : "human" } }; + const comments = [{ id: "human", companyId: "company", issueId: source, authorUserId: "person", body: query }, + { id: "reply", companyId: "company", issueId: source, authorAgentId: "ceo", createdByRunId: "native", body: "Revised pending proposal." }]; + function sync() { + document.body.innerHTML = '
'; + if (clock.now >= readyAt) { const a = document.createElement("a"); a.setAttribute("href", href); a.textContent = "pending approval"; document.querySelector('div:last-child')!.append(a); } + } + type Nodes = { nodes: Element[]; locator(selector: string): Nodes; first(): Nodes; filter(options: { has: Nodes }): Nodes; evaluateAll(read: (nodes: Element[]) => T): Promise; click(): Promise; text(): string }; + function nodes(list: Element[]): Nodes { return { nodes: list, locator: selector => nodes(list.flatMap(n => Array.from(n.querySelectorAll(selector)))), first: () => nodes(list.slice(0,1)), + filter: ({has}) => nodes(list.filter(n => has.nodes.some(child => n.contains(child)))), + evaluateAll: async read => read(list), click: async () => { const a = list[0]; if (!a || !a.hasAttribute("href")) throw Error("Real review link absent"); clicks++; current = new URL(a.getAttribute("href")!, current).href; }, + text: () => options.wrongModel ? "wrong-model" : "requested-model" }; } + const page = { url: () => current, goto: async (url: string) => { current = new URL(url,current).href; }, + locator: (selector: string) => { if (selector === "pre") return nodes([]); if (selector.includes("[id^=")) sync(); return nodes(Array.from(document.querySelectorAll(selector))); }, + getByTestId: (id: string) => nodes(Array.from(document.querySelectorAll('[data-testid="'+id+'"]'))), + getByText: () => ({ first: () => ({ visible: () => !options.notPending && current.endsWith("/approvals/approval") }) }), + getByRole: () => ({ click: async () => {} }), + evaluate: async (read: (value: A) => T, value: A) => { if (options.lateDom) clock.now = 300_001; sync(); return read(value); }, + }; + const deadline = options.deadline === undefined ? 300_000 : options.deadline; + const ctx = { ...(deadline === "missing" ? {} : { observationDeadlineAt: deadline }), scenario: { query }, + input: { page, fixtures: { company: { id: "company", issuePrefix: "ACORN" }, agent: { id: "ceo" } }, api: { baseURL: "https://fixture.test", get: async (url: string) => { + if (url === "/api/issues/source/comments?order=asc") return comments; + if (url === "/api/approvals/approval") return { id: approval, companyId: "company", status: options.notPending ? "approved" : "pending" }; + throw Error("Unexpected read"); + } } }, issue: () => ({ id: options.foreignSource ? "foreign" : source, conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [run], + } as unknown as ChatCatalogContext; + return { ctx, state: { originalApproval: { id: approval, companyId: "company" }, humanId: "person", sourceIssueId: source, cheaperModel: "requested-model" }, clicks: () => clicks, url: () => current }; +} +it("awaits the delayed genuine canonical-reply review link within original300s, clicks and inspects actual pending revised model", async () => { + const f = fixture(); await hireRevisionIdentityCases[0]!.act!(f.ctx, f.state); + expect(f.clicks()).toBe(1); expect(clock.now).toBe(210_000); expect(clock.deadlines.every(x => x === 300_000)).toBe(true); + expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true }); expect(f.url()).toContain("/chats/ceo"); +}); +it.each([{ href: "https://foreign.test/approvals/approval" }, { href: "/OTHER/approvals/approval" }, { href: "/approvals/other" }, { wrongComment: true }, { foreignSource: true }, { wrongActor: true }, { wrongNativeComment: true }, { readyAt: 300_001 }, { lateDom: true }, { deadline: "missing" }, { deadline: NaN }, { deadline: 29_999 }])("rejects absent/foreign/late review readiness before any click %j", async options => { + const f = fixture(options as Options); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks()).toBe(0); +}); +it("accepts the same owned company-prefixed ordinary approval route", async () => { + const f = fixture({ readyAt:30_000, href:"/ACORN/approvals/approval" }); await hireRevisionIdentityCases[0]!.act!(f.ctx,f.state); expect(f.clicks()).toBe(1); +}); +it.each([{ wrongModel: true }, { notPending: true }])("does not turn review visibility into a wrong model/approved proposal PASS %j", async options => { + const f = fixture({ readyAt:30_000, ...options }); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx,f.state)).rejects.toThrow(); +}); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts index 8af04da24d5..4a13046572c 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts @@ -1,7 +1,9 @@ import { expect } from "@playwright/test"; import type { Approval, ApprovalComment } from "../../../packages/shared/src/types/approval.js"; import type { ChatCatalogCase } from "../chat-catalog-contract.js"; -import { checks, entity } from "./internal-support.js"; +import { pollUntil } from "../api.js"; +import { collectConversationLinks, conversationLinkLocator } from "../chat-rendered-links.js"; +import { checks, entity, type Entity } from "./internal-support.js"; import { identityDefinition, identityObservation, observeIdentity, openStandingChat, remainingIdentityObservationBudget, waitForReply } from "./identity-support.js"; import { CHAT_OPENCODE_MODEL } from "../chat-runtime-profile.js"; import { readHireModelChoice, validHireModelChoice } from "./identity-hire-model-choice.js"; @@ -49,21 +51,51 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe await ctx.input.page.getByRole("button", { name: "Request revision", exact: true }).click(); await expect.poll(async () => (await ctx.input.api.get(`/api/approvals/${originalApproval.id}`)).status).toBe("revision_requested"); await ctx.input.page.goto(url, { waitUntil: "domcontentloaded" }); - return { originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, + return { sourceIssueId: source.id, originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id, eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get(`/api/companies/${fixtures.company.id}/agents`)).length }; }, gradeHireRevision, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"], + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"], async act(ctx, state) { - const originalDeadlineAt = ctx.observationDeadlineAt; remainingIdentityObservationBudget(ctx); - await waitForReply(ctx, originalDeadlineAt); const id = entity(state, "originalApproval").id; const source = ctx.input.page.url(); - const links = await ctx.input.page.locator('[id^="comment-"] a[href]').evaluateAll(rows => rows.map(row => row.getAttribute("href")!)); - const link = links.find(href => new URL(href, ctx.input.api.baseURL).pathname.endsWith(`/approvals/${id}`)); - if (!link) throw new Error("Actual pending revised proposal review link is missing"); - await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(link)}]`).first().click(); - await expect(ctx.input.page.getByText("pending", { exact: true }).first()).toBeVisible(); state.pendingProposalVisible = true; - await ctx.input.page.getByRole("button", { name: "See full request", exact: true }).click(); - await expect(ctx.input.page.locator("pre")).toContainText(String(state.cheaperModel)); state.revisedModelVisible = true; - await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); + const remaining = () => remainingIdentityObservationBudget(ctx); + remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining(); + const id = entity(state, "originalApproval").id, sourceId = String(state.sourceIssueId), source = ctx.input.page.url(); + if (ctx.issue()?.id !== sourceId || entity(state, "originalApproval").companyId !== ctx.input.fixtures.company.id) + throw new Error("Hire review source does not match the owned proposal"); + const frame = await pollUntil({ label: "Actual canonical pending hire review link", deadlineAt: ctx.observationDeadlineAt!, + load: async () => { + await ctx.refreshIssue(); remaining(); + const [runs, comments, approval] = await Promise.all([ + ctx.allRuns(), ctx.input.api.get(`/api/issues/${sourceId}/comments?order=asc`, { timeout: remaining() }), + ctx.input.api.get(`/api/approvals/${id}`, { timeout: remaining() }), + ]); + remaining(); + const links = await collectConversationLinks(ctx.input.page, { origin: ctx.input.api.baseURL, companyId: ctx.input.fixtures.company.id, + companyPrefix: ctx.input.fixtures.company.issuePrefix, sourceId, leadId: ctx.input.fixtures.agent.id, comments, runs, projects: [], activity: [] }); + const humans = comments.filter(row => row.authorUserId === state.humanId && !row.createdByRunId && row.body?.trim() === ctx.scenario.query.trim()); + const run = runs[0]; + const bound = ctx.issue()?.id === sourceId && ctx.issue()?.conversationState === "waiting" && runs.length === 1 && !!run + && run.companyId === ctx.input.fixtures.company.id && run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded" + && run.contextSnapshot?.issueId === sourceId && humans.length === 1 && run.contextSnapshot?.commentId === humans[0]?.id; + const link = bound && run ? links.find(row => { + if (row.source !== "reply" || row.conversationId !== sourceId || !comments.some(comment => comment.id === row.commentId + && comment.authorAgentId === run.agentId && comment.createdByRunId === run.id)) return false; + try { const url = new URL(row.href, ctx.input.api.baseURL); return url.origin === new URL(ctx.input.api.baseURL).origin + && [ `/approvals/${id}`, `/${ctx.input.fixtures.company.issuePrefix}/approvals/${id}` ].includes(url.pathname); } + catch { return false; } + }) : undefined; + return { approval, link }; + }, + accept: value => Date.now() < ctx.observationDeadlineAt! && value.approval.id === id && value.approval.companyId === ctx.input.fixtures.company.id + && value.approval.status === "pending" && !!value.link, + reject: value => value.approval.id !== id || value.approval.companyId !== ctx.input.fixtures.company.id || value.approval.status !== "pending" + ? "The owned revised hire proposal is not pending" : undefined, + }); + remaining(); if (!frame.link) throw new Error("Actual pending revised proposal review link is missing"); + await conversationLinkLocator(ctx.input.page, frame.link).first().click({ timeout: remaining() }); + await expect(ctx.input.page.getByText("pending", { exact: true }).first()).toBeVisible({ timeout: remaining() }); state.pendingProposalVisible = true; + await ctx.input.page.getByRole("button", { name: "See full request", exact: true }).click({ timeout: remaining() }); + await expect(ctx.input.page.locator("pre")).toContainText(String(state.cheaperModel), { timeout: remaining() }); state.revisedModelVisible = true; + await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining(); }, async observe(ctx, state) { const id = entity(state, "originalApproval").id;