test(evals): await canonical hire review link within original deadline

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-hire-review-readiness.test.ts

@@ -0,0 +1,70 @@
+import { afterEach, expect, it, vi } from "vitest";
+import type { ChatCatalogContext } from "../chat-catalog-contract.js";
+const clock = vi.hoisted(() => ({ now: 30_000, deadlines: [] as number[] }));
+vi.mock("../codex-subscription-fixture.js", () => ({ codexSubscriptionAgentConfig: (value: unknown) => value }));
+vi.mock("@playwright/test", () => ({ expect: (node: { visible?: () => boolean; text?: () => string }) => ({
+  toBeVisible: async () => { if (!node.visible?.()) throw Error("Actual pending proposal is not visible"); },
+  toContainText: async (text: string) => { if (!node.text?.().includes(text)) throw Error("Actual revised model is missing"); },
+}) }));
+vi.mock("../api.js", () => ({ pollUntil: async (options: { deadlineAt: number; load(): Promise<unknown>; accept(value: unknown): boolean; reject?(value: unknown): string | undefined }) => {
+  clock.deadlines.push(options.deadlineAt);
+  while (Date.now() < options.deadlineAt) {
+    const value = await options.load(), rejected = options.reject?.(value);
+    if (rejected) throw Error(rejected);
+    if (options.accept(value)) return value;
+    clock.now += 10_000;
+  }
+  throw Error("Controlled original observation expired");
+} }));
+import { hireRevisionIdentityCases } from "./identity-hire-revision.js";
+afterEach(() => { vi.restoreAllMocks(); document.body.replaceChildren(); });
+type Options = { deadline?: number | "missing"; readyAt?: number; href?: string; wrongComment?: boolean; foreignSource?: boolean; wrongActor?: boolean; wrongNativeComment?: boolean; wrongModel?: boolean; notPending?: boolean; lateDom?: boolean };
+function fixture(options: Options = {}) {
+  clock.now = 30_000; clock.deadlines.splice(0); vi.spyOn(Date, "now").mockImplementation(() => clock.now);
+  vi.spyOn(Element.prototype, "getClientRects").mockImplementation(() => [{ width: 10, height: 10 }] as unknown as DOMRectList);
+  let current = "https://fixture.test/ACORN/chats/ceo", clicks = 0;
+  const source = "source", approval = "approval", query = "Switch this hiring proposal to a cheaper model and send it back for approval.";
+  const readyAt = options.readyAt ?? 210_000, href = options.href ?? "/approvals/approval";
+  const run = { id: "native", companyId: "company", agentId: options.wrongActor ? "other" : "ceo", runtimeMode: "native", status: "succeeded", contextSnapshot: { issueId: source, commentId: options.wrongNativeComment ? "other" : "human" } };
+  const comments = [{ id: "human", companyId: "company", issueId: source, authorUserId: "person", body: query },
+    { id: "reply", companyId: "company", issueId: source, authorAgentId: "ceo", createdByRunId: "native", body: "Revised pending proposal." }];
+  function sync() {
+    document.body.innerHTML = '<section data-testid="task-chat-thread"><div id="comment-human"></div><div id="comment-' + (options.wrongComment ? "foreign" : "reply") + '"></div></section>';
+    if (clock.now >= readyAt) { const a = document.createElement("a"); a.setAttribute("href", href); a.textContent = "pending approval"; document.querySelector('div:last-child')!.append(a); }
+  }
+  type Nodes = { nodes: Element[]; locator(selector: string): Nodes; first(): Nodes; filter(options: { has: Nodes }): Nodes; evaluateAll<T>(read: (nodes: Element[]) => T): Promise<T>; click(): Promise<void>; text(): string };
+  function nodes(list: Element[]): Nodes { return { nodes: list, locator: selector => nodes(list.flatMap(n => Array.from(n.querySelectorAll(selector)))), first: () => nodes(list.slice(0,1)),
+    filter: ({has}) => nodes(list.filter(n => has.nodes.some(child => n.contains(child)))),
+    evaluateAll: async read => read(list), click: async () => { const a = list[0]; if (!a || !a.hasAttribute("href")) throw Error("Real review link absent"); clicks++; current = new URL(a.getAttribute("href")!, current).href; },
+    text: () => options.wrongModel ? "wrong-model" : "requested-model" }; }
+  const page = { url: () => current, goto: async (url: string) => { current = new URL(url,current).href; },
+    locator: (selector: string) => { if (selector === "pre") return nodes([]); if (selector.includes("[id^=")) sync(); return nodes(Array.from(document.querySelectorAll(selector))); },
+    getByTestId: (id: string) => nodes(Array.from(document.querySelectorAll('[data-testid="'+id+'"]'))),
+    getByText: () => ({ first: () => ({ visible: () => !options.notPending && current.endsWith("/approvals/approval") }) }),
+    getByRole: () => ({ click: async () => {} }),
+    evaluate: async <T, A>(read: (value: A) => T, value: A) => { if (options.lateDom) clock.now = 300_001; sync(); return read(value); },
+  };
+  const deadline = options.deadline === undefined ? 300_000 : options.deadline;
+  const ctx = { ...(deadline === "missing" ? {} : { observationDeadlineAt: deadline }), scenario: { query },
+    input: { page, fixtures: { company: { id: "company", issuePrefix: "ACORN" }, agent: { id: "ceo" } }, api: { baseURL: "https://fixture.test", get: async (url: string) => {
+      if (url === "/api/issues/source/comments?order=asc") return comments;
+      if (url === "/api/approvals/approval") return { id: approval, companyId: "company", status: options.notPending ? "approved" : "pending" };
+      throw Error("Unexpected read");
+    } } }, issue: () => ({ id: options.foreignSource ? "foreign" : source, conversationState: "waiting" }), refreshIssue: async () => {}, allRuns: async () => [run],
+  } as unknown as ChatCatalogContext;
+  return { ctx, state: { originalApproval: { id: approval, companyId: "company" }, humanId: "person", sourceIssueId: source, cheaperModel: "requested-model" }, clicks: () => clicks, url: () => current };
+}
+it("awaits the delayed genuine canonical-reply review link within original300s, clicks and inspects actual pending revised model", async () => {
+  const f = fixture(); await hireRevisionIdentityCases[0]!.act!(f.ctx, f.state);
+  expect(f.clicks()).toBe(1); expect(clock.now).toBe(210_000); expect(clock.deadlines.every(x => x === 300_000)).toBe(true);
+  expect(f.state).toMatchObject({ pendingProposalVisible: true, revisedModelVisible: true }); expect(f.url()).toContain("/chats/ceo");
+});
+it.each([{ href: "https://foreign.test/approvals/approval" }, { href: "/OTHER/approvals/approval" }, { href: "/approvals/other" }, { wrongComment: true }, { foreignSource: true }, { wrongActor: true }, { wrongNativeComment: true }, { readyAt: 300_001 }, { lateDom: true }, { deadline: "missing" }, { deadline: NaN }, { deadline: 29_999 }])("rejects absent/foreign/late review readiness before any click %j", async options => {
+  const f = fixture(options as Options); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx, f.state)).rejects.toThrow(); expect(f.clicks()).toBe(0);
+});
+it("accepts the same owned company-prefixed ordinary approval route", async () => {
+  const f = fixture({ readyAt:30_000, href:"/ACORN/approvals/approval" }); await hireRevisionIdentityCases[0]!.act!(f.ctx,f.state); expect(f.clicks()).toBe(1);
+});
+it.each([{ wrongModel: true }, { notPending: true }])("does not turn review visibility into a wrong model/approved proposal PASS %j", async options => {
+  const f = fixture({ readyAt:30_000, ...options }); await expect(hireRevisionIdentityCases[0]!.act!(f.ctx,f.state)).rejects.toThrow();
+});

tests/runner-e2e/chat-catalog-domains/identity-hire-revision.ts

@@ -1,7 +1,9 @@
 import { expect } from "@playwright/test";
 import type { Approval, ApprovalComment } from "../../../packages/shared/src/types/approval.js";
 import type { ChatCatalogCase } from "../chat-catalog-contract.js";
-import { checks, entity } from "./internal-support.js";
+import { pollUntil } from "../api.js";
+import { collectConversationLinks, conversationLinkLocator } from "../chat-rendered-links.js";
+import { checks, entity, type Entity } from "./internal-support.js";
 import { identityDefinition, identityObservation, observeIdentity, openStandingChat, remainingIdentityObservationBudget, waitForReply } from "./identity-support.js";
 import { CHAT_OPENCODE_MODEL } from "../chat-runtime-profile.js";
 import { readHireModelChoice, validHireModelChoice } from "./identity-hire-model-choice.js";
@@ -49,21 +51,51 @@ export const hireRevisionIdentityCases: readonly ChatCatalogCase[] = [identityDe
   await ctx.input.page.getByRole("button", { name: "Request revision", exact: true }).click();
   await expect.poll(async () => (await ctx.input.api.get<Approval>(`/api/approvals/${originalApproval.id}`)).status).toBe("revision_requested");
   await ctx.input.page.goto(url, { waitUntil: "domcontentloaded" });
-  return { originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id,
+  return { sourceIssueId: source.id, originalApproval, originalHistoryCommentId: history.id, originalModel, cheaperModel, humanId: session.user.id,
     eligibleModels: models.map(row => row.id), modelChoice, beforeAgentCount: (await ctx.input.api.get<unknown[]>(`/api/companies/${fixtures.company.id}/agents`)).length };
 }, gradeHireRevision, {
-  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-scenarios/questions-and-approvals.yaml"],
+  authentication: "human-session", oracleVersion: "paperclip.chat-catalog.hire-revision.native1017.v3", sourceFiles: ["chat-catalog-domains/identity-hire-revision.ts", "chat-catalog-domains/identity-hire-model-choice.ts", "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-rendered-links.ts", "chat-scenarios/questions-and-approvals.yaml"],
   async act(ctx, state) {
-    const originalDeadlineAt = ctx.observationDeadlineAt; remainingIdentityObservationBudget(ctx);
-    await waitForReply(ctx, originalDeadlineAt); const id = entity(state, "originalApproval").id; const source = ctx.input.page.url();
-    const links = await ctx.input.page.locator('[id^="comment-"] a[href]').evaluateAll(rows => rows.map(row => row.getAttribute("href")!));
-    const link = links.find(href => new URL(href, ctx.input.api.baseURL).pathname.endsWith(`/approvals/${id}`));
-    if (!link) throw new Error("Actual pending revised proposal review link is missing");
-    await ctx.input.page.locator(`[id^="comment-"] a[href=${JSON.stringify(link)}]`).first().click();
-    await expect(ctx.input.page.getByText("pending", { exact: true }).first()).toBeVisible(); state.pendingProposalVisible = true;
-    await ctx.input.page.getByRole("button", { name: "See full request", exact: true }).click();
-    await expect(ctx.input.page.locator("pre")).toContainText(String(state.cheaperModel)); state.revisedModelVisible = true;
-    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" });
+    const remaining = () => remainingIdentityObservationBudget(ctx);
+    remaining(); await waitForReply(ctx, ctx.observationDeadlineAt); remaining();
+    const id = entity(state, "originalApproval").id, sourceId = String(state.sourceIssueId), source = ctx.input.page.url();
+    if (ctx.issue()?.id !== sourceId || entity(state, "originalApproval").companyId !== ctx.input.fixtures.company.id)
+      throw new Error("Hire review source does not match the owned proposal");
+    const frame = await pollUntil({ label: "Actual canonical pending hire review link", deadlineAt: ctx.observationDeadlineAt!,
+      load: async () => {
+        await ctx.refreshIssue(); remaining();
+        const [runs, comments, approval] = await Promise.all([
+          ctx.allRuns(), ctx.input.api.get<Entity[]>(`/api/issues/${sourceId}/comments?order=asc`, { timeout: remaining() }),
+          ctx.input.api.get<Approval>(`/api/approvals/${id}`, { timeout: remaining() }),
+        ]);
+        remaining();
+        const links = await collectConversationLinks(ctx.input.page, { origin: ctx.input.api.baseURL, companyId: ctx.input.fixtures.company.id,
+          companyPrefix: ctx.input.fixtures.company.issuePrefix, sourceId, leadId: ctx.input.fixtures.agent.id, comments, runs, projects: [], activity: [] });
+        const humans = comments.filter(row => row.authorUserId === state.humanId && !row.createdByRunId && row.body?.trim() === ctx.scenario.query.trim());
+        const run = runs[0];
+        const bound = ctx.issue()?.id === sourceId && ctx.issue()?.conversationState === "waiting" && runs.length === 1 && !!run
+          && run.companyId === ctx.input.fixtures.company.id && run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded"
+          && run.contextSnapshot?.issueId === sourceId && humans.length === 1 && run.contextSnapshot?.commentId === humans[0]?.id;
+        const link = bound && run ? links.find(row => {
+          if (row.source !== "reply" || row.conversationId !== sourceId || !comments.some(comment => comment.id === row.commentId
+            && comment.authorAgentId === run.agentId && comment.createdByRunId === run.id)) return false;
+          try { const url = new URL(row.href, ctx.input.api.baseURL); return url.origin === new URL(ctx.input.api.baseURL).origin
+            && [ `/approvals/${id}`, `/${ctx.input.fixtures.company.issuePrefix}/approvals/${id}` ].includes(url.pathname); }
+          catch { return false; }
+        }) : undefined;
+        return { approval, link };
+      },
+      accept: value => Date.now() < ctx.observationDeadlineAt! && value.approval.id === id && value.approval.companyId === ctx.input.fixtures.company.id
+        && value.approval.status === "pending" && !!value.link,
+      reject: value => value.approval.id !== id || value.approval.companyId !== ctx.input.fixtures.company.id || value.approval.status !== "pending"
+        ? "The owned revised hire proposal is not pending" : undefined,
+    });
+    remaining(); if (!frame.link) throw new Error("Actual pending revised proposal review link is missing");
+    await conversationLinkLocator(ctx.input.page, frame.link).first().click({ timeout: remaining() });
+    await expect(ctx.input.page.getByText("pending", { exact: true }).first()).toBeVisible({ timeout: remaining() }); state.pendingProposalVisible = true;
+    await ctx.input.page.getByRole("button", { name: "See full request", exact: true }).click({ timeout: remaining() });
+    await expect(ctx.input.page.locator("pre")).toContainText(String(state.cheaperModel), { timeout: remaining() }); state.revisedModelVisible = true;
+    await ctx.input.page.goto(source, { waitUntil: "domcontentloaded", timeout: remaining() }); remaining();
   },
   async observe(ctx, state) {
     const id = entity(state, "originalApproval").id;