diff --git a/tests/runner-e2e/chat-catalog-descriptors.json b/tests/runner-e2e/chat-catalog-descriptors.json index b9997ea75d0..aafad45f81a 100644 --- a/tests/runner-e2e/chat-catalog-descriptors.json +++ b/tests/runner-e2e/chat-catalog-descriptors.json @@ -487,7 +487,7 @@ "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml" ], - "oracleVersion": "paperclip.chat-catalog.runtime-primary.v3", + "oracleVersion": "paperclip.chat-catalog.runtime-primary.v4", "coverageGaps": [], "expectedRunCount": 1, "turnTimeoutMs": 120000, @@ -1238,10 +1238,10 @@ "chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-budgets-costs.ts", - "chat-catalog-domains/internal-comment-reference.ts", + "chat-rendered-links.ts", "chat-scenarios/budgets-costs.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.set-budget.v2", + "oracleVersion": "paperclip.chat-catalog.identity.set-budget.v3", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 300000, @@ -2081,7 +2081,7 @@ "chat-catalog-domains/internal-support.ts", "chat-scenarios/companies.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.fast-responses.v2", + "oracleVersion": "paperclip.chat-catalog.identity.fast-responses.v3", "expectedRunCount": 1, "turnTimeoutMs": 120000, "attemptTimeoutMs": 420000, @@ -4292,7 +4292,7 @@ "chat-catalog-domains/identity-questions-and-approvals.ts", "chat-scenarios/questions-and-approvals.yaml" ], - "oracleVersion": "paperclip.chat-catalog.identity.v1", + "oracleVersion": "paperclip.chat-catalog.identity.necessary-question.v2", "expectedRunCount": 4, "minimumRunCount": 1, "turnTimeoutMs": 120000, diff --git a/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts b/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts index 89ea2cb72f4..e31a5015d96 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-budgets-costs.ts @@ -1,10 +1,19 @@ import { chatCatalogOriginalPhaseDeadline } from "../chat-catalog-contract.js"; -import { expect } from "@playwright/test"; +import { expect, type Locator } from "@playwright/test"; import type { BudgetOverview } from "../../../packages/shared/src/types/budget.js"; import { checks, entity } from "./internal-support.js"; -import { identityDefinition, identityObservation, identityWorker, observeIdentity, waitForReply, probeAgentPermission } from "./identity-support.js"; -import { waitForRenderedReply } from "./internal-comment-reference.js"; +import { identityDefinition, identityObservation, identityWorker, observeIdentity, waitForReply, waitForTurnReplyLinks, probeAgentPermission } from "./identity-support.js"; +/** The budget card remounts when its policy reloads, and a remount resets a typed draft. Resolve once the same + * input node has stayed on the page for `stableMs` (bounded), so a following fill is not lost to a reload. */ +export async function settledInput(input: Locator, stableMs = 1_000, timeoutMs = 15_000) { + for (const deadlineAt = Date.now() + timeoutMs; Date.now() < deadlineAt;) { + await input.evaluate(node => node.setAttribute("data-e2e-settle-mark", "1")); + await new Promise(resolve => setTimeout(resolve, stableMs)); + if (await input.getAttribute("data-e2e-settle-mark").catch(() => null) === "1") return true; + } + return false; +} export function gradeBudget(value: unknown) { const c = checks(); const o = identityObservation(value); c.add("complete-budget-evidence", !!o, "Saved policy and browser-action evidence is required."); if (!o) return c.result(); @@ -31,13 +40,13 @@ export const budgetIdentityCases = [identityDefinition("budgets-costs.set-budget const workerConfig = await employeeConfiguration(ctx, worker.id); return { worker, other, workerConfig }; }, gradeBudget, { - oracleVersion: "paperclip.chat-catalog.identity.set-budget.v2", - sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-budgets-costs.ts", "chat-catalog-domains/internal-comment-reference.ts", "chat-scenarios/budgets-costs.yaml"], + oracleVersion: "paperclip.chat-catalog.identity.set-budget.v3", + sourceFiles: ["chat-catalog-domains/identity-support.ts", "chat-catalog-domains/internal-support.ts", "chat-catalog-domains/identity-budgets-costs.ts", "chat-rendered-links.ts", "chat-scenarios/budgets-costs.yaml"], async act(ctx, state) { await waitForReply(ctx, chatCatalogOriginalPhaseDeadline(ctx)); - // The API sees the saved reply before the open conversation renders it; + // The run is recorded finished before its reply is saved, and the open conversation renders it later still; // read the reply's anchors only once that reply is on the page. - await waitForRenderedReply(ctx, { leadId: ctx.input.fixtures.agent.id, runs: await ctx.allRuns() }); + await waitForTurnReplyLinks(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const worker = entity(state, "worker"); state.agentWriteStatus = await probeAgentPermission(ctx, `${root}/budgets/policies`, "POST", { scopeType: "agent", scopeId: worker.id, amount: 5000, windowKind: "calendar_month_utc" }); @@ -56,8 +65,14 @@ export const budgetIdentityCases = [identityDefinition("budgets-costs.set-budget // form shows the currently saved cap, and confirm the typed value stuck. const amount = ctx.input.page.getByPlaceholder("0.00"); await expect(amount).toHaveValue((Number(state.beforeHumanAmount ?? 0) / 100).toFixed(2)); - await amount.fill("50.00"); await expect(amount).toHaveValue("50.00"); - await ctx.input.page.getByRole("button", { name: "Update budget", exact: true }).click(); + // Act on the settled form: if the card still remounts and drops the typed amount, type it again (bounded). + const save = ctx.input.page.getByRole("button", { name: "Update budget", exact: true }); + for (let attempt = 1; ; attempt++) { + await settledInput(amount); + if (await amount.inputValue() !== "50.00") { await amount.fill("50.00"); await expect(amount).toHaveValue("50.00"); } + if (await save.click({ timeout: 5_000 }).then(() => true, () => false)) break; + if (attempt >= 3) throw new Error("The budget form kept reloading and dropped the typed amount before it could be saved"); + } await expect(amount).toHaveValue("50.00"); state.humanSaved = true; await ctx.input.evidence("identity-human-budget-action.json", { workerId: worker.id, amountCents: 5000, url: ctx.input.page.url() }); await ctx.input.page.goto(source, { waitUntil: "domcontentloaded" }); diff --git a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts index 67943e0e404..be93f54dc19 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-company-models.ts @@ -7,7 +7,7 @@ import type { PromptInjectionPreventionSettings, PromptInjectionTestResult } fro import type { FastResponseSettings, FastResponseTestResult } from "../../../packages/shared/src/fast-response.js"; import type { CostSummary } from "../../../packages/shared/src/types/cost.js"; import { checks } from "./internal-support.js"; -import { followRenderedLink, identityDefinition, identityObservation, observeIdentity, probeAgentPermission } from "./identity-support.js"; +import { followRenderedLink, identityDefinition, identityObservation, observeIdentity, probeAgentPermission, waitForTurnReplyLinks } from "./identity-support.js"; import type { Locator } from "@playwright/test"; export const requestedAuxiliaryServiceModel = "deepseek/deepseek-v4.1-flash-1017"; @@ -125,7 +125,7 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s } catch (error) { const prior = fastExperiments.get(ctx); if (prior !== undefined) { await ctx.input.api.patch("/api/instance/settings/experimental", { enableFastResponses: prior }); fastExperiments.delete(ctx); } throw error; } }, gradeCompanyModel, { sourceFiles: sources, authentication: "human-session", requiredCredentials: ["OPENROUTER_API_KEY"], attemptTimeoutMs: 420_000, - ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v2" } : {}), + ...(spec.id === "companies.fast-responses" ? { oracleVersion: "paperclip.chat-catalog.identity.fast-responses.v3" } : {}), async act(ctx, state) { await waitForCompanySettingsTurn(ctx); const root = `/api/companies/${ctx.input.fixtures.company.id}`; const source = ctx.input.page.url(); state.afterReplyEnabled = (await ctx.input.api.get<{ settings: Settings }>(`${root}/${spec.path}`)).settings.enabled; @@ -134,6 +134,8 @@ export const companyModelIdentityCases: readonly ChatCatalogCase[] = specs.map(s state.agentTestStatus = await probeAgentPermission(ctx, `${root}/${spec.path}/test`, "POST", {}); const section = spec.path === "decision-model" ? ctx.input.page.locator('section[aria-labelledby="decision-model-heading"]') : ctx.input.page.getByRole("region", { name: spec.section, exact: true }); if (spec.path === "fast-response") { + // The run is recorded finished before its reply is saved and rendered; read the reply's links only once it is on the page. + await waitForTurnReplyLinks(ctx); const o = await observeIdentity(ctx, state); const href = await openServiceControlFromReply(ctx, o.links, o.origin, () => section); state.replyLinkedControlHref = href ?? null; diff --git a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts index 07b7cceae1a..0ff77a0e55a 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts @@ -52,6 +52,41 @@ async function answerText(ctx: ChatCatalogContext, answer: string) { } await form.getByRole("button", { name: /^(Submit|Send) answers$/ }).click(); } +/** The one required question of the agent's card, when every other question on it is optional. */ +export function soleRequiredQuestion(card: Pick | undefined) { + const questions = card?.payload.questionSet?.questions ?? []; + const required = questions.filter(question => question.required === true); + return required.length === 1 ? required[0] : undefined; +} +/** Answer the card as a person would: page through it, answer its one required question and Skip + * each optional one. A card with several questions shows Next until its last page. */ +async function answerRequiredQuestion(ctx: ChatCatalogContext, answer: string) { + const form = ctx.input.page.getByTestId("task-chat-composer-takeover"); + await expect(form).toBeVisible(); + const card = (await cards(ctx)).find(row => row.kind === "ask_user_questions" && row.status === "pending"); + const questions = card?.payload.questionSet?.questions ?? []; + if (questions.length <= 1) return answerText(ctx, answer); + const required = soleRequiredQuestion(card); + if (!required) throw new Error("The agent's question card does not have exactly one required question"); + for (const [index, question] of questions.entries()) { + const last = index === questions.length - 1; + // Pages follow the card's question order; only an optional question's page offers Skip. + const skip = form.getByRole("button", { name: "Skip", exact: true }); + if (question.id !== required.id) { + // Skip leaves an optional question unanswered and moves on (or sends, on the last page). + await skip.click(); + continue; + } + await expect(skip).toBeHidden(); + const text = form.getByTestId("question-text-answer-composer"); + if (await text.isVisible()) await text.locator('[contenteditable="true"],textarea').first().fill(answer); + else { + await form.getByRole("radio", { name: question.customAnswer?.label ?? "Other", exact: true }).click(); + await form.getByTestId("question-other-answer-composer").locator('[contenteditable="true"],textarea').first().fill(answer); + } + await form.getByRole("button", last ? { name: /^(Submit|Send) answers$/ } : { name: "Next", exact: true }).click(); + } +} async function beforeAnswer(ctx: ChatCatalogContext, state: Record) { state.beforeAnswerCommentIds = (await ctx.input.api.get>(`/api/issues/${ctx.issue()!.id}/comments`)).map(row => row.id); state.beforeAnswerTasks = (await ctx.input.api.get>(`/api/companies/${ctx.input.fixtures.company.id}/issues?includeAgentCreatedTasks=true`)).filter(row => row.id !== ctx.issue()!.id) @@ -184,12 +219,12 @@ export function gradeQuestions(value: unknown) { c.add("later-answer-resumes-once", o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1 && String(o.state.afterAnswerReply).includes(String(o.state.answer)), "Only the later submitted answer causes one continuation and is used in its actual response."); } else { + // The card may carry optional follow-ups, which the person skips; it must require exactly the one necessary answer. + const necessary = soleRequiredQuestion(original); c.add("agent-created-essential-question", original?.createdByAgentId === o.leadId && !!original.sourceRunId - && o.runs.some(run => run.id === original.sourceRunId && run.agentId === o.leadId && run.companyId === o.companyId && run.status === "succeeded") && original.payload.questionSet?.questions.length === 1 - && (original.payload.questionSet.questions[0]!.answerMode === "text" || original.payload.questionSet.questions[0]!.customAnswer?.enabled === true) - && original.payload.questionSet.questions[0]!.required === true - && /\b(?:product|name)\b/i.test(original.payload.questionSet.questions[0]!.prompt) - && !/model|runtime|adapter|framework|deployment|database|hosting/i.test(original.payload.questionSet.questions[0]!.prompt), "The agent itself asks exactly one necessary free-text product-name question tied to its actual completed run, excluding incidental implementation settings."); + && o.runs.some(run => run.id === original.sourceRunId && run.agentId === o.leadId && run.companyId === o.companyId && run.status === "succeeded") && !!necessary + && (necessary.answerMode === "text" || necessary.customAnswer?.enabled === true), + "The agent itself requires exactly one free-text answer, tied to its actual completed run; any other question on the card is optional. Whether that answer is the product name is proved by the saved brief carrying the person's answer."); c.add("answer-in-saved-brief", o.tasks.some(row => `${row.title}\n${row.description}`.includes(String(o.state.answer)) && row.companyId === o.companyId && hasLink(o, "issues", row)), "A saved, linked task brief uses the actual answer rather than merely acknowledging it in chat."); c.add("one-bound-continuation", o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1 @@ -249,7 +284,7 @@ export const questionIdentityCases = [ await ctx.input.api.patch(`/api/issues/${source.id}`, { description: existingBrief }); return { project: p, sourceId: source.id, existingBrief, answer: `Acorn ${ctx.input.nonce}` }; }, gradeQuestions, { - expectedRunCount: 4, minimumRunCount: 1, attemptTimeoutMs: 540_000, + oracleVersion: "paperclip.chat-catalog.identity.necessary-question.v2", expectedRunCount: 4, minimumRunCount: 1, attemptTimeoutMs: 540_000, observe: observeQuestion, async verifySetup(ctx, state) { const visible = await readAgentResource(ctx, `/api/issues/${state.sourceId}`); @@ -267,10 +302,10 @@ export const questionIdentityCases = [ load: () => cards(ctx), accept: rows => chatCatalogNativePhaseIsTimely(ctx) && rows.some(row => row.kind === "ask_user_questions" && row.status === "pending") }); const original = pending.find(row => row.kind === "ask_user_questions" && row.status === "pending")!; state.original = original; - await expect(ctx.input.page.getByTestId("task-chat-composer-takeover")).toContainText(/product|name/i); + await expect(ctx.input.page.getByTestId("task-chat-composer-takeover")).toBeVisible(); await ctx.input.evidence("identity-question-before-answer.json", { original, runs: await ctx.allRuns() }); await beforeAnswer(ctx, state); - await answerText(ctx, String(state.answer)); + await answerRequiredQuestion(ctx, String(state.answer)); }, }), identityDefinition("questions-and-approvals.multi-part-question", ctx => questionFixture(ctx, true), gradeQuestions, { diff --git a/tests/runner-e2e/chat-catalog-domains/identity-support.ts b/tests/runner-e2e/chat-catalog-domains/identity-support.ts index 80940167d5e..82bd6f68727 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity-support.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity-support.ts @@ -1,5 +1,6 @@ import { expect } from "@playwright/test"; import type { ChatCatalogCase, ChatCatalogContext } from "../chat-catalog-contract.js"; +import type { ChatRun } from "../chat-flow.js"; import { observeInternal, type Entity, type InternalObservation } from "./internal-support.js"; export interface IdentityObservation extends InternalObservation { @@ -62,6 +63,15 @@ export async function waitForReply(ctx: ChatCatalogContext, observationDeadlineA accept: ({ issue, runs }) => (observationDeadlineAt === undefined || Date.now() < observationDeadlineAt) && issue?.conversationState === "waiting" && runs.length === 1 && runs.every(run => run.status === "succeeded"), reject: ({ runs }) => runs.some(run => ["failed", "cancelled", "timed_out"].includes(run.status)) ? "Initial response failed" : undefined }); } +/** Before reading a finished turn's reply links: wait (bounded) for each finished source run's saved reply and its render. */ +export async function waitForTurnReplyLinks(ctx: ChatCatalogContext) { + const { waitForSavedTurnReplyRender } = await import("../chat-rendered-links.js"); + const source = ctx.issue(); if (!source?.id) return "unsaved" as const; + const runIds = (await ctx.allRuns()).filter(run => run.agentId === ctx.input.fixtures.agent.id && run.status === "succeeded" + && ((run as ChatRun & { nativeIssueId?: string }).nativeIssueId ?? run.contextSnapshot?.issueId) === source.id).map(run => run.id); + return waitForSavedTurnReplyRender(ctx.input.page, { leadId: ctx.input.fixtures.agent.id, runIds, + comments: () => ctx.input.api.get(`/api/issues/${source.id}/comments?order=asc`) }); +} export async function identityWorker(ctx: ChatCatalogContext, name = "Morgan") { const { api, execution, fixtures, workspacePath, nonce } = ctx.input; const base = execution.profile.buildAgent({ environmentId: fixtures.environment.id, environmentFixtureId: execution.environment.id, diff --git a/tests/runner-e2e/chat-catalog-domains/identity.test.ts b/tests/runner-e2e/chat-catalog-domains/identity.test.ts index f35e95f3637..3f913a0341e 100644 --- a/tests/runner-e2e/chat-catalog-domains/identity.test.ts +++ b/tests/runner-e2e/chat-catalog-domains/identity.test.ts @@ -163,6 +163,25 @@ describe("identity catalog independent oracle calibration", () => { original.idempotencyKey = "paperclip-runner-question:run:request-7"; (o.state.taskBrief as { createdAt: string }).createdAt = "2026-10-25T12:00:00.000Z"; expect(gradeQuestions(o).passed).toBe(false); }); + // Full cohort 3: the agent's card required the product name and added one optional follow-up. + it("admits optional follow-ups beside the one required product-name question, and nothing else", () => { + const withQuestions = (questions: NonNullable["questions"]) => { + const o = valid("questions-and-approvals.necessary-question"); const original = o.state.original as Interaction; + original.payload.questionSet = { ...original.payload.questionSet!, questions }; return gradeQuestions(o).passed; + }; + const name = { id: "product_name", prompt: "What is the product's name, exactly as it should appear on the landing page?", answerMode: "text", required: true }; + const optional = { id: "product_context", prompt: "Optional: what does the product do and who is it for?", answerMode: "text", required: false }; + expect(withQuestions([name])).toBe(true); + expect(withQuestions([name, optional])).toBe(true); + expect(withQuestions([optional, { ...name, prompt: "What should we call it?" }])).toBe(true); + // Broken: the necessary question is optional, a second answer is required, or the required answer is not free text. + expect(withQuestions([{ ...name, required: false }, optional])).toBe(false); + expect(withQuestions([name, { ...optional, required: true }])).toBe(false); + expect(withQuestions([{ ...name, answerMode: "single_select" }, optional])).toBe(false); + // The answer must reach the saved brief: an asked-and-ignored question still fails. + const ignored = valid("questions-and-approvals.necessary-question"); (ignored.state.original as Interaction).payload.questionSet!.questions = [name, optional]; + ignored.tasks[0]!.description = "An invented product name"; ignored.tasks[0]!.title = "Landing page"; expect(gradeQuestions(ignored).passed).toBe(false); + }); it("approved read is graded on the provider nonce and one provider receipt, not the provider sentence", () => { const grade = (o: IdentityObservation) => toolReviewIdentityCases[0]!.grade(o).passed; // Retained u2 reply (run 42641280837) dropped the "Launch notes: " prefix but carried the actual result. diff --git a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts index d85792f65af..d66f1cd065a 100644 --- a/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts +++ b/tests/runner-e2e/chat-catalog-domains/runtime-primary.ts @@ -1,7 +1,7 @@ import { expect } from "@playwright/test"; import { checks, type Entity } from "./internal-support.js"; import { createAuthenticatedPerson, enrollPrerequisitePerson } from "./identity-auth.js"; -import { probeAgentPermission, waitForReply, remainingIdentityObservationBudget } from "./identity-support.js"; +import { probeAgentPermission, waitForReply, waitForTurnReplyLinks, remainingIdentityObservationBudget } from "./identity-support.js"; import { runtimeDefinition, runtimeWorker, runtimeObservation, observeRuntime, openRuntimeAgentProfile, verifyRuntimeResult, agentLink, type RuntimeAgent } from "./runtime-support.js"; import type { ChatCatalogContext } from "../chat-catalog-contract.js"; interface Primary { companyId: string; userId: string; primaryAgentId: string | null } @@ -89,7 +89,7 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a return { worker, primaryBefore, otherCompany, otherPersonBefore: await personPrimary(person, ctx.input.fixtures.company.id), otherCompanyBefore: await ctx.input.api.get(endpoint(otherCompany.id)) }; }, gradePrimary, { - authentication: "human-session", oracleVersion: "paperclip.chat-catalog.runtime-primary.v3", + authentication: "human-session", oracleVersion: "paperclip.chat-catalog.runtime-primary.v4", sourceFiles: ["chat-catalog-domains/runtime-primary.ts", "chat-catalog-domains/runtime-support.ts", "chat-catalog-domains/identity-support.ts", "chat-scenarios/agents.yaml"], async act(ctx, state) { remainingIdentityObservationBudget(ctx); @@ -98,6 +98,8 @@ export const runtimePrimaryCases = [runtimeDefinition("agents.choose-primary", a const worker = state.worker as RuntimeAgent; state.beforeHumanPrimary = (await ctx.input.api.get(endpoint(ctx.input.fixtures.company.id))).primaryAgentId; state.agentWriteStatus = await probeAgentPermission(ctx, endpoint(ctx.input.fixtures.company.id), "PUT", { primaryAgentId: worker.id }); + // The run is recorded finished before its reply is saved and rendered; read the reply's links only once it is on the page. + await waitForTurnReplyLinks(ctx); const o = await observeRuntime(ctx, state); const source = ctx.input.page.url(); const href = agentLink(o, worker); diff --git a/tests/runner-e2e/chat-rendered-links.test.ts b/tests/runner-e2e/chat-rendered-links.test.ts index 09463a79943..3a1d12be30f 100644 --- a/tests/runner-e2e/chat-rendered-links.test.ts +++ b/tests/runner-e2e/chat-rendered-links.test.ts @@ -1,7 +1,7 @@ // @vitest-environment jsdom import { readFileSync } from "node:fs"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { collectConversationLinks, conversationLinkBindings, readConversationDomLinks, readLatestConversationReplyRender, waitForLatestConversationReplyRender, type ConversationLinkInput } from "./chat-rendered-links.js"; +import { collectConversationLinks, conversationLinkBindings, readConversationDomLinks, readLatestConversationReplyRender, waitForLatestConversationReplyRender, waitForSavedTurnReplyRender, type ConversationLinkInput } from "./chat-rendered-links.js"; const card = '
'; const thread = (content = card) => `
Create a project called Launch for the new website.
${content}
Created the Launch project. No repositories are connected yet.
`; @@ -124,3 +124,23 @@ describe("waits for the latest saved reply to render before reading links", () = expect(await waitForLatestConversationReplyRender(p, bindings, 5_000, 50)).toBe("rendered"); }); }); + +// Full cohort 3 (fast-responses, choose-primary, set-budget): the run was recorded finished 0.2-1.3s before its +// reply comment was saved, so a link read right after the run finished saw no reply at all. +describe("waits for a finished run's reply to be saved, then rendered, before its links are read", () => { + const page = () => ({ evaluate: async (read: (arg: A) => T, arg: A) => read(arg) }) as unknown as Parameters[0]; + it("waits for the reply saved after the run finished and for its render", async () => { + const saved = input().comments, before = saved.filter(row => row.id !== "reply"); + document.querySelector('#comment-reply')!.remove(); + let reads = 0; const comments = async () => (reads++ < 3 ? before : saved); + setTimeout(() => document.querySelector('[data-testid="task-chat-scroller"]')!.insertAdjacentHTML("beforeend", ''), 400); + expect(await waitForSavedTurnReplyRender(page(), { comments, leadId: "ceo", runIds: ["run"] }, 3_000, 50)).toBe("rendered"); + expect(reads).toBeGreaterThan(3); + }); + it("reports a run that saved no reply in time, without waiting past its bound", async () => { + const before = input().comments.filter(row => row.id !== "reply"); + const started = Date.now(); + expect(await waitForSavedTurnReplyRender(page(), { comments: async () => before, leadId: "ceo", runIds: ["run"] }, 300, 50)).toBe("unsaved"); + expect(Date.now() - started).toBeLessThan(1_500); + }); +}); diff --git a/tests/runner-e2e/chat-rendered-links.ts b/tests/runner-e2e/chat-rendered-links.ts index 752818dee93..af2766ad075 100644 --- a/tests/runner-e2e/chat-rendered-links.ts +++ b/tests/runner-e2e/chat-rendered-links.ts @@ -137,6 +137,26 @@ export async function waitForLatestConversationReplyRender(page: Pick setTimeout(resolve, intervalMs)); } } +/** A finished run's reply comment can be saved shortly after the run itself is recorded as finished, and the open + * conversation renders it later still. Before acting on a turn's rendered links, wait (bounded) for each finished + * lead run's saved reply, then for the latest to render. "unsaved" means no run saved a reply in time. */ +export const TURN_REPLY_SAVE_WAIT_MS = 15_000; +export async function waitForSavedTurnReplyRender(page: Pick, input: { + comments: () => Promise; leadId: string; runIds: readonly string[]; +}, timeoutMs = TURN_REPLY_SAVE_WAIT_MS, intervalMs = 250): Promise { + const deadlineAt = Date.now() + timeoutMs; + for (let attempt = 0, attempts = Math.max(1, Math.ceil(timeoutMs / intervalMs)); ; attempt++) { + const rows = await input.comments(); + const replies = rows.filter(row => row.authorAgentId === input.leadId && input.runIds.includes(String(row.createdByRunId))); + const saved = input.runIds.length > 0 && input.runIds.every(id => replies.some(row => row.createdByRunId === id)); + if (saved || attempt + 1 >= attempts || Date.now() >= deadlineAt) { + if (!replies.length) return "unsaved"; + return waitForLatestConversationReplyRender(page, { anchors: rows.map(row => String(row.id)), replies: replies.map(row => String(row.id)) }, + Math.max(CONVERSATION_REPLY_RENDER_WAIT_MS, deadlineAt - Date.now()), intervalMs); + } + await new Promise(resolve => setTimeout(resolve, intervalMs)); + } +} export async function collectConversationLinks(page: Page, input: ConversationLinkInput) { const bindings = conversationLinkBindings(input); await waitForLatestConversationReplyRender(page, bindings);