diff --git a/tests/runner-e2e/catalog.test.ts b/tests/runner-e2e/catalog.test.ts index 969b43626cc..6a37e2ab92f 100644 --- a/tests/runner-e2e/catalog.test.ts +++ b/tests/runner-e2e/catalog.test.ts @@ -357,7 +357,7 @@ describe("runner E2E catalog", () => { (total, execution) => total + execution.task.expectedRunCount, 0, ), - ).toBe(385); + ).toBe(397); expect( runnerTasks.find((task) => task.id === "plan-revise-accept") ?.attemptTimeoutMs, diff --git a/tests/runner-e2e/chat-cases.ts b/tests/runner-e2e/chat-cases.ts index 50d22658b4e..3684c003be2 100644 --- a/tests/runner-e2e/chat-cases.ts +++ b/tests/runner-e2e/chat-cases.ts @@ -15,9 +15,10 @@ export const CHAT_CASES = [ ["continuity-restart", "Conversation continuity across restart", 3], ["new-session", "Fresh context within preserved history", 2], ["stop-new-resume", "Stop, reset, and resume", 3], - ["plan-handoff", "Draft, revise, approve, and hand off a plan", 4], - ["clarify-reuse", "Clarify and reuse an existing project", 3], - ["multi-repository", "Create a project with multiple repository URLs", 2], + // Handoff cases include the source chat's one completion-reply turn. + ["plan-handoff", "Draft, revise, approve, and hand off a plan", 5], + ["clarify-reuse", "Clarify and reuse an existing project", 4], + ["multi-repository", "Create a project with multiple repository URLs", 3], ] as const; export type ChatCase = (typeof CHAT_CASES)[number][0]; const HARDENING_CASES = [ diff --git a/tests/runner-e2e/chat-flow.ts b/tests/runner-e2e/chat-flow.ts index a5a76a5a5ff..0a8808fb826 100644 --- a/tests/runner-e2e/chat-flow.ts +++ b/tests/runner-e2e/chat-flow.ts @@ -183,6 +183,12 @@ export async function readChatOutputDocument( export const isResetRun = (run: ChatRun) => run.contextSnapshot?.conversationReset === true || run.resultJson?.conversationReset === true; +/** The source chat's automatic turn that reports one completed handoff. */ +export const isChatCompletionReply = (run: ChatRun, chatIssueId: string, taskId: string) => + run.contextSnapshot?.issueId === chatIssueId && + run.contextSnapshot?.wakeReason === "chat_task_completed" && + Array.isArray(run.contextSnapshot?.chatCompletionUpdates) && + (run.contextSnapshot.chatCompletionUpdates as Array<{ id?: unknown }>).some((update) => update?.id === taskId); export function chatRunFailure( runs: ChatRun[], allowCancelled = false, @@ -944,7 +950,28 @@ export async function runChatFlow(input: ChatFlowInput) { accept: (state) => state.task.status === "done", reject: (state) => chatTaskCompletionFailure(state.task, state.runs), }); - runs = await allRuns(); + // A completed handoff returns to the source chat (#14408) as one + // correlated provider turn. Observe that delivery instead of racing it. + const completionReply = await pollUntil({ + label: `completion of ${child.id} is reported in the chat`, + deadlineAt: Date.now() + 240_000, + intervalMs: 1000, + load: async () => { + const rows = await allRuns(); + const reply = rows.filter((run) => isChatCompletionReply(run, issue!.id, child.id)); + return { runs: rows, reply, comments: await comments() }; + }, + // A native reply comment is presented after its run succeeds, so wait + // for the comment too; a failed reply turn settles immediately. + accept: (state) => state.reply.length > 0 && + state.reply.every((run) => ["succeeded", "failed", "cancelled", "timed_out"].includes(run.status)) && + (state.reply.some((run) => run.status !== "succeeded") || + state.comments.some((c) => c.authorAgentId === f.agent.id && state.reply.some((run) => run.id === c.createdByRunId))), + }); + expect(completionReply.reply.map((run) => run.status), "one completion reply turn").toEqual(["succeeded"]); + expect(completionReply.comments.filter((c) => c.authorAgentId === f.agent.id && + c.createdByRunId === completionReply.reply[0]!.id), "completion reply posted in the chat").toHaveLength(1); + runs = completionReply.runs; input.observe(issue!, runs); const plan = caseId === "plan-handoff" diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index df999ff2537..06a8b3f6f12 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -12,7 +12,7 @@ import { answerableRuntimeRunIds, isSingleClaudeQuestion } from "./runtime-quest import { expect, type Page } from "@playwright/test"; import path from "node:path"; import { continuationAnswerCommitted, continuationInitialReady, continuationCheckpointReady } from "./continuation-readiness.js"; -import { captureLoadedContinuation } from "./continuation-screenshot.js"; +import { captureLoadedContinuation, waitForTaskChatRendered } from "./continuation-screenshot.js"; import { seedContinuationContext } from "./continuation-workspace.js"; import { pollUntil, type RunnerApi } from "./api.js"; import { @@ -130,10 +130,13 @@ export async function runContinuationFlow(input: { }); } async function open() { + // A restarted dev server can leave DOMContentLoaded pending after the app + // is interactive (see chat-flow). Require the rendered task thread instead. await page.goto( `/${fixtures.company.issuePrefix}/issues/${issue!.identifier ?? issue!.id}`, - { waitUntil: "domcontentloaded" }, + { waitUntil: "commit", timeout: 60_000 }, ); + await waitForTaskChatRendered(page, undefined, 60_000); } async function snapshot(phase: ContinuationCheckpoint["phase"]) { const [tasks, summaries, comments, interactions, attachments, activity] =