From be153175a55dbaeecaead3c64006060df5295b12 Mon Sep 17 00:00:00 2001 From: Dotta Date: Wed, 21 Oct 2026 09:09:02 -0500 Subject: [PATCH] test: open and inspect the delivered question result Co-Authored-By: Paperclip --- tests/runner-e2e/continuation-flow.ts | 20 ++++++++++++++++++-- tests/runner-e2e/continuation-scoring.ts | 2 +- tests/runner-e2e/necessary-question.test.ts | 9 +++++++-- tests/runner-e2e/necessary-question.ts | 8 +++++--- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/runner-e2e/continuation-flow.ts b/tests/runner-e2e/continuation-flow.ts index f8c96c7c57c..e667575ea90 100644 --- a/tests/runner-e2e/continuation-flow.ts +++ b/tests/runner-e2e/continuation-flow.ts @@ -181,8 +181,24 @@ export async function runContinuationFlow(input: { const question = interactions.find(interaction => interaction.kind === "ask_user_questions" && interaction.status === "pending"); checkpoints.at(-1)!.questionPresentation = { interactionId: question?.id ?? "", promptVisible: true, contextVisible: true }; } else if (phase === "final") { - await expect(page.locator('a[href*="#document-"]').last()).toBeVisible({ timeout: 30_000 }); - checkpoints.at(-1)!.outputLinkVisible = true; + expect(documents).toHaveLength(1); + const document = documents[0]!; + expect(document.latestRevisionId).toBeTruthy(); + const href = `/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue!.identifier ?? issue!.id)}#document-${encodeURIComponent(document.key)}`; + const url = new URL(href, page.url()).href; + const link = page.getByTestId("task-chat-agent-bubble").locator(`a[href=${JSON.stringify(href)}]`).last(); + await expect(link).toBeVisible({ timeout: 30_000 }); + await link.click(); + await expect(page).toHaveURL(url); + const target = page.locator([ + `[id=${JSON.stringify(`document-${document.key}`)}]:visible`, + `[id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`, + ].join(", ")); + await expect(target).toHaveCount(1); + await expect(target).toBeVisible(); + const visibleBody = `Booking confirmed: ${scenario.answer}`; + await expect(target).toContainText(visibleBody); + checkpoints.at(-1)!.outputDocument = { issueId: issue!.id, key: document.key, revisionId: document.latestRevisionId, href, visibleBody }; } } await captureLoadedContinuation(page, String(issue!.title), () => input.capture( diff --git a/tests/runner-e2e/continuation-scoring.ts b/tests/runner-e2e/continuation-scoring.ts index 3c8eb1d7f7c..c056b7a43ce 100644 --- a/tests/runner-e2e/continuation-scoring.ts +++ b/tests/runner-e2e/continuation-scoring.ts @@ -15,7 +15,7 @@ export interface ContinuationCheckpoint { attachments: unknown[]; comments: unknown[]; questionPresentation?: { interactionId: string; promptVisible: boolean; contextVisible: boolean }; - outputLinkVisible?: boolean; + outputDocument?: { issueId: string; key: string; revisionId: string; href: string; visibleBody: string }; interactions: unknown[]; runs: Array<{ id: string; status: string; runtimeMode?: string }>; } diff --git a/tests/runner-e2e/necessary-question.test.ts b/tests/runner-e2e/necessary-question.test.ts index 47c9f48a2d9..b3e32a6ae10 100644 --- a/tests/runner-e2e/necessary-question.test.ts +++ b/tests/runner-e2e/necessary-question.test.ts @@ -22,12 +22,13 @@ function recording(provider = false): any[] { runs: [run], interactions: [card], documents: [], children: [], attachments: [], comments: [], questionPresentation: { interactionId: "card", promptVisible: true, contextVisible: true } }; const final = structuredClone(initial) as Row; - final.phase = "final"; final.issue.status = "done"; final.lifecycle.executionRunId = null; final.outputLinkVisible = true; + final.phase = "final"; final.issue.status = "done"; final.lifecycle.executionRunId = null; final.interactions[0] = { ...final.interactions[0], status: "answered", resolvedByUserId: "local-board", result: { answers: [{ questionId: "reference", optionIds: [], otherText: scenario.answer }] } }; if (provider) { final.runs[0].status = "succeeded"; final.runs[0].nativePhase = "committed"; final.runs[0].resultJson.finalizationPhase = "committed"; } else final.runs.push({ ...structuredClone(run), id: "second", contextSnapshot: { issueId: "task", sourceRunId: "first", interactionId: "card", interactionKind: "ask_user_questions", interactionStatus: "answered" } }); + final.outputDocument = { issueId: "task", key: "confirmation", revisionId: "revision", href: "/QA/issues/task#document-confirmation", visibleBody: `Booking confirmed: ${scenario.answer}` }; final.documents = [{ key: "confirmation", body: `Booking confirmed: ${scenario.answer}`, latestRevisionId: "revision" }]; return [structuredClone(initial), final]; } @@ -61,7 +62,11 @@ describe("necessary free-text question journey", () => { ["wrong wake binding", (points: Row[]) => { points[1].runs[1].contextSnapshot.interactionId = "foreign"; }], ["uncommitted result", (points: Row[]) => { points[1].runs[1].resultJson.finalizationPhase = "pending"; }], ["wrong saved result", (points: Row[]) => { points[1].documents[0].body = "Claimed complete"; }], - ["inaccessible result", (points: Row[]) => { points[1].outputLinkVisible = false; }], + ["inaccessible result", (points: Row[]) => { delete points[1].outputDocument; }], + ["foreign linked task", (points: Row[]) => { points[1].outputDocument.issueId = "foreign"; }], + ["different linked document", (points: Row[]) => { points[1].outputDocument.key = "other"; }], + ["stale linked revision", (points: Row[]) => { points[1].outputDocument.revisionId = "older"; }], + ["wrong displayed content", (points: Row[]) => { points[1].outputDocument.visibleBody = "Claimed complete"; }], ["duplicate question", (points: Row[]) => { points[1].interactions.push(points[1].interactions[0]); }], ] as const)("rejects %s", (_name, change) => { const points = recording(); change(points); expect(failures(points).length).toBeGreaterThan(0); }); it("does not put the required answer in the original request", () => { diff --git a/tests/runner-e2e/necessary-question.ts b/tests/runner-e2e/necessary-question.ts index 98447a5a0f9..bc6309fbae0 100644 --- a/tests/runner-e2e/necessary-question.ts +++ b/tests/runner-e2e/necessary-question.ts @@ -21,7 +21,7 @@ export function gradeNecessaryQuestion(checkpoints: ContinuationCheckpoint[], an const pending = cards(initial), finished = cards(final), question = pending[0], resolved = finished[0]; const field = question?.payload?.questionSet?.questions?.[0], contract = necessaryQuestionContract(); const check = (id: string, passed: boolean, detail: string) => ({ id, passed: Boolean(passed), detail }); - const response = resolved?.result?.answers; + const response = resolved?.result?.answers, document = final?.documents[0], navigation = final?.outputDocument; return [ check("necessary-question.form", checkpoints.length === 2 && !!initial && pending.length === 1 && question?.status === "pending" && initial.documents.length === 0 && initial.attachments.length === 0 && validResumeQuestionForm(initial, question, false) && field?.required === true && field.prompt === contract.prompt && @@ -39,8 +39,10 @@ export function gradeNecessaryQuestion(checkpoints: ContinuationCheckpoint[], an final.runs.every((run: Row) => run.status === "succeeded" && run.runtimeMode === "native" && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed"), "The answer must resume its bound provider run or admit exactly one bound semantic successor, without duplicate work."), check("necessary-question.result", final?.issue.status === "done" && final.documents.length === 1 && - !!final.documents[0]?.latestRevisionId && final.documents[0]?.body.trim() === `Booking confirmed: ${answer}` && final.outputLinkVisible === true, - "The saved confirmation must contain the exact answer and have an accessible link in the rendered task."), + !!document?.latestRevisionId && document?.body.trim() === `Booking confirmed: ${answer}` && + navigation?.issueId === final.issue.id && navigation?.key === document.key && navigation?.revisionId === document.latestRevisionId && + navigation?.visibleBody === document.body.trim(), + "The final reply must open this task's saved confirmation and show its exact answer."), ]; }