test: open and inspect the delivered question result

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/continuation-flow.ts

@@ -181,8 +181,24 @@ export async function runContinuationFlow(input: {
         const question = interactions.find(interaction => interaction.kind === "ask_user_questions" && interaction.status === "pending");
         checkpoints.at(-1)!.questionPresentation = { interactionId: question?.id ?? "", promptVisible: true, contextVisible: true };
       } else if (phase === "final") {
-        await expect(page.locator('a[href*="#document-"]').last()).toBeVisible({ timeout: 30_000 });
-        checkpoints.at(-1)!.outputLinkVisible = true;
+        expect(documents).toHaveLength(1);
+        const document = documents[0]!;
+        expect(document.latestRevisionId).toBeTruthy();
+        const href = `/${encodeURIComponent(fixtures.company.issuePrefix!)}/issues/${encodeURIComponent(issue!.identifier ?? issue!.id)}#document-${encodeURIComponent(document.key)}`;
+        const url = new URL(href, page.url()).href;
+        const link = page.getByTestId("task-chat-agent-bubble").locator(`a[href=${JSON.stringify(href)}]`).last();
+        await expect(link).toBeVisible({ timeout: 30_000 });
+        await link.click();
+        await expect(page).toHaveURL(url);
+        const target = page.locator([
+          `[id=${JSON.stringify(`document-${document.key}`)}]:visible`,
+          `[id=${JSON.stringify(`side-panel-content-document:${document.key}`)}]:visible`,
+        ].join(", "));
+        await expect(target).toHaveCount(1);
+        await expect(target).toBeVisible();
+        const visibleBody = `Booking confirmed: ${scenario.answer}`;
+        await expect(target).toContainText(visibleBody);
+        checkpoints.at(-1)!.outputDocument = { issueId: issue!.id, key: document.key, revisionId: document.latestRevisionId, href, visibleBody };
       }
     }
     await captureLoadedContinuation(page, String(issue!.title), () => input.capture(

tests/runner-e2e/continuation-scoring.ts

@@ -15,7 +15,7 @@ export interface ContinuationCheckpoint {
   attachments: unknown[];
   comments: unknown[];
   questionPresentation?: { interactionId: string; promptVisible: boolean; contextVisible: boolean };
-  outputLinkVisible?: boolean;
+  outputDocument?: { issueId: string; key: string; revisionId: string; href: string; visibleBody: string };
   interactions: unknown[];
   runs: Array<{ id: string; status: string; runtimeMode?: string }>;
 }

tests/runner-e2e/necessary-question.test.ts

@@ -22,12 +22,13 @@ function recording(provider = false): any[] {
     runs: [run], interactions: [card], documents: [], children: [], attachments: [], comments: [],
     questionPresentation: { interactionId: "card", promptVisible: true, contextVisible: true } };
   const final = structuredClone(initial) as Row;
-  final.phase = "final"; final.issue.status = "done"; final.lifecycle.executionRunId = null; final.outputLinkVisible = true;
+  final.phase = "final"; final.issue.status = "done"; final.lifecycle.executionRunId = null;
   final.interactions[0] = { ...final.interactions[0], status: "answered", resolvedByUserId: "local-board",
     result: { answers: [{ questionId: "reference", optionIds: [], otherText: scenario.answer }] } };
   if (provider) { final.runs[0].status = "succeeded"; final.runs[0].nativePhase = "committed"; final.runs[0].resultJson.finalizationPhase = "committed"; }
   else final.runs.push({ ...structuredClone(run), id: "second", contextSnapshot: { issueId: "task", sourceRunId: "first",
     interactionId: "card", interactionKind: "ask_user_questions", interactionStatus: "answered" } });
+  final.outputDocument = { issueId: "task", key: "confirmation", revisionId: "revision", href: "/QA/issues/task#document-confirmation", visibleBody: `Booking confirmed: ${scenario.answer}` };
   final.documents = [{ key: "confirmation", body: `Booking confirmed: ${scenario.answer}`, latestRevisionId: "revision" }];
   return [structuredClone(initial), final];
 }
@@ -61,7 +62,11 @@ describe("necessary free-text question journey", () => {
     ["wrong wake binding", (points: Row[]) => { points[1].runs[1].contextSnapshot.interactionId = "foreign"; }],
     ["uncommitted result", (points: Row[]) => { points[1].runs[1].resultJson.finalizationPhase = "pending"; }],
     ["wrong saved result", (points: Row[]) => { points[1].documents[0].body = "Claimed complete"; }],
-    ["inaccessible result", (points: Row[]) => { points[1].outputLinkVisible = false; }],
+    ["inaccessible result", (points: Row[]) => { delete points[1].outputDocument; }],
+    ["foreign linked task", (points: Row[]) => { points[1].outputDocument.issueId = "foreign"; }],
+    ["different linked document", (points: Row[]) => { points[1].outputDocument.key = "other"; }],
+    ["stale linked revision", (points: Row[]) => { points[1].outputDocument.revisionId = "older"; }],
+    ["wrong displayed content", (points: Row[]) => { points[1].outputDocument.visibleBody = "Claimed complete"; }],
     ["duplicate question", (points: Row[]) => { points[1].interactions.push(points[1].interactions[0]); }],
   ] as const)("rejects %s", (_name, change) => { const points = recording(); change(points); expect(failures(points).length).toBeGreaterThan(0); });
   it("does not put the required answer in the original request", () => {

tests/runner-e2e/necessary-question.ts

@@ -21,7 +21,7 @@ export function gradeNecessaryQuestion(checkpoints: ContinuationCheckpoint[], an
   const pending = cards(initial), finished = cards(final), question = pending[0], resolved = finished[0];
   const field = question?.payload?.questionSet?.questions?.[0], contract = necessaryQuestionContract();
   const check = (id: string, passed: boolean, detail: string) => ({ id, passed: Boolean(passed), detail });
-  const response = resolved?.result?.answers;
+  const response = resolved?.result?.answers, document = final?.documents[0], navigation = final?.outputDocument;
   return [
     check("necessary-question.form", checkpoints.length === 2 && !!initial && pending.length === 1 && question?.status === "pending" && initial.documents.length === 0 && initial.attachments.length === 0 &&
       validResumeQuestionForm(initial, question, false) && field?.required === true && field.prompt === contract.prompt &&
@@ -39,8 +39,10 @@ export function gradeNecessaryQuestion(checkpoints: ContinuationCheckpoint[], an
       final.runs.every((run: Row) => run.status === "succeeded" && run.runtimeMode === "native" && run.nativePhase === "committed" && run.resultJson?.finalizationPhase === "committed"),
       "The answer must resume its bound provider run or admit exactly one bound semantic successor, without duplicate work."),
     check("necessary-question.result", final?.issue.status === "done" && final.documents.length === 1 &&
-      !!final.documents[0]?.latestRevisionId && final.documents[0]?.body.trim() === `Booking confirmed: ${answer}` && final.outputLinkVisible === true,
-      "The saved confirmation must contain the exact answer and have an accessible link in the rendered task."),
+      !!document?.latestRevisionId && document?.body.trim() === `Booking confirmed: ${answer}` &&
+      navigation?.issueId === final.issue.id && navigation?.key === document.key && navigation?.revisionId === document.latestRevisionId &&
+      navigation?.visibleBody === document.body.trim(),
+      "The final reply must open this task's saved confirmation and show its exact answer."),
   ];
 }