test(evals): grade rendered question context text

Co-Authored-By: Paperclip <noreply@paperclip.ing>

tests/runner-e2e/chat-catalog-domains/identity-questions-and-approvals.ts

@@ -56,8 +56,10 @@ async function observeQuestion(ctx: ChatCatalogContext, state: Record<string, un
       if (!await row.isVisible()) continue;
       await row.click();
       const context = ctx.input.page.getByTestId("task-chat-composer-takeover").getByRole("region", { name: "Question context", exact: true });
-      if (await context.getByText(description, { exact: true }).isVisible())
-        (state.visibleContinuationDescriptions as Array<{ id: string; description: string }>).push({ id: card.id, description });
+      if (await context.isVisible())
+        (state.visibleContinuationDescriptions as Array<{ id: string; rawDescription: string; visibleText: string }>).push({
+          id: card.id, rawDescription: description, visibleText: await context.innerText(),
+        });
     }
   }
   const brief = observed.tasks.find(row => `${row.title}\n${row.description}`.includes(String(state.answer)));
@@ -116,9 +118,10 @@ export function gradeQuestions(value: unknown) {
       && answers.some(row => row.questionId === "headline" && row.otherText === o.state.answer), "The submitted choice/choice/custom-text set preserves earlier pages.");
     c.add("no-default-consent", o.state.unsubmittedPending === true && o.state.validationObserved === true && o.state.retainedAudience === true,
       "Missing required input cannot submit, unfinished pages stay pending, and back navigation retains the first selection.");
-    const visible = o.state.visibleContinuationDescriptions as Array<{ id: string; description: string }> | undefined;
+    const visible = o.state.visibleContinuationDescriptions as Array<{ id: string; rawDescription: string; visibleText: string }> | undefined;
     const summary = [String(o.state.afterAnswerReply), ...multipartContinuationCards(o).flatMap(card =>
-      visible?.some(row => row.id === card.id && row.description === card.payload.questionSet?.description) ? [card.payload.questionSet!.description!] : [])].join("\n");
+      visible?.filter(row => row.id === card.id && row.rawDescription === card.payload.questionSet?.description)
+        .map(row => row.visibleText) ?? [])].join("\n");
     c.add("submitted-set-only-continuation", o.runs.filter(run => run.contextSnapshot?.interactionId === original?.id).length === 1
       && /researchers/i.test(summary) && /clear/i.test(summary) && summary.includes(String(o.state.answer)),
       "The submitted set causes one request-bound continuation whose actual later visible chat response accurately summarizes all three answers.");

tests/runner-e2e/chat-catalog-domains/identity.test.ts

@@ -155,12 +155,22 @@ describe("multipart continuation on the visible question surface", () => {
     o.interactions.push({ id: "later", companyId: "company", issueId: "source", createdByAgentId: "ceo", sourceRunId: o.runs[1]!.id,
       kind: "ask_user_questions", status: "pending", payload: { questionSet: { description, questions: [] } } });
     o.state.afterAnswerReply = "";
-    o.state.visibleContinuationDescriptions = [{ id: "later", description }];
+    o.state.visibleContinuationDescriptions = [{ id: "later", rawDescription: description, visibleText: description }];
     return o;
   }
   it("accepts the actual visible bound card summary without an extra comment", () => {
     expect(gradeQuestions(cardSummary()).passed).toBe(true);
   });
+  it("uses rendered Markdown text rather than its source markup", () => {
+    const o = cardSummary(); const card = o.interactions.at(-1)!;
+    card.payload.questionSet!.description = "**Researchers**\n\nTone: `Clear`; headline: **Acorn 42**.";
+    o.state.visibleContinuationDescriptions = [{ id: card.id, rawDescription: card.payload.questionSet!.description,
+      visibleText: "Researchers\n\nTone: Clear; headline: Acorn 42." }];
+    expect(gradeQuestions(o).passed).toBe(true);
+    o.state.visibleContinuationDescriptions = [{ id: card.id, rawDescription: card.payload.questionSet!.description,
+      visibleText: "Researchers; tone: Clear" }];
+    expect(gradeQuestions(o).passed).toBe(false);
+  });
   it.each(["hidden", "wrong-run", "wrong-issue", "wrong-agent", "historical", "missing-answer"])("rejects %s card evidence", mode => {
     const o = cardSummary(); const card = o.interactions.at(-1)!;
     if (mode === "hidden") o.state.visibleContinuationDescriptions = [];
@@ -170,7 +180,7 @@ describe("multipart continuation on the visible question surface", () => {
     if (mode === "historical") card.id = "question";
     if (mode === "missing-answer") {
       card.payload.questionSet!.description = "Researchers, Clear";
-      o.state.visibleContinuationDescriptions = [{ id: card.id, description: card.payload.questionSet!.description }];
+      o.state.visibleContinuationDescriptions = [{ id: card.id, rawDescription: card.payload.questionSet!.description, visibleText: card.payload.questionSet!.description }];
     }
     expect(gradeQuestions(o).passed).toBe(false);
   });