Fix rendered Markdown reply evidence in product journeys

Match full visible content and exact link destinations while retaining source Markdown and native run attribution. Preserve the original browser regression and strengthen decline source checks.

Co-Authored-By: Paperclip <noreply@paperclip.ing>

package.json

@@ -101,6 +101,7 @@
     "agentmail": "^0.5.31",
     "cross-env": "^10.1.0",
     "esbuild": "^0.28.2",
+    "mdast-util-from-markdown": "2.0.3",
     "typescript": "^7.0.2",
     "vitest": "^5.0.3",
     "yaml": "^2.8.1",

pnpm-lock.yaml

@@ -68,6 +68,9 @@ importers:
       esbuild:
         specifier: ^0.28.2
         version: 0.28.2
+      mdast-util-from-markdown:
+        specifier: 2.0.3
+        version: 2.0.3
       turbo:
         specifier: 2.11.10
         version: 2.11.10

tests/runner-e2e/PRODUCT-JOURNEYS.md

@@ -115,6 +115,13 @@ satisfy the check. A pending or suppressed comment can use the same proven nativ
 reply path. A finish summary by itself, an earlier turn, a progress
 bubble, or a cancelled run cannot satisfy it. Private receipts record the run,
 thread anchor and observed text; classic protocol comment matchers stay strict.
+Markdown replies compare complete text and exact link destinations after parsing
+CommonMark. Whitespace and inline formatting can change during rendering. A
+document or issue chip can replace a link label, but must retain its destination.
+Receipts retain both source Markdown and observed content. Unsupported constructs
+such as images, raw HTML and GFM-only formatting fail closed. Exact marker checks
+remain exact, and decline checks inspect both visible explanations and source
+text for unread service data.
 The decline oracle accepts this explicit native receipt without inventing an
 issue comment. The final run must start after the saved decline, and the usual
 truthful explanation, zero service calls and no unread marker checks still apply.

tests/runner-e2e/connection-guidance-evidence.ts

@@ -1,5 +1,6 @@
 import type { StoryComment } from "./everyday-observations.js";
 import type { BoundTaskFinalReply } from "./native-task-reply.js";
+import { taskReplyContentMatches } from "./task-reply-content.js";
 
 type Decision = {
   id: string; kind: string; status: string; resolvedAt?: string | null;
@@ -51,7 +52,7 @@ function declineReplyEvidence(input: ConnectionGuidanceDeclineInput) {
   const native = input.nativeFinalReply;
   const nativeAttributed = input.caseId === "service-decline" && validDecision && native?.source === "native-turn"
     && native.agentId === input.leadAgentId && native.observedAfterReload && native.visible && native.settled && native.nativeFinal
-    && native.finalEvent?.body === native.body
+    && native.finalEvent?.body === (native.sourceBody ?? native.body) && taskReplyContentMatches(native, native.finalEvent.body)
     && (native.anchor === `${native.runId}:turn` || native.anchor?.startsWith(`${native.runId}:turn:`) === true)
     && input.runs.some(run => run.id === native.runId && run.agentId === input.leadAgentId && run.nativeIssueId === input.issueId
       && run.runtimeMode === "native" && run.status === "succeeded" && Date.parse(run.startedAt ?? "") >= resolvedAt
@@ -59,7 +60,8 @@ function declineReplyEvidence(input: ConnectionGuidanceDeclineInput) {
   // Keep native evidence explicit. It is a saved final-message/DOM receipt,
   // never a synthetic issue comment or a finish-summary fallback.
   return { validDecision, afterDecision: [...afterDecision, ...(nativeAttributed ? [native!] : [])],
-    attributed: [...attributed, ...(nativeAttributed ? [native!] : [])] };
+    attributed: [...attributed, ...(nativeAttributed ? [native!] : [])],
+    nativeSourceBody: nativeAttributed ? native!.finalEvent!.body : null };
 }
 
 /** Readiness waits for storage, not a passing explanation. Wrong wording must still fail grading. */
@@ -74,15 +76,17 @@ export function explainsConnectionUnavailable(text: string): boolean {
 }
 
 export function gradeConnectionGuidanceDecline(input: ConnectionGuidanceDeclineInput) {
-  const { validDecision, afterDecision, attributed } = declineReplyEvidence(input);
+  const { validDecision, afterDecision, attributed, nativeSourceBody } = declineReplyEvidence(input);
   return [
     { id: "guidance-decline-decision", passed: validDecision && input.decisions.length === 1,
       detail: "Exactly one correctly typed, resolved decline belongs to the selected decision." },
     { id: "guidance-decline-attributed-explanation",
-      passed: attributed.some(reply => typeof reply.body === "string" && explainsConnectionUnavailable(reply.body)),
+      passed: attributed.some(reply => typeof reply.body === "string" && explainsConnectionUnavailable(reply.body))
+        && (!input.nativeFinalReply || explainsConnectionUnavailable(input.nativeFinalReply.body)),
       detail: "A saved explanation follows the decision and joins to the final successful task run; native task replies also require the reloaded final-message receipt." },
     { id: "guidance-decline-no-use", passed: (input.caseId === "connection-decline" || input.calls === 0) && input.sameConnections &&
-        afterDecision.every(reply => (typeof reply.body !== "string" || !reply.body.includes(input.marker))),
+        afterDecision.every(reply => (typeof reply.body !== "string" || !reply.body.includes(input.marker)))
+        && (nativeSourceBody === null || !nativeSourceBody.includes(input.marker)),
       detail: "No connection changes or unread marker in replies. Installed-service/provider declines also require an observed zero fixture-call count; Notion setup does not execute a service." },
   ];
 }

tests/runner-e2e/connection-guidance.test.ts

@@ -62,6 +62,17 @@ describe("neutral decline evidence", () => {
     expect(passes(input)).toBe(true);
     expect(input.replies).toEqual([]);
   });
+  it("checks both the visible explanation and preserved native Markdown when links become chips", () => {
+    const input = native();
+    const body = "Contacts are unavailable. See [Connection outcome](/QA/issues/QA-123#document-outcome).";
+    Object.assign(input.nativeFinalReply, { sourceBody: body, body: "Contacts are unavailable. See QA-123.", origin: "https://fixture.test",
+      content: [{ text: "Contacts are unavailable. See " }, { href: "/QA/issues/QA-123#document-outcome" }, { text: "." }] });
+    input.nativeFinalReply.finalEvent.body = body;
+    expect(passes(input)).toBe(true);
+    Object.assign(input.nativeFinalReply, { sourceBody: body.replace("Connection outcome", "PRIVATE_MARKER") });
+    input.nativeFinalReply.finalEvent.body = body.replace("Connection outcome", "PRIVATE_MARKER");
+    expect(passes(input)).toBe(false);
+  });
   it("excludes an earlier successful native reply when a later successful run exists", () => {
     const input = native();
     input.runs.push({ ...input.runs[0], id: "latest", startedAt: "2026-10-06T22:00:04Z", finishedAt: "2026-10-06T22:00:06Z" });

tests/runner-e2e/native-task-reply.spec.ts

@@ -0,0 +1,29 @@
+import { expect, test } from "@playwright/test";
+import { readBoundTaskFinalReply } from "./native-task-reply.js";
+
+const markdown = "I **couldn't retrieve** the pages. See [Connection outcome](/QA/issues/QA-123#document-outcome).";
+const rendered = "I <strong>couldn't retrieve</strong> the pages. See <a href='/QA/issues/QA-123#document-outcome'><svg aria-hidden='true'></svg>QA-123</a>.";
+const run = { id: "final-run", companyId: "company", agentId: "agent", status: "succeeded", runtimeMode: "native",
+  resultJson: { presentationDecision: { commentId: "selected" } } };
+const comment = { id: "selected", authorAgentId: "agent", createdByRunId: run.id, body: markdown };
+
+for (const [name, content, passes] of [
+  ["decorated document link and bold text", rendered, true],
+  ["wrong destination", rendered.replace("#document-outcome", "#document-other"), false],
+  ["missing link", rendered.replace(/<a.*?<\/a>/, "QA-123"), false],
+  ["extra visible text", rendered + " Extra.", false],
+  ["hidden source text", rendered.replace("couldn't retrieve", "<span hidden>couldn't retrieve</span>"), false],
+] as const) test(`reloaded task reply: ${name}`, async ({ page }) => {
+  await page.route("https://fixture.test/task", route => route.fulfill({ contentType: "text/html", body:
+    `<div data-testid='task-chat-thread'><div data-thread-anchor='selected'><div data-testid='task-chat-agent-bubble'>${content}</div></div></div>` }));
+  await page.goto("https://fixture.test/task");
+  const result = readBoundTaskFinalReply({ page, runId: run.id, agentId: "agent", deadlineAt: Date.now() + (passes ? 3000 : 300),
+    load: async () => ({ run, comments: [comment], events: [] }) });
+  if (passes) {
+    const receipt = await result;
+    expect(receipt.source).toBe("posted-comment");
+    expect(receipt.sourceBody).toBe(markdown);
+    expect(receipt.body).toBe("I couldn't retrieve the pages. See QA-123.");
+    expect(receipt.observedAfterReload).toBe(true);
+  } else await expect(result).rejects.toThrow();
+});

tests/runner-e2e/native-task-reply.test.ts

@@ -73,3 +73,18 @@ it("accepts a proven native final before the selected comment is projected, and
     expect(taskReplyBelongsToRun(observed, pendingProjection, "owned-agent", [])).toBe(false);
   }
 });
+
+it("binds complete Markdown content and the exact link destination despite a decorated issue label", () => {
+  const body = "I **couldn't retrieve** the pages. See [Connection outcome](/QA/issues/QA-123#document-outcome).";
+  const comment = { id: "selected", authorAgentId: "owned-agent", createdByRunId: "final-run", body };
+  const published = { ...run, resultJson: { presentationDecision: { commentId: "selected" } } };
+  const bubble: TaskReplyObservation = { ...observed, anchor: "selected", origin: "https://fixture.test",
+    body: "I couldn't retrieve the pages. See QA-123.",
+    content: [{ text: "I couldn't retrieve the pages. See " }, { href: "https://fixture.test/QA/issues/QA-123#document-outcome" }, { text: "." }] };
+  expect(taskReplyBelongsToRun(bubble, published, "owned-agent", [comment])).toBe(true);
+  const copy = events(); copy[0]!.payload.prpEvent.payload.text = body;
+  expect(taskReplyBelongsToRun({ ...bubble, anchor: "final-run:turn" }, run, "owned-agent", [], copy)).toBe(true);
+  for (const content of [undefined, [{ text: bubble.body }], [...bubble.content!, { text: " extra" }],
+    [{ text: "I couldn't retrieve the pages. See " }, { href: "/other-task" }, { text: "." }]])
+    expect(taskReplyBelongsToRun({ ...bubble, content }, published, "owned-agent", [comment])).toBe(false);
+});

tests/runner-e2e/native-task-reply.ts

@@ -1,11 +1,13 @@
 import { expect, type Page } from "@playwright/test";
 import { isValidNativePrpEnvelope } from "./native-event-envelope.js";
+import { taskReplyContentMatches, type ReplyContentPart } from "./task-reply-content.js";
 
 type Row = Record<string, any>;
 export interface TaskReplyObservation {
   body: string; anchor: string | null; settled: boolean; nativeFinal: boolean; visible: boolean;
+  content?: ReplyContentPart[]; origin?: string;
 }
-export type BoundTaskFinalReply = Awaited<ReturnType<typeof readBoundTaskFinalReply>>;
+export type BoundTaskFinalReply = Omit<Awaited<ReturnType<typeof readBoundTaskFinalReply>>, "sourceBody"> & { sourceBody?: string };
 
 /** Native task replies can render from a saved turn before a completion comment
  * is projected. Respect an explicit presentation decision when one exists. */
@@ -36,10 +38,10 @@ export function taskReplyBelongsToRun(observation: TaskReplyObservation, run: Ro
   const projected = typeof decision?.commentId === "string" ? comments.find(comment => comment.id === decision.commentId) : null;
   if (projected) {
     return observation.anchor === projected.id && projected.authorAgentId === agentId
-      && projected.createdByRunId === run.id && typeof projected.body === "string" && projected.body.trim() === observation.body;
+      && projected.createdByRunId === run.id && typeof projected.body === "string" && taskReplyContentMatches(observation, projected.body);
   }
   const finals = nativeFinalReplyEvents(events, run);
-  return finals.length === 1 && finals[0]!.body === observation.body && observation.settled && observation.nativeFinal
+  return finals.length === 1 && taskReplyContentMatches(observation, finals[0]!.body) && observation.settled && observation.nativeFinal
     && (observation.anchor === `${run.id}:turn` || observation.anchor?.startsWith(`${run.id}:turn:`) === true);
 }
 
@@ -50,6 +52,34 @@ export function uniqueBoundTaskReply(all: TaskReplyObservation[], run: Row, agen
     ? matches[0]! : null;
 }
 
+export function observeTaskReplyElements(elements: Array<HTMLElement | SVGElement>): TaskReplyObservation[] {
+  return elements.map(element => {
+    const content: Array<{ text: string } | { href: string }> = [];
+    const walk = (node: Node) => {
+      if (node.nodeType === Node.TEXT_NODE) { content.push({ text: node.textContent ?? "" }); return; }
+      if (!(node instanceof HTMLElement) || node.getAttribute("aria-hidden") === "true"
+        || getComputedStyle(node).display === "none" || getComputedStyle(node).visibility === "hidden") return;
+      if (node instanceof HTMLAnchorElement) {
+        if (node.getClientRects().length && (node.innerText.trim() || node.getAttribute("aria-label"))) content.push({ href: node.href });
+        return;
+      }
+      const block = ["P", "DIV", "BR", "LI", "PRE", "BLOCKQUOTE", "H1", "H2", "H3", "H4", "H5", "H6"].includes(node.tagName);
+      if (block) content.push({ text: " " });
+      node.childNodes.forEach(walk);
+      if (block) content.push({ text: " " });
+    };
+    walk(element);
+    return {
+      body: (element as HTMLElement).innerText.trim(),
+      content, origin: location.origin,
+      anchor: element.closest("[data-thread-anchor]")?.getAttribute("data-thread-anchor") ?? null,
+      settled: element.closest('[data-testid="task-chat-turn"]')?.getAttribute("data-settled") === "true",
+      nativeFinal: element.closest('[data-testid="task-chat-final-response"]') !== null,
+      visible: element.getClientRects().length > 0 && getComputedStyle(element).visibility !== "hidden" && getComputedStyle(element).display !== "none",
+    };
+  });
+}
+
 /** Observe the real task thread after a fresh load, rather than a finish summary
  * or a provider's claimed result. The caller separately validates task/run scope. */
 export async function readBoundTaskFinalReply(input: {
@@ -58,13 +88,7 @@ export async function readBoundTaskFinalReply(input: {
 }) {
   await input.page.reload();
   const replies = input.page.getByTestId("task-chat-thread").getByTestId("task-chat-agent-bubble");
-  const observe = () => replies.evaluateAll(elements => elements.map(element => ({
-    body: (element as HTMLElement).innerText.trim(),
-    anchor: element.closest("[data-thread-anchor]")?.getAttribute("data-thread-anchor") ?? null,
-    settled: element.closest('[data-testid="task-chat-turn"]')?.getAttribute("data-settled") === "true",
-    nativeFinal: element.closest('[data-testid="task-chat-final-response"]') !== null,
-    visible: element.getClientRects().length > 0 && getComputedStyle(element).visibility !== "hidden" && getComputedStyle(element).display !== "none",
-  })));
+  const observe = () => replies.evaluateAll(observeTaskReplyElements);
   const read = async () => {
     const state = await input.load(), all = await observe();
     const match = state.run.id === input.runId ? uniqueBoundTaskReply(all, state.run, input.agentId, state.comments, input.expectedText, state.events) : null;
@@ -75,9 +99,12 @@ export async function readBoundTaskFinalReply(input: {
   await expect.poll(async () => {
     return Boolean((await read()).match);
   }, { timeout, intervals: [1000, 2000, 5000] }).toBe(true);
-  const { match, run, events } = await read();
+  const { match, run, events, comments } = await read();
   if (!match) throw new Error("native_reply_changed");
   const source = match.anchor === run.resultJson?.presentationDecision?.commentId ? "posted-comment" : "native-turn";
+  const finalEvent = source === "native-turn" ? nativeFinalReplyEvents(events, run)[0] : null;
+  const sourceBody = source === "posted-comment" ? comments.find(comment => comment.id === match.anchor)?.body : finalEvent?.body;
+  if (typeof sourceBody !== "string" || !taskReplyContentMatches(match, sourceBody)) throw new Error("native_reply_source_changed");
   return { runId: input.runId, agentId: input.agentId, observedAfterReload: true,
-    source, finalEvent: source === "native-turn" ? nativeFinalReplyEvents(events, run)[0] : null, ...match };
+    source, sourceBody, finalEvent, ...match };
 }

tests/runner-e2e/playwright-support.config.ts

@@ -3,7 +3,7 @@ import { defineConfig } from "@playwright/test";
 /** Browser-only harness regressions: no Paperclip instance or provider credentials. */
 export default defineConfig({
   testDir: ".",
-  testMatch: ["screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts"],
+  testMatch: ["native-task-reply.spec.ts", "screenshot-readiness.spec.ts", "service-worker-reload.spec.ts", "lost-send.spec.ts", "chat-restart.spec.ts", "settings-toggle.spec.ts", "browser-bootstrap-diagnostics.spec.ts", "service-worker-module-loading.spec.ts", "browser-bootstrap-recovery.spec.ts", "blocker-input.spec.ts", "native-blocker-visible.spec.ts", "copilot-permission-card.spec.ts", "github-review-check.spec.ts", "antigravity-journey-browser.spec.ts"],
   workers: 1,
   retries: 0,
   timeout: 10_000,

tests/runner-e2e/task-reply-content.test.ts

@@ -0,0 +1,14 @@
+import { expect, it } from "vitest";
+import { taskReplyContentMatches } from "./task-reply-content.js";
+
+it("parses CommonMark escapes, entities, inline formatting and paragraph boundaries", () => {
+  expect(taskReplyContentMatches({ body: "Couldn't retrieve a_b & c. No retry." },
+    "**Couldn't retrieve** `a_b` & c.\n\nNo _retry_.")).toBe(true);
+  expect(taskReplyContentMatches({ body: "Couldn't retrieve a_b & c. No retry. Extra." },
+    "**Couldn't retrieve** `a_b` & c.\n\nNo _retry_.")).toBe(false);
+});
+
+it("rejects unsupported Markdown rather than stripping unknown or invisible content", () => {
+  for (const source of ["Visible <!-- hidden -->", "Visible ![image](https://fixture.test/image.png)", "~~Visible~~"])
+    expect(taskReplyContentMatches({ body: "Visible" }, source)).toBe(false);
+});

tests/runner-e2e/task-reply-content.ts

@@ -0,0 +1,48 @@
+import { fromMarkdown } from "mdast-util-from-markdown";
+
+export type ReplyContentPart = { text: string } | { href: string };
+type MarkdownNode = { type: string; value?: string; url?: string; children?: MarkdownNode[] };
+
+function normalized(parts: ReplyContentPart[], origin: string) {
+  const result: ReplyContentPart[] = [];
+  for (const part of parts) {
+    if ("href" in part) {
+      try { result.push({ href: new URL(part.href, origin).href }); } catch { return null; }
+    } else {
+      const text = part.text.replace(/\s+/g, " ");
+      const previous = result.at(-1);
+      if (previous && "text" in previous) previous.text = (previous.text + text).replace(/\s+/g, " ");
+      else if (text) result.push({ text });
+    }
+  }
+  const first = result[0], last = result.at(-1);
+  if (first && "text" in first) first.text = first.text.trimStart();
+  if (last && "text" in last) last.text = last.text.trimEnd();
+  return result.filter(part => !("text" in part) || part.text);
+}
+
+/** Compare complete message content while allowing the UI to decorate a link
+ * with an issue/document chip. Its exact destination still has to match.
+ * Unsupported Markdown fails closed rather than stripping arbitrary syntax. */
+export function taskReplyContentMatches(
+  observation: { body: string; content?: ReplyContentPart[]; origin?: string },
+  markdown: string,
+) {
+  const parts: ReplyContentPart[] = [];
+  const inlineLabel = (node: MarkdownNode): boolean => ["text", "inlineCode", "strong", "emphasis"].includes(node.type)
+    && (node.children ?? []).every(inlineLabel);
+  function visit(node: MarkdownNode): boolean {
+    if (node.type === "text" || node.type === "inlineCode") { parts.push({ text: node.value ?? "" }); return true; }
+    if (node.type === "break") { parts.push({ text: " " }); return true; }
+    if (node.type === "link" && node.url && (node.children ?? []).every(inlineLabel)) { parts.push({ href: node.url }); return true; }
+    if (!["root", "paragraph", "heading", "blockquote", "list", "listItem", "strong", "emphasis"].includes(node.type)) return false;
+    for (const child of node.children ?? []) if (!visit(child)) return false;
+    if (!["strong", "emphasis"].includes(node.type)) parts.push({ text: " " });
+    return true;
+  }
+  if (!visit(fromMarkdown(markdown) as MarkdownNode)) return false;
+  const origin = observation.origin ?? "https://reply.invalid";
+  const expected = normalized(parts, origin);
+  const actual = normalized(observation.content ?? [{ text: observation.body }], origin);
+  return expected !== null && actual !== null && JSON.stringify(expected) === JSON.stringify(actual);
+}