From 6df66df9d94d45b9b671ae91f4401440aa626586 Mon Sep 17 00:00:00 2001 From: Dotta Date: Thu, 29 Oct 2026 23:51:32 -0500 Subject: [PATCH] Match production auto-links and require preserved reply source Use the existing GFM and issue-reference plugins with the viewer company prefixes. Keep source Markdown mandatory in native receipts and cover bare URLs, known issue keys, foreign plain keys and changed destinations. Co-Authored-By: Paperclip --- package.json | 4 +++- tests/runner-e2e/PRODUCT-JOURNEYS.md | 8 ++++--- .../connection-guidance-evidence.ts | 2 +- tests/runner-e2e/connection-guidance.test.ts | 6 +++--- tests/runner-e2e/native-task-reply.spec.ts | 21 +++++++++++++++++-- tests/runner-e2e/native-task-reply.ts | 8 +++---- .../product-journeys/agent-flows.ts | 5 ++++- tests/runner-e2e/task-reply-content.test.ts | 11 +++++++++- tests/runner-e2e/task-reply-content.ts | 19 ++++++++++------- 9 files changed, 61 insertions(+), 23 deletions(-) diff --git a/package.json b/package.json index 87959b9fe63..f1f0b4d851f 100644 --- a/package.json +++ b/package.json @@ -101,8 +101,10 @@ "agentmail": "^0.5.31", "cross-env": "^10.1.0", "esbuild": "^0.28.2", - "mdast-util-from-markdown": "2.0.3", + "remark-gfm": "4.0.1", + "remark-parse": "11.0.0", "typescript": "^7.0.2", + "unified": "11.0.5", "vitest": "^5.0.3", "yaml": "^2.8.1", "turbo": "2.11.10" diff --git a/tests/runner-e2e/PRODUCT-JOURNEYS.md b/tests/runner-e2e/PRODUCT-JOURNEYS.md index 2a23d7dbbb7..014a9ff6fe6 100644 --- a/tests/runner-e2e/PRODUCT-JOURNEYS.md +++ b/tests/runner-e2e/PRODUCT-JOURNEYS.md @@ -116,12 +116,14 @@ reply path. A finish summary by itself, an earlier turn, a progress bubble, or a cancelled run cannot satisfy it. Private receipts record the run, thread anchor and observed text; classic protocol comment matchers stay strict. Markdown replies compare complete text and exact link destinations after parsing -CommonMark. Whitespace and inline formatting can change during rendering. A +with the production GFM and issue-reference plugins. The observer receives the +same visible-company prefix list used by the UI, so bare issue keys and URLs +match their rendered links. Whitespace and inline formatting can change during rendering. A document or issue chip can replace a link label, but must retain its destination. Ordinary links retain their source label. Destination resolution uses the existing production company-route helper and the current task URL for fragments. -Receipts retain both source Markdown and observed content. Unsupported constructs -such as images, raw HTML and GFM-only formatting fail closed. Exact marker checks +Receipts retain both source Markdown and observed content. Unsupported structures +such as images, raw HTML, tables and fenced code fail closed. Exact marker checks remain exact, and decline checks inspect both visible explanations and source text for unread service data. The decline oracle accepts this explicit native receipt without inventing an diff --git a/tests/runner-e2e/connection-guidance-evidence.ts b/tests/runner-e2e/connection-guidance-evidence.ts index 08fb4e16a27..3c750678f48 100644 --- a/tests/runner-e2e/connection-guidance-evidence.ts +++ b/tests/runner-e2e/connection-guidance-evidence.ts @@ -52,7 +52,7 @@ function declineReplyEvidence(input: ConnectionGuidanceDeclineInput) { const native = input.nativeFinalReply; const nativeAttributed = input.caseId === "service-decline" && validDecision && native?.source === "native-turn" && native.agentId === input.leadAgentId && native.observedAfterReload && native.visible && native.settled && native.nativeFinal - && native.finalEvent?.body === (native.sourceBody ?? native.body) && taskReplyContentMatches(native, native.finalEvent.body) + && typeof native.sourceBody === "string" && native.finalEvent?.body === native.sourceBody && taskReplyContentMatches(native, native.finalEvent.body) && (native.anchor === `${native.runId}:turn` || native.anchor?.startsWith(`${native.runId}:turn:`) === true) && input.runs.some(run => run.id === native.runId && run.agentId === input.leadAgentId && run.nativeIssueId === input.issueId && run.runtimeMode === "native" && run.status === "succeeded" && Date.parse(run.startedAt ?? "") >= resolvedAt diff --git a/tests/runner-e2e/connection-guidance.test.ts b/tests/runner-e2e/connection-guidance.test.ts index 6beade64324..adb70c0281c 100644 --- a/tests/runner-e2e/connection-guidance.test.ts +++ b/tests/runner-e2e/connection-guidance.test.ts @@ -52,7 +52,7 @@ describe("neutral decline evidence", () => { const native = () => ({ ...structuredClone(valid), caseId: "service-decline" as const, replies: [], decisions: [{ id: "decision", kind: "request_confirmation", status: "rejected", resolvedAt: "2026-10-06T22:00:00Z" }], runs: [{ ...valid.runs[0], runtimeMode: "native", startedAt: "2026-10-06T22:00:01Z" }], - nativeFinalReply: { runId: "run", agentId: "lead", body: valid.replies[0].body, source: "native-turn", observedAfterReload: true, + nativeFinalReply: { runId: "run", agentId: "lead", body: valid.replies[0].body, sourceBody: valid.replies[0].body, source: "native-turn", observedAfterReload: true, visible: true, settled: true, nativeFinal: true, anchor: "run:turn", finalEvent: { seq: 10, sourceEventId: "final", sourceInstanceId: "runner", sourceSeq: 10, protocolSchemaVersion: 1, body: valid.replies[0].body } }, }); @@ -84,7 +84,7 @@ describe("neutral decline evidence", () => { }); it("rejects stale, unbound or untruthful native decline output and unread service data", () => { for (const change of [{ runId: "other" }, { agentId: "other" }, { source: "summary" }, { observedAfterReload: false }, - { settled: false }, { nativeFinal: false }, { visible: false }, { anchor: "other:turn" }, { finalEvent: null }]) { + { settled: false }, { nativeFinal: false }, { visible: false }, { anchor: "other:turn" }, { finalEvent: null }, { sourceBody: undefined }]) { const input = native(); Object.assign(input.nativeFinalReply, change); expect(passes(input)).toBe(false); } for (const change of [{ startedAt: "2026-10-06T21:59:00Z" }, { startedAt: "invalid" }, { runtimeMode: "legacy" }, @@ -92,7 +92,7 @@ describe("neutral decline evidence", () => { const input = native(); Object.assign(input.runs[0], change); expect(passes(input)).toBe(false); } for (const body of ["Done.", "Unavailable but PRIVATE_MARKER"]) { - const input = native(); input.nativeFinalReply.body = input.nativeFinalReply.finalEvent.body = body; + const input = native(); input.nativeFinalReply.body = input.nativeFinalReply.sourceBody = input.nativeFinalReply.finalEvent.body = body; expect(hasConnectionGuidanceDeclineReply(input)).toBe(true); expect(passes(input)).toBe(false); } for (const change of [{ calls: 1 }, { sameConnections: false }, { decisions: [] }]) expect(passes({ ...native(), ...change })).toBe(false); diff --git a/tests/runner-e2e/native-task-reply.spec.ts b/tests/runner-e2e/native-task-reply.spec.ts index b075aa115fd..558df0385d5 100644 --- a/tests/runner-e2e/native-task-reply.spec.ts +++ b/tests/runner-e2e/native-task-reply.spec.ts @@ -20,6 +20,23 @@ test("the QA tsx loader produces a self-contained browser callback", async ({ pa expect(observations[0].content.some((part: any) => part.label === "QA-123" && part.decorated)).toBe(true); }); +for (const [source, html, passes] of [ + ["Depends on QA-123.", "Depends on QA-123.", true], + ["See https://example.com/", "See https://example.com/", true], + ["Depends on TREE-604.", "Depends on TREE-604.", true], + ["Depends on QA-123.", "Depends on QA-123.", false], + ["See https://example.com/", "See https://example.com/", false], +] as const) test(`reloaded auto-linked content: ${source} (${passes})`, async ({ page }) => { + const url = "https://fixture.test/QA/issues/QA-1"; + await page.route(url, route => route.fulfill({ contentType: "text/html", body: + `
${html}
` })); + await page.goto(url); + const result = readBoundTaskFinalReply({ page, runId: run.id, agentId: "agent", knownCompanyPrefixes: ["QA"], deadlineAt: Date.now() + (passes ? 3000 : 300), + load: async () => ({ run, comments: [{ ...comment, body: source }], events: [] }) }); + if (passes) expect((await result).sourceBody).toBe(source); + else await expect(result).rejects.toThrow(); +}); + for (const [name, content, passes] of [ ["decorated document link and bold text", rendered, true], ["ordinary link with its source label", rendered.replace(" data-mention-kind='issue'", "").replace(">QA-123", ">Connection outcome"), true], @@ -32,7 +49,7 @@ for (const [name, content, passes] of [ await page.route("https://fixture.test/QA/issues/QA-123", route => route.fulfill({ contentType: "text/html", body: `
${content}
` })); await page.goto("https://fixture.test/QA/issues/QA-123"); - const result = readBoundTaskFinalReply({ page, runId: run.id, agentId: "agent", deadlineAt: Date.now() + (passes ? 3000 : 300), + const result = readBoundTaskFinalReply({ page, runId: run.id, agentId: "agent", knownCompanyPrefixes: ["QA"], deadlineAt: Date.now() + (passes ? 3000 : 300), load: async () => ({ run, comments: [comment], events: [] }) }); if (passes) { const receipt = await result; @@ -55,7 +72,7 @@ for (const [sourceHref, renderedHref, passes] of [ await page.route(url, route => route.fulfill({ contentType: "text/html", body: `
Read Outcome
` })); await page.goto(url); - const result = readBoundTaskFinalReply({ page, runId: run.id, agentId: "agent", deadlineAt: Date.now() + (passes ? 3000 : 300), + const result = readBoundTaskFinalReply({ page, runId: run.id, agentId: "agent", knownCompanyPrefixes: ["QA"], deadlineAt: Date.now() + (passes ? 3000 : 300), load: async () => ({ run, comments: [{ ...comment, body: `Read [Outcome](${sourceHref})` }], events: [] }) }); if (passes) expect((await result).pageUrl).toBe(url); else await expect(result).rejects.toThrow(); diff --git a/tests/runner-e2e/native-task-reply.ts b/tests/runner-e2e/native-task-reply.ts index 0cd7074f7c5..c9c77ead298 100644 --- a/tests/runner-e2e/native-task-reply.ts +++ b/tests/runner-e2e/native-task-reply.ts @@ -5,9 +5,9 @@ import { taskReplyContentMatches, type ReplyContentPart } from "./task-reply-con type Row = Record; export interface TaskReplyObservation { body: string; anchor: string | null; settled: boolean; nativeFinal: boolean; visible: boolean; - content?: ReplyContentPart[]; origin?: string; pageUrl?: string; + content?: ReplyContentPart[]; origin?: string; pageUrl?: string; knownCompanyPrefixes?: string[]; } -export type BoundTaskFinalReply = Omit>, "sourceBody"> & { sourceBody?: string }; +export type BoundTaskFinalReply = Awaited>; /** Native task replies can render from a saved turn before a completion comment * is projected. Respect an explicit presentation decision when one exists. */ @@ -85,14 +85,14 @@ export function observeTaskReplyElements(elements: Array; }) { await input.page.reload(); const replies = input.page.getByTestId("task-chat-thread").getByTestId("task-chat-agent-bubble"); const observe = () => replies.evaluateAll(observeTaskReplyElements); const read = async () => { - const state = await input.load(), all = await observe(); + const state = await input.load(), all = (await observe()).map(row => ({ ...row, knownCompanyPrefixes: input.knownCompanyPrefixes })); const match = state.run.id === input.runId ? uniqueBoundTaskReply(all, state.run, input.agentId, state.comments, input.expectedText, state.events) : null; return { ...state, match }; }; diff --git a/tests/runner-e2e/product-journeys/agent-flows.ts b/tests/runner-e2e/product-journeys/agent-flows.ts index d2daa20c5f9..e0bd0b14a96 100644 --- a/tests/runner-e2e/product-journeys/agent-flows.ts +++ b/tests/runner-e2e/product-journeys/agent-flows.ts @@ -97,6 +97,9 @@ export async function runAgentJourney(id: AgentJourney, context: JourneyContext, const runs = state.taskRuns.filter(run => run.status === "succeeded"); if (!runs.length || runs.some(run => !Number.isFinite(Date.parse(run.finishedAt)))) throw new JourneyError("failed", "final_run_time_missing"); const run = [...runs].sort((a, b) => Date.parse(a.finishedAt) - Date.parse(b.finishedAt)).at(-1)!; + const companies = await context.api.get("/api/companies"); + if (!Array.isArray(companies) || !companies.some(row => row.id === context.company.id) + || companies.some(row => typeof row.issuePrefix !== "string" || !row.issuePrefix)) throw new JourneyError("failed", "reply_company_context_missing"); await open(issue); const receipt = await readBoundTaskFinalReply({ page: context.page, runId: run.id, agentId: agent.id, load: async () => { @@ -104,7 +107,7 @@ export async function runAgentJourney(id: AgentJourney, context: JourneyContext, context.api.get(`/api/heartbeat-runs/${run.id}/events?afterSeq=${afterSeq}&limit=${limit}`))]); return { run: latest.taskRuns.find(row => row.id === run.id)!, comments: latest.comments, events }; }, - expectedText: marker, deadlineAt: options.deadlineAt }); + expectedText: marker, deadlineAt: options.deadlineAt, knownCompanyPrefixes: companies.map(row => row.issuePrefix) }); await retain(`${issue.id}-final-reply.json`, receipt); return receipt; }; diff --git a/tests/runner-e2e/task-reply-content.test.ts b/tests/runner-e2e/task-reply-content.test.ts index 5e08a9bf072..f802c1255f3 100644 --- a/tests/runner-e2e/task-reply-content.test.ts +++ b/tests/runner-e2e/task-reply-content.test.ts @@ -9,10 +9,19 @@ it("parses CommonMark escapes, entities, inline formatting and paragraph boundar }); it("rejects unsupported Markdown rather than stripping unknown or invisible content", () => { - for (const source of ["Visible ", "Visible ![image](https://fixture.test/image.png)", "~~Visible~~"]) + for (const source of ["Visible ", "Visible ![image](https://fixture.test/image.png)", "|Column|\n|---|\n|Visible|"]) expect(taskReplyContentMatches({ body: "Visible" }, source)).toBe(false); }); +it("uses production GFM autolinks and issue linking with the viewer's known companies", () => { + const context = { pageUrl: "https://fixture.test/QA/issues/QA-1", knownCompanyPrefixes: ["QA"] }; + expect(taskReplyContentMatches({ ...context, body: "Depends on QA-123.", content: [{ text: "Depends on " }, + { href: "/QA/issues/QA-123", label: "QA-123", decorated: true }, { text: "." }] }, "Depends on QA-123.")).toBe(true); + expect(taskReplyContentMatches({ ...context, body: "See https://example.com/", content: [{ text: "See " }, + { href: "https://example.com/", label: "https://example.com/" }] }, "See https://example.com/")).toBe(true); + expect(taskReplyContentMatches({ ...context, body: "Depends on TREE-604." }, "Depends on TREE-604.")).toBe(true); +}); + it("uses the production company prefix and current task URL without accepting a changed target", () => { const observation = { body: "Read Outcome", pageUrl: "https://fixture.test/QA/issues/QA-123", content: [{ text: "Read " }, { href: "/QA/issues/QA-123#document-outcome", label: "Outcome" }] }; diff --git a/tests/runner-e2e/task-reply-content.ts b/tests/runner-e2e/task-reply-content.ts index a2ee32ecfc9..a79397bb57b 100644 --- a/tests/runner-e2e/task-reply-content.ts +++ b/tests/runner-e2e/task-reply-content.ts @@ -1,6 +1,8 @@ -import { fromMarkdown } from "mdast-util-from-markdown"; +import { unified } from "unified"; +import remarkParse from "remark-parse"; +import remarkGfm from "remark-gfm"; import { applyCompanyPrefix, extractCompanyPrefixFromPath } from "../../ui/src/lib/company-routes.js"; -import { parseIssueReferenceFromHref } from "../../ui/src/lib/issue-reference.js"; +import { parseIssueReferenceFromHref, remarkLinkIssueReferences } from "../../ui/src/lib/issue-reference.js"; export type ReplyContentPart = { text: string } | { href: string; label?: string; decorated?: boolean }; type MarkdownNode = { type: string; value?: string; url?: string; children?: MarkdownNode[] }; @@ -31,7 +33,7 @@ function normalized(parts: ReplyContentPart[], pageUrl: string, source = false) * with an issue/document chip. Its exact destination still has to match. * Unsupported Markdown fails closed rather than stripping arbitrary syntax. */ export function taskReplyContentMatches( - observation: { body: string; content?: ReplyContentPart[]; origin?: string; pageUrl?: string }, + observation: { body: string; content?: ReplyContentPart[]; origin?: string; pageUrl?: string; knownCompanyPrefixes?: string[] }, markdown: string, ) { if (observation.content) { @@ -42,19 +44,22 @@ export function taskReplyContentMatches( if (displayed !== observation.body.replace(/\s+/g, " ").trim()) return false; } const parts: ReplyContentPart[] = []; - const inlineLabel = (node: MarkdownNode): boolean => ["text", "inlineCode", "strong", "emphasis"].includes(node.type) + const inlineLabel = (node: MarkdownNode): boolean => ["text", "inlineCode", "strong", "emphasis", "delete"].includes(node.type) && (node.children ?? []).every(inlineLabel); const labelText = (node: MarkdownNode): string => node.value ?? (node.children ?? []).map(labelText).join(""); function visit(node: MarkdownNode): boolean { if (node.type === "text" || node.type === "inlineCode") { parts.push({ text: node.value ?? "" }); return true; } if (node.type === "break") { parts.push({ text: " " }); return true; } if (node.type === "link" && node.url && (node.children ?? []).every(inlineLabel)) { parts.push({ href: node.url, label: labelText(node) }); return true; } - if (!["root", "paragraph", "heading", "blockquote", "list", "listItem", "strong", "emphasis"].includes(node.type)) return false; + if (!["root", "paragraph", "heading", "blockquote", "list", "listItem", "strong", "emphasis", "delete"].includes(node.type)) return false; for (const child of node.children ?? []) if (!visit(child)) return false; - if (!["strong", "emphasis"].includes(node.type)) parts.push({ text: " " }); + if (!["strong", "emphasis", "delete"].includes(node.type)) parts.push({ text: " " }); return true; } - if (!visit(fromMarkdown(markdown) as MarkdownNode)) return false; + const parser = unified().use(remarkParse).use(remarkGfm); + const tree = parser.runSync(parser.parse(markdown)) as MarkdownNode; + remarkLinkIssueReferences({ knownPrefixes: observation.knownCompanyPrefixes })(tree); + if (!visit(tree)) return false; const pageUrl = observation.pageUrl ?? observation.origin ?? "https://reply.invalid"; const expected = normalized(parts, pageUrl, true); const actual = normalized(observation.content ?? [{ text: observation.body }], pageUrl);