diff --git a/ui/package.json b/ui/package.json index 01eec21155d..bf30bc3bdd3 100644 --- a/ui/package.json +++ b/ui/package.json @@ -68,6 +68,7 @@ "i18next": "^26.4.2", "lexical": "0.48.0", "lucide-react": "^1.48.0", + "mdast-util-from-markdown": "^2.0.3", "mermaid": "^12.1.0", "motion": "^13.1.1", "radix-ui": "^1.6.7", diff --git a/ui/src/components/MarkdownEditor.test.tsx b/ui/src/components/MarkdownEditor.test.tsx index b6c37e2234d..82f87566e86 100644 --- a/ui/src/components/MarkdownEditor.test.tsx +++ b/ui/src/components/MarkdownEditor.test.tsx @@ -980,6 +980,71 @@ describe("MarkdownEditor", () => { }); }); + it("stores intraword underscores as typed instead of the exporter's escaped form", async () => { + const value = "Your final answer must be exactly E2E_ASK_12_journey_1793913565974_507b01b8."; + // The real exporter escapes every literal underscore, including inside words. + mdxEditorMockState.emitMountChange = + "Your final answer must be exactly E2E\\_ASK\\_12\\_journey\\_1793913565974\\_507b01b8."; + const handleChange = vi.fn(); + const root = createRoot(container); + + await act(async () => { + root.render( + , + ); + }); + + await flush(); + + expect(handleChange).toHaveBeenCalledWith(value); + expect(handleChange.mock.calls.every(([next]) => !String(next).includes("\\_"))).toBe(true); + + await act(async () => { + root.unmount(); + }); + }); + + it("round-trips code with its backslashes while storing prose underscores as typed", async () => { + const stored = "Run snake_case:\n\n```py\nprint('a\\_b')\n```\n\nThen `keep\\_this` and \\_\\_init\\_\\_."; + // The exporter escapes the prose underscore; code and the boundary run are as typed. + mdxEditorMockState.emitMountChange = + "Run snake\\_case:\n\n```py\nprint('a\\_b')\n```\n\nThen `keep\\_this` and \\_\\_init\\_\\_."; + const handleChange = vi.fn(); + const root = createRoot(container); + + await act(async () => { + root.render(); + }); + + await flush(); + + expect(handleChange).toHaveBeenCalledWith(stored); + + await act(async () => { + root.unmount(); + }); + }); + + it("keeps a code span after an escaped HTML-like tag byte-identical", async () => { + const stored = "
\n`a\\_b` and snake_case"; + // The exporter escapes the `<` (reversed on storage) and the prose underscore. + mdxEditorMockState.emitMountChange = "\\
\n`a\\_b` and snake\\_case"; + const handleChange = vi.fn(); + const root = createRoot(container); + + await act(async () => { + root.render(); + }); + + await flush(); + + expect(handleChange).toHaveBeenCalledWith(stored); + + await act(async () => { + root.unmount(); + }); + }); + it("does not notify the parent when a prop sync echoes escaped markdown back", async () => { const handleChange = vi.fn(); const root = createRoot(container); diff --git a/ui/src/components/MarkdownEditor.tsx b/ui/src/components/MarkdownEditor.tsx index 487d3add984..fe991f82ead 100644 --- a/ui/src/components/MarkdownEditor.tsx +++ b/ui/src/components/MarkdownEditor.tsx @@ -58,6 +58,7 @@ import { unescapeAngleBracketEscapes, } from "../lib/angle-bracket-markdown"; import { unescapeBlockquoteMarkers } from "../lib/blockquote-markdown"; +import { unescapeIntrawordUnderscores } from "../lib/underscore-markdown"; import { pasteNormalizationPlugin } from "../lib/paste-normalization"; import { cn } from "../lib/utils"; import { useEditorAutocomplete, type SlashCommandOption } from "../context/EditorAutocompleteContext"; @@ -188,7 +189,9 @@ function prepareMarkdownForEditor(value: string): string { * feed agent prompts and must stay in the clean, human-authored form. */ function toStoredMarkdown(markdown: string): string { - return unescapeAngleBracketEscapes(unescapeBlockquoteMarkers(markdown)); + // Unescape underscores first, on the exporter's exact text: removing `\<` or + // `\>` can turn following lines into HTML or blockquotes and move code ranges. + return unescapeAngleBracketEscapes(unescapeBlockquoteMarkers(unescapeIntrawordUnderscores(markdown))); } function escapeRegExp(value: string): string { diff --git a/ui/src/lib/underscore-markdown.test.ts b/ui/src/lib/underscore-markdown.test.ts new file mode 100644 index 00000000000..76915f4919d --- /dev/null +++ b/ui/src/lib/underscore-markdown.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { unescapeIntrawordUnderscores } from "./underscore-markdown"; + +const unchanged = (value: string) => expect(unescapeIntrawordUnderscores(value)).toBe(value); + +describe("unescapeIntrawordUnderscores", () => { + it("restores underscores the person typed inside words and identifiers", () => { + expect(unescapeIntrawordUnderscores("Reply with E2E\\_ASK\\_12\\_journey\\_1793913565974\\_507b01b8.")) + .toBe("Reply with E2E_ASK_12_journey_1793913565974_507b01b8."); + expect(unescapeIntrawordUnderscores("Set PAPERCLIP\\_TASK\\_ID and snake\\_case")).toBe("Set PAPERCLIP_TASK_ID and snake_case"); + expect(unescapeIntrawordUnderscores("ünïcode\\_wörd")).toBe("ünïcode_wörd"); + expect(unescapeIntrawordUnderscores("foo\\_\\_bar and a\\_\\_\\_b")).toBe("foo__bar and a___b"); + expect(unescapeIntrawordUnderscores("> quoted snake\\_case\n\n- listed snake\\_case")).toBe("> quoted snake_case\n\n- listed snake_case"); + }); + + it("keeps word-boundary escapes and escaped backslashes", () => { + for (const value of [ + "\\_not emphasis\\_", + // `__init__` at word boundaries would render as strong emphasis. + "\\_\\_init\\_\\_", + "word \\_start", + "end\\_ word", + "a\\\\_b", + ]) unchanged(value); + }); + + it("keeps every code span and code block byte-identical", () => { + for (const value of [ + "Use `snake\\_case` literally", + "``keep\\_this ` inside``", + "```\nE2E\\_ASK\n```", + "~~~ts\nconst a\\_b = 1;\n~~~", + " indented\\_code", + "> ```\n> snake\\_case\n> ```", + "- ```\n snake\\_case\n ```", + "1. ~~~\n snake\\_case\n ~~~", + // Greptile: a fence indented inside a list item, closed at the item's indent. + "- item\n\n ```\n keep\\_this\n ```\n still\\_code\n ```", + // A top-level fence whose content has a quoted fence-like line. + "```\n> ```\nkeep\\_this\n```", + "\\\\`snake\\_case`", + // Raw HTML does not process backslash escapes. + "
\nraw\\_html\n
", + ]) unchanged(value); + }); + + it("rewrites only the prose around code", () => { + expect(unescapeIntrawordUnderscores("```\nkeep\\_this\n```\nfix\\_this")).toBe("```\nkeep\\_this\n```\nfix_this"); + expect(unescapeIntrawordUnderscores("> ```\n> keep\\_this\n> ```\n\nfix\\_this")).toBe("> ```\n> keep\\_this\n> ```\n\nfix_this"); + expect(unescapeIntrawordUnderscores("`keep\\_this` and fix\\_this")).toBe("`keep\\_this` and fix_this"); + expect(unescapeIntrawordUnderscores("\\\\`keep\\_this` then fix\\_this")).toBe("\\\\`keep\\_this` then fix_this"); + expect(unescapeIntrawordUnderscores("- item\n\n ```\n keep\\_this\n ```\n\nfix\\_this")) + .toBe("- item\n\n ```\n keep\\_this\n ```\n\nfix_this"); + }); + + it("treats escaped or unmatched backticks as text", () => { + expect(unescapeIntrawordUnderscores("A literal \\` then snake\\_case")).toBe("A literal \\` then snake_case"); + expect(unescapeIntrawordUnderscores("An unmatched ` then snake\\_case")).toBe("An unmatched ` then snake_case"); + }); +}); diff --git a/ui/src/lib/underscore-markdown.ts b/ui/src/lib/underscore-markdown.ts new file mode 100644 index 00000000000..351eafbdd74 --- /dev/null +++ b/ui/src/lib/underscore-markdown.ts @@ -0,0 +1,70 @@ +import { fromMarkdown } from "mdast-util-from-markdown"; + +/** + * MDXEditor (via `mdast-util-to-markdown`) exports every literal `_` as `\_`, + * including underscores inside a word such as `snake_case` or `E2E_ASK_12`. + * CommonMark never treats an intraword underscore as emphasis, so that escape + * is pure transport noise. Stored task text feeds agent prompts, and agents then + * see (and repeat) backslashes the person never typed. + * + * This helper removes the escape only from a run of escaped underscores that + * sits between two letters or digits (`snake\_case`, `foo\_\_bar`), which cannot + * change how the markdown renders. It parses the markdown so that code spans and + * fenced or indented code blocks at any container depth, and raw HTML, stay + * byte-identical. Run it on the editor's exact export, before any other + * transport-escape removal changes how that text parses. + * Word-boundary escapes (`\_emphasis\_`, `\_\_init\_\_`, which would otherwise + * render as emphasis) and escaped backslashes (`\\_`) are left unchanged. + */ + +const INTRAWORD_ESCAPED_RUN_RE = /(?<=[\p{L}\p{N}])((?:\\_)+)(?=[\p{L}\p{N}])/gu; + +interface PositionedNode { + type: string; + position?: { start: { offset?: number }; end: { offset?: number } }; + children?: PositionedNode[]; +} + +/** True when the character at `index` is escaped by an odd number of backslashes. */ +function isEscaped(text: string, index: number): boolean { + let count = 0; + for (let i = index - 1; i >= 0 && text[i] === "\\"; i -= 1) count += 1; + return count % 2 === 1; +} + +function unescapeProse(text: string): string { + return text.replace(INTRAWORD_ESCAPED_RUN_RE, (run: string, _group: string, offset: number, whole: string) => + // `a\\_b` is an escaped backslash followed by a literal underscore: keep it. + isEscaped(whole, offset) ? run : "_".repeat(run.length / 2)); +} + +/** Source ranges of code spans and code blocks, in document order. */ +function codeRanges(node: PositionedNode, ranges: Array<[number, number]>): Array<[number, number]> { + // Raw HTML does not process backslash escapes, so a `\_` there is literal too. + if (node.type === "code" || node.type === "inlineCode" || node.type === "html") { + const start = node.position?.start.offset, end = node.position?.end.offset; + if (typeof start === "number" && typeof end === "number") ranges.push([start, end]); + return ranges; + } + for (const child of node.children ?? []) codeRanges(child, ranges); + return ranges; +} + +export function unescapeIntrawordUnderscores(markdown: string): string { + if (!markdown.includes("\\_")) return markdown; + let ranges: Array<[number, number]>; + try { + ranges = codeRanges(fromMarkdown(markdown) as PositionedNode, []); + } catch { + // Never rewrite text the parser could not place. + return markdown; + } + let result = ""; + let cursor = 0; + for (const [start, end] of ranges) { + if (start < cursor) continue; + result += unescapeProse(markdown.slice(cursor, start)) + markdown.slice(start, end); + cursor = end; + } + return result + unescapeProse(markdown.slice(cursor)); +}