From 348ed0ae1041ba7ef92c650d04e374ec710adb94 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sun, 15 Nov 2026 08:44:00 -0600 Subject: [PATCH] fix(runner): keep closing tool-call tags and model control tokens out of OpenCode chat (#21761) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The Paperclip Runner OpenCode driver streams model text into the chat as progress and selects the final reply > - DeepSeek v4.1 flash (through OpenRouter) wrote tool-call scaffolding and tokenizer control tokens as ordinary text > - #21361 hides tool-call markup, but its detector matched only opening DSML tags, and nothing recognised control tokens > - A person saw raw `` lines in a progress message, and in another conversation got a degenerate reply instead of an answer > - This pull request extends the same detector to closing DSML tags and to model control tokens, on the streaming path and at completion > - The benefit is that scaffolding never reaches the chat, and a degenerate reply goes to the existing correction instead of being posted ## Linked Issues or Issue Description Refs #21361 **What happened?** Chat-eval cohort 16 (OpenCode with `openrouter/deepseek/deepseek-v4.1-flash-1017`, master 819f61f1fd) showed two defects. 1. In `task-coordination.block-on-design`, after a real `paperclip_finish` call, the model streamed a text part that held only closing tags (`\r\n\n…\r\n`). The driver emitted it as an `agentMessage` progress delta, so the chat showed raw markup. The cause: `SENTINEL_FORMS` matched `<|DSML|` but not `\n\nSearch output truncated. Let me search more specifically…\n`. The driver took it as the final text part and posted it. The question was never answered. **Expected behavior** Tool-call scaffolding and model control tokens never appear in progress or in the reply. A reply made only of them goes to the empty-response correction, which asks the model for the real answer. Real answers, including answers about XML, chat templates or tool tags, stay as written. **Steps to reproduce** Feed the recorded text parts above to the OpenCode driver (the new `closing-markup` and `control-token-reply` cases in `empty-response.test.ts` do this). ## What Changed - `tool-call-markup.ts`: closing DSML tags (``, `<|im_start|>`, `<|eot_id|>`) outside code. A new `visibleBeforeModelScaffolding` holds back both kinds while text streams. - Sentinels inside inline code spans are ignored, as fenced code already was. - A per-line fast path skips the sentinel scan when a line has no `<|`, `<|`, `` tag, control tokens in inline code and in a fence, HTML, and comparison operators are left alone. A driver case checks that `answer.<|end▁of▁sentence|>` keeps the answer with no correction. - Per-message cost (local tsx micro-benchmark, 20k calls, one prose line with `<`): 500 chars 0.3 → 0.8 µs; 2 KB 2.2 → 2.6 µs; 8 KB about 8–16 µs before and 7–9 µs after (noise range). The detector runs per text-part update, as before. Nothing runs before the turn starts. - Paid narrow eval run: results are added in a PR comment. ## Risks - A real answer that writes a literal control token outside code is cut at the token. Prose before it stays. That text is never meaningful prose. - Closing DSML tags in plain prose (outside code) now hide the rest of the part. DSML tags are model-internal, so prose does not contain them. ## Model Used - Claude, `claude-opus-5-5` (Claude Code agent with tool use and code execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../drivers/opencode/empty-response.test.ts | 35 +++-- .../opencode/opencode-server-driver.ts | 35 ++++- .../drivers/opencode/tool-call-markup.test.ts | 55 ++++++- .../src/drivers/opencode/tool-call-markup.ts | 142 ++++++++++++++++-- 4 files changed, 234 insertions(+), 33 deletions(-) diff --git a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts index 85c0beb8c80..10036e90ab5 100644 --- a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts @@ -5,7 +5,7 @@ import { expect, it, vi } from "vitest"; import { OpenCodeServerDriver } from "./opencode-server-driver.js"; import type { PrpEvent } from "../../protocol/replay-contract.js"; -it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "tool-work-text", "tool-work-empty", "stale-tool-work"] as const)("handles an empty OpenCode response: %s", async mode => { +it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "closing-markup", "control-token-reply", "control-token-trailing", "tool-work-text", "tool-work-empty", "stale-tool-work"] as const)("handles an empty OpenCode response: %s", async mode => { const root = await mkdtemp(join(tmpdir(), "opencode-empty-response-")); const command = join(root, "fake-opencode.mjs"); await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command); @@ -16,6 +16,13 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow // ordinary text part, and no completion call. const recordedReasoning = "Task created and already executing (status in_progress, run dafb...). Good. Reply with link. Use finish. Keep concise.\n\nCompletion contract revision 1, criterion \"objective\". I should call paperclip_finish with done. Include evidence refs maybe task link. Let's do."; const recordedMarkup = "\n\n<|DSML| calls>\n<|DSML| invoke name=\"paperclip_paperclip_finish\">\n<|DSML| parameter name=\"reportedWorkDisposition\" string=\"true\">done\n<|DSML| parameter name=\"summary\" string=\"true\">Created `website-fixture.txt` in the task workspace.\n\n"; + // Recorded from chat-eval cohort 16: closing DSML tags streamed alone as + // text (task-coordination.block-on-design), and a reply of a + // begin-of-sentence token plus an imitation tool-output fragment + // (cases.case-history). + const recordedClosingMarkup = "\r\n\n\r\n\n\r\n\r\n\n"; + const recordedControlTokenReply = "<|begin▁of▁sentence|>\n\nSearch output truncated. Let me search more specifically for \"case document\" and \"case events\".\n"; + const leakedText = mode === "closing-markup" ? recordedClosingMarkup : mode === "control-token-reply" ? recordedControlTokenReply : recordedMarkup; let toolTimer: ReturnType | undefined; let stream: ReadableStreamDefaultController | undefined; let streamOpened!: () => void; @@ -95,19 +102,19 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); return; } - if (mode === "tool-call-markup" && number === 1) { + if (["tool-call-markup", "closing-markup", "control-token-reply"].includes(mode) && number === 1) { const leaked = { id: "part-1-text", messageID, sessionID, type: "text", text: "", time: { start: 1 } }; emit({ type: "message.part.updated", properties: { sessionID, part: leaked } }); - for (const delta of [recordedMarkup.slice(0, 4), recordedMarkup.slice(4, 9), recordedMarkup.slice(9)]) + for (const delta of [leakedText.slice(0, 4), leakedText.slice(4, 9), leakedText.slice(9)]) emit({ type: "message.part.delta", properties: { sessionID, messageID, partID: leaked.id, field: "text", delta } }); - emit({ type: "message.part.updated", properties: { sessionID, part: { ...leaked, text: recordedMarkup, time: { start: 1, end: 2 } } } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { ...leaked, text: leakedText, time: { start: 1, end: 2 } } } }); emit({ type: "session.idle", properties: { sessionID } }); emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); return; } - const answer = mode === "normal" || mode === "stale-tool-work" && number === 1 || mode === "foreign-idle" || mode === "queued-tool" || number === 3 && mode === "tool-work-text" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"].includes(mode); + const answer = mode === "normal" || mode === "control-token-trailing" || mode === "stale-tool-work" && number === 1 || mode === "foreign-idle" || mode === "queued-tool" || number === 3 && mode === "tool-work-text" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "closing-markup", "control-token-reply"].includes(mode); emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID, - type: answer ? "text" : "reasoning", text: answer ? (mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer.") : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } }); + type: answer ? "text" : "reasoning", text: answer ? (mode === "reasoning-echo-correction-matches" ? recordedReasoning : mode === "control-token-trailing" ? "Here is the actual final answer.<|end▁of▁sentence|>" : "Here is the actual final answer.") : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } }); emit({ type: "session.idle", properties: { sessionID } }); emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); }); @@ -145,14 +152,15 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow } await vi.advanceTimersByTimeAsync(mode.endsWith("idle-stall") ? 20_001 : mode === "slow-native-tool" ? 1_999 : 5_001); const events = await completed; - const succeeds = ["stale-tool-work", "tool-work-text", "reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"].includes(mode); + const succeeds = ["stale-tool-work", "tool-work-text", "reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "closing-markup", "control-token-reply", "control-token-trailing"].includes(mode); expect(events.at(-1)?.eventType).toBe(succeeds ? "turn.completed" : "turn.failed"); expect(events.filter(event => event.eventType === "turn.started")).toHaveLength(1); expect(events.every(event => !event.turnId || event.turnId === turnId)).toBe(true); const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final"); expect(final.map(event => event.payload.text)).toEqual(succeeds ? [mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer."] : []); // Only a turn that already did tool work gets the second, plain-text request. - expect(prompts).toHaveLength(["normal", "foreign-idle", "queued-tool", "stale-tool-work"].includes(mode) ? 1 : mode.startsWith("tool-work") ? 3 : 2); + // A trailing end-of-sentence token after a real answer is not a degenerate reply. + expect(prompts).toHaveLength(["normal", "foreign-idle", "queued-tool", "stale-tool-work", "control-token-trailing"].includes(mode) ? 1 : mode.startsWith("tool-work") ? 3 : 2); if (prompts.length === 3) { expect(prompts[2]).toMatchObject({ providerID: prompts[0]!.providerID, modelID: prompts[0]!.modelID, system: prompts[0]!.system, variant: "paperclip-no-reasoning" }); // Text first, so a chat reply survives a dropped call; then the @@ -173,13 +181,22 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow expect(assistantText.join("")).not.toContain(recordedReasoning.slice(0, 20)); expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_reasoning_echo_discarded")).toBe(true); } - if (mode === "tool-call-markup") { + if (mode === "tool-call-markup" || mode === "closing-markup") { const assistantText = events.filter(event => event.payload.kind === "agentMessage").map(event => String(event.payload.text ?? "")); expect(assistantText.join("")).not.toContain("DSML"); expect(assistantText.join("")).not.toContain("<|"); expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_tool_call_markup_discarded")).toBe(true); expect(JSON.stringify(prompts[1]!.parts)).toContain("wrote a tool call as plain text"); } + if (mode === "control-token-reply") { + // Neither progress nor the reply shows the degenerate text; the + // correction asks for the answer the person is still owed. + const assistantText = events.filter(event => event.payload.kind === "agentMessage").map(event => String(event.payload.text ?? "")); + expect(assistantText.join("")).not.toContain("begin▁of▁sentence"); + expect(assistantText.join("")).not.toContain("Search output truncated"); + expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_control_token_reply_discarded")).toBe(true); + expect(JSON.stringify(prompts[1]!.parts)).toContain("raw model control tokens"); + } if (mode === "stale-tool-work") { const nextEvents = collect(); await session.startTurn({ message: { role: "user", text: "Answer the next request." }, reasoningMode: "disabled" }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index bb22a014938..50ce8fbddbf 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -95,7 +95,7 @@ import { type OpenCodeMcpBridge, type OpenCodeMcpBridgeOptions, } from "./mcp-bridge.js"; -import { toolCallMarkupStart, visibleBeforeToolCallMarkup } from "./tool-call-markup.js"; +import { controlTokenStart, toolCallMarkupStart, visibleBeforeModelScaffolding } from "./tool-call-markup.js"; import { materializeNativeRuntimeSkills, releaseMaterializedNativeRuntimeSkills, @@ -568,6 +568,7 @@ class OpenCodeHarnessSession implements HarnessSession { #semanticResultProviderMessageId: string | null = null; #lastNonTerminalToolSourceSeq = 0; #lastToolCallMarkupSourceSeq = 0; + #lastControlTokenReplySourceSeq = 0; #usage: Record | null = null; #upstreamServed: UpstreamServed | null = null; readonly #conversationMode: "task" | "prepared"; @@ -707,6 +708,7 @@ class OpenCodeHarnessSession implements HarnessSession { this.#semanticResultProviderMessageId = null; this.#lastNonTerminalToolSourceSeq = 0; this.#lastToolCallMarkupSourceSeq = 0; + this.#lastControlTokenReplySourceSeq = 0; this.#completedTextPartIds.clear(); this.#completedReasoningPartIds.clear(); this.#reasoningTexts.clear(); @@ -920,7 +922,9 @@ class OpenCodeHarnessSession implements HarnessSession { ? "Your previous response reached the output limit and was discarded as incomplete." : this.#lastToolCallMarkupSourceSeq > this.#turnStartSourceSeq ? "Your previous response wrote a tool call as plain text, so the tool did not run. Call tools only through the tool-calling interface, never as text." - : "Your response ended without a user-visible final answer or an accepted completion report."; + : this.#lastControlTokenReplySourceSeq > this.#turnStartSourceSeq + ? "Your previous response contained raw model control tokens instead of an answer, so it was discarded." + : "Your response ended without a user-visible final answer or an accepted completion report."; const prompt = attempt === 1 ? `${opening} Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.` : "Your last response was empty again. Start this response with a short plain-text final answer for the user that says what you did and what, if anything, is left. After that text, submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions."; @@ -2210,12 +2214,16 @@ class OpenCodeHarnessSession implements HarnessSession { // the start of a completed reasoning part, and release it once it differs. if (partType === "reasoning" && content.trim()) this.#reasoningTexts.set(partId, content.trim()); const reasoningEcho = partType === "text" && this.#echoesReasoning(content, completed); + // Tool-call markup and model control tokens are never prose: hide them + // from progress as they stream, and keep them out of the reply. const markupStart = partType === "text" && completed ? toolCallMarkupStart(content) : -1; + const controlStart = partType === "text" && completed ? controlTokenStart(content) : -1; + const cut = Math.min(...[markupStart, controlStart].filter(start => start >= 0), content.length); const visible = partType !== "text" ? content : completed - ? markupStart < 0 ? content : content.slice(0, markupStart) - : visibleBeforeToolCallMarkup(content); + ? content.slice(0, cut) + : visibleBeforeModelScaffolding(content); if (!reasoningEcho) this.#partText.set(partId, visible); if (partType === "text" && visible.length > 0 && !reasoningEcho) { for (const reference of paperclipWorkspaceFileReferencesFromText( @@ -2330,15 +2338,30 @@ class OpenCodeHarnessSession implements HarnessSession { ); return; } + // Prose before a control token is the answer (a trailing end-of-turn + // token, say). With no prose before it, the part is degenerate output, + // so the empty-response correction asks for the real answer. Unlike + // tool-call markup it says nothing about earlier prose, which stays + // eligible as the reply. + const answer = controlStart < 0 ? content : content.slice(0, controlStart).trimEnd(); + if (!answer.trim()) { + this.#lastControlTokenReplySourceSeq = this.#sourceSequence; + this.#emit( + "harness.diagnostic", + { code: "opencode_control_token_reply_discarded", message: "OpenCode returned model control tokens instead of an answer; the text was not used as a response." }, + { turnId }, + ); + return; + } this.#completedTextParts.push({ partId, messageId, - text: content, + text: answer, item: bounded({ ...part, type: "agentMessage", phase: "final_answer", - text: content, + text: answer, }), observedSourceSeq: this.#sourceSequence, }); diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts index 625e9513618..470c576b6ef 100644 --- a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "vitest"; -import { containsToolCallMarkup, visibleBeforeToolCallMarkup } from "./tool-call-markup.js"; +import { containsToolCallMarkup, controlTokenStart, visibleBeforeModelScaffolding, visibleBeforeToolCallMarkup } from "./tool-call-markup.js"; const dsml = "<|DSML| calls>\n<|DSML| invoke name=\"paperclip_paperclip_finish\">\n<|DSML| parameter name=\"summary\" string=\"true\">Done.\n\n"; const markupForms = { @@ -9,6 +9,9 @@ const markupForms = { "ASCII call tokens": "<|tool_calls_begin|><|tool_call_begin|>function", "function_calls wrapper": "\n\n\n", "tool_call wrapper": "\n{\"name\": \"paperclip_finish\", \"arguments\": {}}\n", + // Recorded from chat-eval cohort 16 (task-coordination.block-on-design): after + // a real paperclip_finish call, DeepSeek streamed only closing tags as text. + "closing DSML tags only": "\r\n\n\r\n\n\r\n\r\n\n\r\n", }; describe("leaked tool-call markup", () => { @@ -33,6 +36,7 @@ describe("leaked tool-call markup", () => { ["other fence characters inside a fence", `~~~\n\`\`\`\n${dsml}\n~~~`], ["an indented fence line inside a top-level fence", `\`\`\`\n \`\`\`\n${dsml}\n\`\`\``], ["a list-indented fence", `10. Example:\n\n \`\`\`\n ${dsml}\n \`\`\``], + ["a sentinel in inline code", "DeepSeek opens a call with `<|DSML| invoke>` and closes it with ``."], ])("ignores %s", (_label, text) => { expect(containsToolCallMarkup(text)).toBe(false); }); @@ -49,7 +53,7 @@ describe("leaked tool-call markup", () => { expect(visible.startsWith(shown)).toBe(true); shown = visible; } - expect(shown).toBe("Saved the file. Done <\n\n"); + expect(shown.trimEnd()).toBe("Saved the file. Done <"); }); it("releases a held fragment once it cannot become markup", () => { @@ -60,3 +64,50 @@ describe("leaked tool-call markup", () => { expect(visibleBeforeToolCallMarkup("Done \n\nSearch output truncated. Let me search more specifically for \"case document\" and \"case events\".\n"; + +describe("model control tokens", () => { + it.each([ + ["the recorded degenerate reply", degenerateReply, 0], + ["a ChatML turn after an answer", "The fact changed twice.\n<|im_start|>user\nthanks", 24], + ["a token after an escaped backtick", "Use \\`code. <|im_end|> then `x`.", 12], + ["a token after an unmatched backtick", "One ` tick. <|eot_id|>", 12], + ["a trailing end-of-sentence token", "Riley changed it.<|end▁of▁sentence|>", 17], + ["a Llama end-of-turn token", "Done.<|eot_id|>", 5], + ])("finds %s", (_label, text, start) => { + expect(controlTokenStart(text)).toBe(start); + }); + + it.each([ + ["an XML answer", "The export is XML:\n\n\n overlay\n"], + ["an answer about an tag", "The tool wraps results in an element, and closes it."], + ["an answer explaining chat templates in inline code", "ChatML starts a turn with `<|im_start|>` and ends it with `<|im_end|>`."], + ["a double-backtick code span", "Write ``<|im_start|>`` with two backticks, then `x`."], + ["a code span across lines", "The template opens with `<|im_start|>\nuser<|im_end|>` and so on."], + ["a code span after a fence holding a lone backtick", "~~~text\n`\n~~~\nUse ``<|im_start|>`` here."], + ["control tokens in a fence", "The template looks like this:\n```\n<|im_start|>user\nhi<|im_end|>\n```"], + ["HTML", "
\n

Hello

\n
"], + ["comparison operators", "Use x < 3 and y > 2."], + ])("leaves %s alone", (_label, text) => { + expect(controlTokenStart(text)).toBe(-1); + expect(visibleBeforeModelScaffolding(text)).toBe(text); + }); + + it("never streams any of the recorded degenerate reply, at every split point", () => { + for (let end = 1; end <= degenerateReply.length; end += 1) + expect(visibleBeforeModelScaffolding(degenerateReply.slice(0, end))).toBe(""); + }); + + it("streams prose up to a trailing token without showing the token", () => { + const answer = "Riley changed it.<|end▁of▁sentence|>"; + let shown = ""; + for (let end = 1; end <= answer.length; end += 1) { + const visible = visibleBeforeModelScaffolding(answer.slice(0, end)); + expect(visible.startsWith(shown)).toBe(true); + shown = visible; + } + expect(shown).toBe("Riley changed it."); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.ts b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.ts index bf4106a289e..e7e3d457d26 100644 --- a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.ts +++ b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.ts @@ -1,8 +1,19 @@ +// DeepSeek writes a call as DSML tags. After a real tool call it can also +// trail only the closing tags (``), so both forms count. const SENTINEL_FORMS = ["|", "|"].flatMap(bar => [ `<${bar}DSML${bar}`, + ` `<${bar}${name}${bar}>`), ]); +// Tokenizer control tokens are never prose. A model that writes one has +// restarted a document or a turn (cohort 16 replied `<|begin▁of▁sentence|>` +// followed by an imitation tool-output fragment). +const CONTROL_TOKEN_FORMS = [ + ...["|", "|"].flatMap(bar => ["begin▁of▁sentence", "end▁of▁sentence", "begin_of_sentence", "end_of_sentence", "User", "Assistant"].map(name => `<${bar}${name}${bar}>`)), + ...["im_start", "im_end", "endoftext", "begin_of_text", "end_of_text", "eot_id", "start_header_id", "end_header_id"].map(name => `<|${name}|>`), +]; + const LINE_FORMS = [ { wrapper: "", body: '\s*", body: '{"name":', pattern: /^\s*\{\s*"name"\s*:/ }, @@ -19,16 +30,95 @@ function couldBecomeLineForm(rest: string): boolean { : compact.startsWith(wrapper) && body.startsWith(compact.slice(wrapper.length))); } -function firstSentinel(line: string): number { +const SENTINEL_OPENING = /<\/?[||]/; + +/** + * Inline code spans as [start, end) offsets: a backtick run closes at the next + * run of the same length within its paragraph. An escaped backtick opens + * nothing, an unmatched run is literal text, and backticks inside fenced + * blocks never pair with ones outside. + */ +function codeSpans(source: string): Array<[number, number]> { + const spans: Array<[number, number]> = []; + if (!source.includes("`")) return spans; + const content = withoutFencedBlocks(source); + for (let index = 0; index < content.length; index += 1) { + const char = content[index]; + if (char === "\\") { + index += 1; + continue; + } + if (char !== "`") continue; + let length = 1; + while (content[index + length] === "`") length += 1; + const paragraphEnd = content.indexOf("\n\n", index); + const limit = paragraphEnd < 0 ? content.length : paragraphEnd; + let close = -1; + for (let at = index + length; at < limit; at += 1) { + if (content[at] !== "`") continue; + let run = 1; + while (content[at + run] === "`") run += 1; + if (run === length) { + close = at; + break; + } + at += run - 1; + } + if (close < 0) { + index += length - 1; + continue; + } + spans.push([index, close + length]); + index = close + length - 1; + } + return spans; +} + +/** First sentinel on the line that is not inside an inline code span. */ +function firstSentinel(line: string, lineStart: number, spans: Array<[number, number]>, forms: readonly string[]): number { + // Every sentinel starts with `<|`, `<|`, ` spans.some(([start, end]) => offset >= start && offset < end); let first = -1; - for (const form of SENTINEL_FORMS) { - const index = line.indexOf(form); - if (index >= 0 && (first < 0 || index < first)) first = index; + for (const form of forms) { + for (let index = line.indexOf(form); index >= 0 && (first < 0 || index < first); index = line.indexOf(form, index + 1)) { + if (!inCode(lineStart + index)) { + first = index; + break; + } + } } return first; } -function fenceOf(line: string): { indent: number; char: string; length: number; rest: string } | null { +/** The content with every fenced block's lines blanked, keeping offsets. */ +function withoutFencedBlocks(content: string): string { + let openFence: { indent: number; char: string; length: number } | null = null; + return content.split("\n").map(line => { + const fence = fenceOf(line); + if (openFence) { + if (closesFence(fence, openFence)) openFence = null; + return " ".repeat(line.length); + } + if (opensFence(fence)) { + openFence = { indent: fence.indent, char: fence.char, length: fence.length }; + return " ".repeat(line.length); + } + return line; + }).join("\n"); +} + +type Fence = { indent: number; char: string; length: number; rest: string }; + +function closesFence(fence: Fence | null, open: { indent: number; char: string; length: number }): boolean { + return !!fence && fence.indent <= open.indent + 3 && fence.char === open.char && fence.length >= open.length && !fence.rest.trim(); +} + +function opensFence(fence: Fence | null): fence is Fence { + return !!fence && !(fence.char === "`" && fence.rest.includes("`")); +} + +function fenceOf(line: string): Fence | null { const match = /^([ \t]*)(`{3,}|~{3,})(.*)$/.exec(line); return match ? { indent: match[1]!.replace(/\t/g, " ").length, char: match[2]![0]!, length: match[2]!.length, rest: match[3]! } @@ -36,7 +126,17 @@ function fenceOf(line: string): { indent: number; char: string; length: number; } export function toolCallMarkupStart(content: string, options: { streaming?: boolean } = {}): number { + return scaffoldingStart(content, SENTINEL_FORMS, true, options.streaming === true); +} + +/** Where the first model control token outside code starts, or -1. */ +export function controlTokenStart(content: string, options: { streaming?: boolean } = {}): number { + return scaffoldingStart(content, CONTROL_TOKEN_FORMS, false, options.streaming === true); +} + +function scaffoldingStart(content: string, forms: readonly string[], lineForms: boolean, streaming: boolean): number { if (!content.includes("<")) return -1; + const spans = codeSpans(content); let offset = 0; let openFence: { indent: number; char: string; length: number } | null = null; const lines = content.split("\n"); @@ -45,25 +145,25 @@ export function toolCallMarkupStart(content: string, options: { streaming?: bool offset += line.length + 1; const fence = fenceOf(line); if (openFence) { - if (fence && fence.indent <= openFence.indent + 3 && fence.char === openFence.char && fence.length >= openFence.length && !fence.rest.trim()) - openFence = null; + if (closesFence(fence, openFence)) openFence = null; continue; } - if (fence && !(fence.char === "`" && fence.rest.includes("`"))) { + if (opensFence(fence)) { openFence = { indent: fence.indent, char: fence.char, length: fence.length }; continue; } if (!line.trim()) continue; - const sentinel = firstSentinel(line); + const sentinel = firstSentinel(line, lineStart, spans, forms); if (sentinel >= 0) return lineStart + sentinel; - const indent = line.length - line.trimStart().length; - const rest = line[indent] === "<" ? content.slice(lineStart + indent, lineStart + indent + LOOK_AHEAD) : ""; - if (rest && LINE_FORMS.some(({ pattern }) => pattern.test(rest))) return lineStart + indent; - if (!options.streaming) continue; - if (rest && couldBecomeLineForm(rest)) return lineStart + indent; - if (index === lines.length - 1) { + if (lineForms) { + const indent = line.length - line.trimStart().length; + const rest = line[indent] === "<" ? content.slice(lineStart + indent, lineStart + indent + LOOK_AHEAD) : ""; + if (rest && LINE_FORMS.some(({ pattern }) => pattern.test(rest))) return lineStart + indent; + if (streaming && rest && couldBecomeLineForm(rest)) return lineStart + indent; + } + if (streaming && index === lines.length - 1) { const lastOpen = line.lastIndexOf("<"); - if (lastOpen >= 0 && SENTINEL_FORMS.some(form => form.startsWith(line.slice(lastOpen)))) return lineStart + lastOpen; + if (lastOpen >= 0 && forms.some(form => form.startsWith(line.slice(lastOpen)))) return lineStart + lastOpen; } } return -1; @@ -77,3 +177,13 @@ export function visibleBeforeToolCallMarkup(content: string): string { const start = toolCallMarkupStart(content, { streaming: true }); return start < 0 ? content : content.slice(0, start); } + +/** + * The prose a person may see while a text part still streams: everything + * before tool-call markup or a model control token, holding back a trailing + * fragment that could still become either. + */ +export function visibleBeforeModelScaffolding(content: string): string { + const starts = [toolCallMarkupStart(content, { streaming: true }), controlTokenStart(content, { streaming: true })].filter(start => start >= 0); + return starts.length ? content.slice(0, Math.min(...starts)) : content; +}