diff --git a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts index b9fb24ff1ac..712f8effda2 100644 --- a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts @@ -5,7 +5,7 @@ import { expect, it, vi } from "vitest"; import { OpenCodeServerDriver } from "./opencode-server-driver.js"; import type { PrpEvent } from "../../protocol/replay-contract.js"; -it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches"] as const)("handles an empty OpenCode response: %s", async mode => { +it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"] as const)("handles an empty OpenCode response: %s", async mode => { const root = await mkdtemp(join(tmpdir(), "opencode-empty-response-")); const command = join(root, "fake-opencode.mjs"); await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command); @@ -15,6 +15,7 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow // through OpenRouter): the step's reasoning, then the same text again as an // ordinary text part, and no completion call. const recordedReasoning = "Task created and already executing (status in_progress, run dafb...). Good. Reply with link. Use finish. Keep concise.\n\nCompletion contract revision 1, criterion \"objective\". I should call paperclip_finish with done. Include evidence refs maybe task link. Let's do."; + const recordedMarkup = "\n\n<|DSML| calls>\n<|DSML| invoke name=\"paperclip_paperclip_finish\">\n<|DSML| parameter name=\"reportedWorkDisposition\" string=\"true\">done\n<|DSML| parameter name=\"summary\" string=\"true\">Created `website-fixture.txt` in the task workspace.\n\n"; let toolTimer: ReturnType | undefined; let stream: ReadableStreamDefaultController | undefined; let streamOpened!: () => void; @@ -83,7 +84,17 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); return; } - const answer = mode === "normal" || mode === "foreign-idle" || mode === "queued-tool" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches"].includes(mode); + if (mode === "tool-call-markup" && number === 1) { + const leaked = { id: "part-1-text", messageID, sessionID, type: "text", text: "", time: { start: 1 } }; + emit({ type: "message.part.updated", properties: { sessionID, part: leaked } }); + for (const delta of [recordedMarkup.slice(0, 4), recordedMarkup.slice(4, 9), recordedMarkup.slice(9)]) + emit({ type: "message.part.delta", properties: { sessionID, messageID, partID: leaked.id, field: "text", delta } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { ...leaked, text: recordedMarkup, time: { start: 1, end: 2 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); + return; + } + const answer = mode === "normal" || mode === "foreign-idle" || mode === "queued-tool" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"].includes(mode); emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID, type: answer ? "text" : "reasoning", text: answer ? (mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer.") : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } }); emit({ type: "session.idle", properties: { sessionID } }); @@ -123,7 +134,7 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow } await vi.advanceTimersByTimeAsync(mode.endsWith("idle-stall") ? 20_001 : mode === "slow-native-tool" ? 1_999 : 5_001); const events = await completed; - const succeeds = ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches"].includes(mode); + const succeeds = ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"].includes(mode); expect(events.at(-1)?.eventType).toBe(succeeds ? "turn.completed" : "turn.failed"); expect(events.filter(event => event.eventType === "turn.started")).toHaveLength(1); expect(events.every(event => !event.turnId || event.turnId === turnId)).toBe(true); @@ -141,6 +152,13 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow expect(assistantText.join("")).not.toContain(recordedReasoning.slice(0, 20)); expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_reasoning_echo_discarded")).toBe(true); } + if (mode === "tool-call-markup") { + const assistantText = events.filter(event => event.payload.kind === "agentMessage").map(event => String(event.payload.text ?? "")); + expect(assistantText.join("")).not.toContain("DSML"); + expect(assistantText.join("")).not.toContain("<|"); + expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_tool_call_markup_discarded")).toBe(true); + expect(JSON.stringify(prompts[1]!.parts)).toContain("wrote a tool call as plain text"); + } if (mode === "two-turn-idle") { const nextEvents = collect(); const next = await session.startTurn({ message: { role: "user", text: "Answer the next request." }, reasoningMode: "disabled" }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index fe9ace59c80..800fdc599d0 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -96,6 +96,7 @@ import { type OpenCodeMcpBridgeOptions, } from "./mcp-bridge.js"; import { nativeMcpLaunchBinding } from "../native-mcp.js"; +import { toolCallMarkupStart, visibleBeforeToolCallMarkup } from "./tool-call-markup.js"; import { materializeNativeRuntimeSkills, releaseMaterializedNativeRuntimeSkills, @@ -567,6 +568,7 @@ class OpenCodeHarnessSession implements HarnessSession { #semanticResultTextBoundary: number | null = null; #semanticResultProviderMessageId: string | null = null; #lastNonTerminalToolSourceSeq = 0; + #lastToolCallMarkupSourceSeq = 0; #usage: Record | null = null; #upstreamServed: UpstreamServed | null = null; readonly #conversationMode: "task" | "prepared"; @@ -703,6 +705,7 @@ class OpenCodeHarnessSession implements HarnessSession { this.#semanticResultTextBoundary = null; this.#semanticResultProviderMessageId = null; this.#lastNonTerminalToolSourceSeq = 0; + this.#lastToolCallMarkupSourceSeq = 0; this.#completedTextPartIds.clear(); this.#completedReasoningPartIds.clear(); this.#reasoningTexts.clear(); @@ -870,7 +873,7 @@ class OpenCodeHarnessSession implements HarnessSession { // request. Keep in-flight tool work gated by the checks above. if (this.#interruptedTurnId === turnId) return false; const recovery = this.#emptyResponseRecovery; - const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq); + const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq, this.#lastToolCallMarkupSourceSeq); if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary && !this.#isOutputLimited(part) && !this.#isReasoningEcho(part))) return false; // Both idle event forms can remain queued from the empty response. Wait // for correction evidence. A completed answer part is sufficient even @@ -894,7 +897,9 @@ class OpenCodeHarnessSession implements HarnessSession { this.#armEmptyResponseAckTimer(turnId, "OpenCode did not start its requested final response."); const opening = this.#turnHitOutputLimit(turnId) ? "Your previous response reached the output limit and was discarded as incomplete." - : "Your response ended without a user-visible final answer or an accepted completion report."; + : this.#lastToolCallMarkupSourceSeq > this.#turnStartSourceSeq + ? "Your previous response wrote a tool call as plain text, so the tool did not run. Call tools only through the tool-calling interface, never as text." + : "Your response ended without a user-visible final answer or an accepted completion report."; void this.#submitPrompt(`${opening} Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.`, "disabled") .catch(() => this.#failEmptyResponse(turnId, "OpenCode could not request a final response after an empty completion.")); return true; @@ -2179,11 +2184,17 @@ class OpenCodeHarnessSession implements HarnessSession { // the start of a completed reasoning part, and release it once it differs. if (partType === "reasoning" && content.trim()) this.#reasoningTexts.set(partId, content.trim()); const reasoningEcho = partType === "text" && this.#echoesReasoning(content, completed); - if (!reasoningEcho) this.#partText.set(partId, content); - if (partType === "text" && content.length > 0 && !reasoningEcho) { + const markupStart = partType === "text" && completed ? toolCallMarkupStart(content) : -1; + const visible = partType !== "text" + ? content + : completed + ? markupStart < 0 ? content : content.slice(0, markupStart) + : visibleBeforeToolCallMarkup(content); + if (!reasoningEcho) this.#partText.set(partId, visible); + if (partType === "text" && visible.length > 0 && !reasoningEcho) { for (const reference of paperclipWorkspaceFileReferencesFromText( this.#workingDirectory, - content, + visible, turnId, )) { if (this.#emittedFileReferences.has(reference.referenceId)) continue; @@ -2195,9 +2206,9 @@ class OpenCodeHarnessSession implements HarnessSession { ); } } - const delta = content.startsWith(previous) - ? content.slice(previous.length) - : content; + const delta = visible.startsWith(previous) + ? visible.slice(previous.length) + : visible; if (delta && !reasoningEcho) { // OpenCode calls assistant prose a `text` part, but `text` is not a PRP // item identity and is consequently ignored by the task projection. @@ -2284,6 +2295,15 @@ class OpenCodeHarnessSession implements HarnessSession { ); return; } + if (markupStart >= 0) { + this.#lastToolCallMarkupSourceSeq = this.#sourceSequence; + this.#emit( + "harness.diagnostic", + { code: "opencode_tool_call_markup_discarded", message: "OpenCode returned a tool call as text instead of a tool call; the text was not used as a response." }, + { turnId }, + ); + return; + } this.#completedTextParts.push({ partId, messageId, @@ -2309,7 +2329,7 @@ class OpenCodeHarnessSession implements HarnessSession { // be a final response. When no such text exists, emit no final agentMessage // and let the accepted semantic summary resolve the durable reply. const afterLastWorkTool = (part: { observedSourceSeq: number }) => - part.observedSourceSeq > this.#lastNonTerminalToolSourceSeq; + part.observedSourceSeq > Math.max(this.#lastNonTerminalToolSourceSeq, this.#lastToolCallMarkupSourceSeq); const eligibleText = this.#completedTextParts.filter(part => afterLastWorkTool(part) && !this.#isOutputLimited(part) && !this.#isReasoningEcho(part)); // OpenCode exposes no prose channel, so final selection must use provider // structure rather than text length or content. When OpenCode reports the diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts new file mode 100644 index 00000000000..625e9513618 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { containsToolCallMarkup, visibleBeforeToolCallMarkup } from "./tool-call-markup.js"; + +const dsml = "<|DSML| calls>\n<|DSML| invoke name=\"paperclip_paperclip_finish\">\n<|DSML| parameter name=\"summary\" string=\"true\">Done.\n\n"; +const markupForms = { + "recorded DSML block": dsml, + "ASCII-bar DSML": "<|DSML| invoke name=\"x\">", + "DeepSeek call tokens": "<|tool▁calls▁begin|><|tool▁call▁begin|>function", + "ASCII call tokens": "<|tool_calls_begin|><|tool_call_begin|>function", + "function_calls wrapper": "\n\n\n", + "tool_call wrapper": "\n{\"name\": \"paperclip_finish\", \"arguments\": {}}\n", +}; + +describe("leaked tool-call markup", () => { + it.each(Object.entries(markupForms))("detects %s alone and after prose", (_label, markup) => { + expect(containsToolCallMarkup(`\n\n${markup}`)).toBe(true); + expect(containsToolCallMarkup(`Saved the file.\n${markup}`)).toBe(true); + }); + + it("detects a sentinel inline after prose", () => { + expect(containsToolCallMarkup(`Saved the file.${dsml}`)).toBe(true); + }); + + it.each([ + ["plain prose", "Created the file and saved the plan."], + ["prose naming a tag", "The model emits a tag when it wants a tool."], + ["comparison operators", "Use x < 3 and y > 2."], + ["HTML answer", "
\n

Hello

\n
"], + ["wrapper without a call body", "\nis the tag Hermes models use."], + ["markup in a backtick fence", `Here is the format:\n\`\`\`\n${dsml}\n\n\n\`\`\`\nThat is all.`], + ["markup in a tilde fence", `~~~text\n${dsml}\n~~~`], + ["shorter fence lines inside a longer fence", `\`\`\`\`markdown\n\`\`\`\n${dsml}\n\`\`\`\n\`\`\`\``], + ["other fence characters inside a fence", `~~~\n\`\`\`\n${dsml}\n~~~`], + ["an indented fence line inside a top-level fence", `\`\`\`\n \`\`\`\n${dsml}\n\`\`\``], + ["a list-indented fence", `10. Example:\n\n \`\`\`\n ${dsml}\n \`\`\``], + ])("ignores %s", (_label, text) => { + expect(containsToolCallMarkup(text)).toBe(false); + }); + + it("detects markup after a closed fence", () => { + expect(containsToolCallMarkup(`\`\`\`\`\nexample\n\`\`\`\`\n${dsml}`)).toBe(true); + }); + + it.each(Object.entries(markupForms))("streams %s without showing any of it, at every split point", (_label, markup) => { + const answer = `Saved the file. Done <\n\n${markup}`; + let shown = ""; + for (let end = 1; end <= answer.length; end += 1) { + const visible = visibleBeforeToolCallMarkup(answer.slice(0, end)); + expect(visible.startsWith(shown)).toBe(true); + shown = visible; + } + expect(shown).toBe("Saved the file. Done <\n\n"); + }); + + it("releases a held fragment once it cannot become markup", () => { + expect(visibleBeforeToolCallMarkup("a < b")).toBe("a < b"); + expect(visibleBeforeToolCallMarkup("Done <")).toBe("Done "); + expect(visibleBeforeToolCallMarkup("Done <|tool_call")).toBe("Done "); + expect(visibleBeforeToolCallMarkup("Done <|tool_call>")).toBe("Done <|tool_call>"); + expect(visibleBeforeToolCallMarkup("Done [ + `<${bar}DSML${bar}`, + ...["tool▁calls▁begin", "tool_calls_begin", "tool▁call▁begin", "tool_call_begin"].map(name => `<${bar}${name}${bar}>`), +]); + +const LINE_FORMS = [ + { wrapper: "", body: '\s*", body: '{"name":', pattern: /^\s*\{\s*"name"\s*:/ }, +]; + +const LOOK_AHEAD = 256; + +function couldBecomeLineForm(rest: string): boolean { + const compact = rest.replace(/\s+/g, ""); + if (!compact) return false; + return LINE_FORMS.some(({ wrapper, body }) => + compact.length <= wrapper.length + ? wrapper.startsWith(compact) + : compact.startsWith(wrapper) && body.startsWith(compact.slice(wrapper.length))); +} + +function firstSentinel(line: string): number { + let first = -1; + for (const form of SENTINEL_FORMS) { + const index = line.indexOf(form); + if (index >= 0 && (first < 0 || index < first)) first = index; + } + return first; +} + +function fenceOf(line: string): { indent: number; char: string; length: number; rest: string } | null { + const match = /^([ \t]*)(`{3,}|~{3,})(.*)$/.exec(line); + return match + ? { indent: match[1]!.replace(/\t/g, " ").length, char: match[2]![0]!, length: match[2]!.length, rest: match[3]! } + : null; +} + +export function toolCallMarkupStart(content: string, options: { streaming?: boolean } = {}): number { + if (!content.includes("<")) return -1; + let offset = 0; + let openFence: { indent: number; char: string; length: number } | null = null; + const lines = content.split("\n"); + for (const [index, line] of lines.entries()) { + const lineStart = offset; + offset += line.length + 1; + const fence = fenceOf(line); + if (openFence) { + if (fence && fence.indent <= openFence.indent + 3 && fence.char === openFence.char && fence.length >= openFence.length && !fence.rest.trim()) + openFence = null; + continue; + } + if (fence && !(fence.char === "`" && fence.rest.includes("`"))) { + openFence = { indent: fence.indent, char: fence.char, length: fence.length }; + continue; + } + if (!line.trim()) continue; + const sentinel = firstSentinel(line); + if (sentinel >= 0) return lineStart + sentinel; + const indent = line.length - line.trimStart().length; + const rest = line[indent] === "<" ? content.slice(lineStart + indent, lineStart + indent + LOOK_AHEAD) : ""; + if (rest && LINE_FORMS.some(({ pattern }) => pattern.test(rest))) return lineStart + indent; + if (!options.streaming) continue; + if (rest && couldBecomeLineForm(rest)) return lineStart + indent; + if (index === lines.length - 1) { + const lastOpen = line.lastIndexOf("<"); + if (lastOpen >= 0 && SENTINEL_FORMS.some(form => form.startsWith(line.slice(lastOpen)))) return lineStart + lastOpen; + } + } + return -1; +} + +export function containsToolCallMarkup(content: string): boolean { + return toolCallMarkupStart(content) >= 0; +} + +export function visibleBeforeToolCallMarkup(content: string): string { + const start = toolCallMarkupStart(content, { streaming: true }); + return start < 0 ? content : content.slice(0, start); +}