diff --git a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts
index 85c0beb8c80..10036e90ab5 100644
--- a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts
+++ b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts
@@ -5,7 +5,7 @@ import { expect, it, vi } from "vitest";
import { OpenCodeServerDriver } from "./opencode-server-driver.js";
import type { PrpEvent } from "../../protocol/replay-contract.js";
-it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "tool-work-text", "tool-work-empty", "stale-tool-work"] as const)("handles an empty OpenCode response: %s", async mode => {
+it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "closing-markup", "control-token-reply", "control-token-trailing", "tool-work-text", "tool-work-empty", "stale-tool-work"] as const)("handles an empty OpenCode response: %s", async mode => {
const root = await mkdtemp(join(tmpdir(), "opencode-empty-response-"));
const command = join(root, "fake-opencode.mjs");
await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command);
@@ -16,6 +16,13 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow
// ordinary text part, and no completion call.
const recordedReasoning = "Task created and already executing (status in_progress, run dafb...). Good. Reply with link. Use finish. Keep concise.\n\nCompletion contract revision 1, criterion \"objective\". I should call paperclip_finish with done. Include evidence refs maybe task link. Let's do.";
const recordedMarkup = "\n\n<|DSML| calls>\n<|DSML| invoke name=\"paperclip_paperclip_finish\">\n<|DSML| parameter name=\"reportedWorkDisposition\" string=\"true\">done|DSML| parameter>\n<|DSML| parameter name=\"summary\" string=\"true\">Created `website-fixture.txt` in the task workspace.|DSML| parameter>\n|DSML| invoke>\n|DSML| calls>";
+ // Recorded from chat-eval cohort 16: closing DSML tags streamed alone as
+ // text (task-coordination.block-on-design), and a reply of a
+ // begin-of-sentence token plus an imitation tool-output fragment
+ // (cases.case-history).
+ const recordedClosingMarkup = "\r\n|DSML| invoke>\n|DSML| calls>\r\n|DSML| invoke>\n|DSML| calls>\r\n\r\n|DSML| parameter>\n";
+ const recordedControlTokenReply = "<|begin▁of▁sentence|>\n";
+ const leakedText = mode === "closing-markup" ? recordedClosingMarkup : mode === "control-token-reply" ? recordedControlTokenReply : recordedMarkup;
let toolTimer: ReturnType | undefined;
let stream: ReadableStreamDefaultController | undefined;
let streamOpened!: () => void;
@@ -95,19 +102,19 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow
emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } });
return;
}
- if (mode === "tool-call-markup" && number === 1) {
+ if (["tool-call-markup", "closing-markup", "control-token-reply"].includes(mode) && number === 1) {
const leaked = { id: "part-1-text", messageID, sessionID, type: "text", text: "", time: { start: 1 } };
emit({ type: "message.part.updated", properties: { sessionID, part: leaked } });
- for (const delta of [recordedMarkup.slice(0, 4), recordedMarkup.slice(4, 9), recordedMarkup.slice(9)])
+ for (const delta of [leakedText.slice(0, 4), leakedText.slice(4, 9), leakedText.slice(9)])
emit({ type: "message.part.delta", properties: { sessionID, messageID, partID: leaked.id, field: "text", delta } });
- emit({ type: "message.part.updated", properties: { sessionID, part: { ...leaked, text: recordedMarkup, time: { start: 1, end: 2 } } } });
+ emit({ type: "message.part.updated", properties: { sessionID, part: { ...leaked, text: leakedText, time: { start: 1, end: 2 } } } });
emit({ type: "session.idle", properties: { sessionID } });
emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } });
return;
}
- const answer = mode === "normal" || mode === "stale-tool-work" && number === 1 || mode === "foreign-idle" || mode === "queued-tool" || number === 3 && mode === "tool-work-text" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"].includes(mode);
+ const answer = mode === "normal" || mode === "control-token-trailing" || mode === "stale-tool-work" && number === 1 || mode === "foreign-idle" || mode === "queued-tool" || number === 3 && mode === "tool-work-text" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "closing-markup", "control-token-reply"].includes(mode);
emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID,
- type: answer ? "text" : "reasoning", text: answer ? (mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer.") : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } });
+ type: answer ? "text" : "reasoning", text: answer ? (mode === "reasoning-echo-correction-matches" ? recordedReasoning : mode === "control-token-trailing" ? "Here is the actual final answer.<|end▁of▁sentence|>" : "Here is the actual final answer.") : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } });
emit({ type: "session.idle", properties: { sessionID } });
emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } });
});
@@ -145,14 +152,15 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow
}
await vi.advanceTimersByTimeAsync(mode.endsWith("idle-stall") ? 20_001 : mode === "slow-native-tool" ? 1_999 : 5_001);
const events = await completed;
- const succeeds = ["stale-tool-work", "tool-work-text", "reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup"].includes(mode);
+ const succeeds = ["stale-tool-work", "tool-work-text", "reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches", "tool-call-markup", "closing-markup", "control-token-reply", "control-token-trailing"].includes(mode);
expect(events.at(-1)?.eventType).toBe(succeeds ? "turn.completed" : "turn.failed");
expect(events.filter(event => event.eventType === "turn.started")).toHaveLength(1);
expect(events.every(event => !event.turnId || event.turnId === turnId)).toBe(true);
const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final");
expect(final.map(event => event.payload.text)).toEqual(succeeds ? [mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer."] : []);
// Only a turn that already did tool work gets the second, plain-text request.
- expect(prompts).toHaveLength(["normal", "foreign-idle", "queued-tool", "stale-tool-work"].includes(mode) ? 1 : mode.startsWith("tool-work") ? 3 : 2);
+ // A trailing end-of-sentence token after a real answer is not a degenerate reply.
+ expect(prompts).toHaveLength(["normal", "foreign-idle", "queued-tool", "stale-tool-work", "control-token-trailing"].includes(mode) ? 1 : mode.startsWith("tool-work") ? 3 : 2);
if (prompts.length === 3) {
expect(prompts[2]).toMatchObject({ providerID: prompts[0]!.providerID, modelID: prompts[0]!.modelID, system: prompts[0]!.system, variant: "paperclip-no-reasoning" });
// Text first, so a chat reply survives a dropped call; then the
@@ -173,13 +181,22 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow
expect(assistantText.join("")).not.toContain(recordedReasoning.slice(0, 20));
expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_reasoning_echo_discarded")).toBe(true);
}
- if (mode === "tool-call-markup") {
+ if (mode === "tool-call-markup" || mode === "closing-markup") {
const assistantText = events.filter(event => event.payload.kind === "agentMessage").map(event => String(event.payload.text ?? ""));
expect(assistantText.join("")).not.toContain("DSML");
expect(assistantText.join("")).not.toContain("<|");
expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_tool_call_markup_discarded")).toBe(true);
expect(JSON.stringify(prompts[1]!.parts)).toContain("wrote a tool call as plain text");
}
+ if (mode === "control-token-reply") {
+ // Neither progress nor the reply shows the degenerate text; the
+ // correction asks for the answer the person is still owed.
+ const assistantText = events.filter(event => event.payload.kind === "agentMessage").map(event => String(event.payload.text ?? ""));
+ expect(assistantText.join("")).not.toContain("begin▁of▁sentence");
+ expect(assistantText.join("")).not.toContain("Search output truncated");
+ expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_control_token_reply_discarded")).toBe(true);
+ expect(JSON.stringify(prompts[1]!.parts)).toContain("raw model control tokens");
+ }
if (mode === "stale-tool-work") {
const nextEvents = collect();
await session.startTurn({ message: { role: "user", text: "Answer the next request." }, reasoningMode: "disabled" });
diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
index bb22a014938..50ce8fbddbf 100644
--- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
+++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts
@@ -95,7 +95,7 @@ import {
type OpenCodeMcpBridge,
type OpenCodeMcpBridgeOptions,
} from "./mcp-bridge.js";
-import { toolCallMarkupStart, visibleBeforeToolCallMarkup } from "./tool-call-markup.js";
+import { controlTokenStart, toolCallMarkupStart, visibleBeforeModelScaffolding } from "./tool-call-markup.js";
import {
materializeNativeRuntimeSkills,
releaseMaterializedNativeRuntimeSkills,
@@ -568,6 +568,7 @@ class OpenCodeHarnessSession implements HarnessSession {
#semanticResultProviderMessageId: string | null = null;
#lastNonTerminalToolSourceSeq = 0;
#lastToolCallMarkupSourceSeq = 0;
+ #lastControlTokenReplySourceSeq = 0;
#usage: Record | null = null;
#upstreamServed: UpstreamServed | null = null;
readonly #conversationMode: "task" | "prepared";
@@ -707,6 +708,7 @@ class OpenCodeHarnessSession implements HarnessSession {
this.#semanticResultProviderMessageId = null;
this.#lastNonTerminalToolSourceSeq = 0;
this.#lastToolCallMarkupSourceSeq = 0;
+ this.#lastControlTokenReplySourceSeq = 0;
this.#completedTextPartIds.clear();
this.#completedReasoningPartIds.clear();
this.#reasoningTexts.clear();
@@ -920,7 +922,9 @@ class OpenCodeHarnessSession implements HarnessSession {
? "Your previous response reached the output limit and was discarded as incomplete."
: this.#lastToolCallMarkupSourceSeq > this.#turnStartSourceSeq
? "Your previous response wrote a tool call as plain text, so the tool did not run. Call tools only through the tool-calling interface, never as text."
- : "Your response ended without a user-visible final answer or an accepted completion report.";
+ : this.#lastControlTokenReplySourceSeq > this.#turnStartSourceSeq
+ ? "Your previous response contained raw model control tokens instead of an answer, so it was discarded."
+ : "Your response ended without a user-visible final answer or an accepted completion report.";
const prompt = attempt === 1
? `${opening} Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.`
: "Your last response was empty again. Start this response with a short plain-text final answer for the user that says what you did and what, if anything, is left. After that text, submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.";
@@ -2210,12 +2214,16 @@ class OpenCodeHarnessSession implements HarnessSession {
// the start of a completed reasoning part, and release it once it differs.
if (partType === "reasoning" && content.trim()) this.#reasoningTexts.set(partId, content.trim());
const reasoningEcho = partType === "text" && this.#echoesReasoning(content, completed);
+ // Tool-call markup and model control tokens are never prose: hide them
+ // from progress as they stream, and keep them out of the reply.
const markupStart = partType === "text" && completed ? toolCallMarkupStart(content) : -1;
+ const controlStart = partType === "text" && completed ? controlTokenStart(content) : -1;
+ const cut = Math.min(...[markupStart, controlStart].filter(start => start >= 0), content.length);
const visible = partType !== "text"
? content
: completed
- ? markupStart < 0 ? content : content.slice(0, markupStart)
- : visibleBeforeToolCallMarkup(content);
+ ? content.slice(0, cut)
+ : visibleBeforeModelScaffolding(content);
if (!reasoningEcho) this.#partText.set(partId, visible);
if (partType === "text" && visible.length > 0 && !reasoningEcho) {
for (const reference of paperclipWorkspaceFileReferencesFromText(
@@ -2330,15 +2338,30 @@ class OpenCodeHarnessSession implements HarnessSession {
);
return;
}
+ // Prose before a control token is the answer (a trailing end-of-turn
+ // token, say). With no prose before it, the part is degenerate output,
+ // so the empty-response correction asks for the real answer. Unlike
+ // tool-call markup it says nothing about earlier prose, which stays
+ // eligible as the reply.
+ const answer = controlStart < 0 ? content : content.slice(0, controlStart).trimEnd();
+ if (!answer.trim()) {
+ this.#lastControlTokenReplySourceSeq = this.#sourceSequence;
+ this.#emit(
+ "harness.diagnostic",
+ { code: "opencode_control_token_reply_discarded", message: "OpenCode returned model control tokens instead of an answer; the text was not used as a response." },
+ { turnId },
+ );
+ return;
+ }
this.#completedTextParts.push({
partId,
messageId,
- text: content,
+ text: answer,
item: bounded({
...part,
type: "agentMessage",
phase: "final_answer",
- text: content,
+ text: answer,
}),
observedSourceSeq: this.#sourceSequence,
});
diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts
index 625e9513618..470c576b6ef 100644
--- a/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts
+++ b/packages/paperclip-runner/src/drivers/opencode/tool-call-markup.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it } from "vitest";
-import { containsToolCallMarkup, visibleBeforeToolCallMarkup } from "./tool-call-markup.js";
+import { containsToolCallMarkup, controlTokenStart, visibleBeforeModelScaffolding, visibleBeforeToolCallMarkup } from "./tool-call-markup.js";
const dsml = "<|DSML| calls>\n<|DSML| invoke name=\"paperclip_paperclip_finish\">\n<|DSML| parameter name=\"summary\" string=\"true\">Done.|DSML| parameter>\n|DSML| invoke>\n|DSML| calls>";
const markupForms = {
@@ -9,6 +9,9 @@ const markupForms = {
"ASCII call tokens": "<|tool_calls_begin|><|tool_call_begin|>function",
"function_calls wrapper": "\n\n\n",
"tool_call wrapper": "\n{\"name\": \"paperclip_finish\", \"arguments\": {}}\n",
+ // Recorded from chat-eval cohort 16 (task-coordination.block-on-design): after
+ // a real paperclip_finish call, DeepSeek streamed only closing tags as text.
+ "closing DSML tags only": "\r\n|DSML| invoke>\n|DSML| calls>\r\n|DSML| invoke>\n|DSML| calls>\r\n\r\n|DSML| parameter>\n\r\n",
};
describe("leaked tool-call markup", () => {
@@ -33,6 +36,7 @@ describe("leaked tool-call markup", () => {
["other fence characters inside a fence", `~~~\n\`\`\`\n${dsml}\n~~~`],
["an indented fence line inside a top-level fence", `\`\`\`\n \`\`\`\n${dsml}\n\`\`\``],
["a list-indented fence", `10. Example:\n\n \`\`\`\n ${dsml}\n \`\`\``],
+ ["a sentinel in inline code", "DeepSeek opens a call with `<|DSML| invoke>` and closes it with `|DSML| invoke>`."],
])("ignores %s", (_label, text) => {
expect(containsToolCallMarkup(text)).toBe(false);
});
@@ -49,7 +53,7 @@ describe("leaked tool-call markup", () => {
expect(visible.startsWith(shown)).toBe(true);
shown = visible;
}
- expect(shown).toBe("Saved the file. Done <\n\n");
+ expect(shown.trimEnd()).toBe("Saved the file. Done <");
});
it("releases a held fragment once it cannot become markup", () => {
@@ -60,3 +64,50 @@ describe("leaked tool-call markup", () => {
expect(visibleBeforeToolCallMarkup("Done \n";
+
+describe("model control tokens", () => {
+ it.each([
+ ["the recorded degenerate reply", degenerateReply, 0],
+ ["a ChatML turn after an answer", "The fact changed twice.\n<|im_start|>user\nthanks", 24],
+ ["a token after an escaped backtick", "Use \\`code. <|im_end|> then `x`.", 12],
+ ["a token after an unmatched backtick", "One ` tick. <|eot_id|>", 12],
+ ["a trailing end-of-sentence token", "Riley changed it.<|end▁of▁sentence|>", 17],
+ ["a Llama end-of-turn token", "Done.<|eot_id|>", 5],
+ ])("finds %s", (_label, text, start) => {
+ expect(controlTokenStart(text)).toBe(start);
+ });
+
+ it.each([
+ ["an XML answer", "The export is XML:\n\n\n overlay\n"],
+ ["an answer about an