From 26391707fb5ba3ee8fa698141326df0fd5d9b145 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Thu, 5 Nov 2026 08:11:55 -0600 Subject: [PATCH] fix(runner): never settle an OpenCode reply truncated at the output limit (#19242) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Native OpenCode runs turn the provider's last assistant text into the task's visible reply > - A model can loop until it reaches the output limit. The provider then reports `finish: "length"` > - The driver used that cut-off text as the final answer, and the run succeeded with a broken reply > - This pull request treats a response that stopped at the output limit as incomplete > - The benefit is that users get the accepted completion summary or one corrected answer, never a truncated wall of text ## Linked Issues or Issue Description **What happened?** In a QA staging run, an OpenCode agent on `openrouter/deepseek/deepseek-v4.1-flash-1017` called `paperclip_finish` correctly. Then it produced about 32,000 output tokens of repeated text (`E2E_QUIP_QUIP_...`) until the output limit stopped it. The driver selected this truncated text as the settled final answer. The run succeeded, and the task reply was wrong. **Expected behavior** A response that the provider stopped at the output limit is not a complete answer. If an accepted completion report exists, its summary is the reply. If not, the driver asks for one concise final answer. If that also stops at the limit, the turn fails. **Steps to reproduce** 1. Run a native OpenCode turn where the model calls `paperclip_finish`. 2. Let the next assistant message run to the output limit (OpenCode reports `finish: "length"`). 3. Before this change, that message text is emitted as the final `agentMessage`. **Paperclip version or commit** master at e8163fc860. ## What Changed - `opencode-server-driver.ts`: record assistant messages that finish with `length`. Emit a `harness.diagnostic` with code `opencode_output_limit_reached`. - Exclude text from these messages from final-answer selection and from the empty-response check. An accepted completion summary then resolves the reply (existing behavior when no final text exists). - Without an accepted result, reuse the existing single correction prompt. Its first sentence says that the previous response reached the output limit and was discarded. - If the correction also fails, the turn fails with `opencode_output_limit_reached`, not `opencode_empty_response`. - `native-provider-failure.ts`: add `opencode_output_limit_reached` to the known OpenCode runner failure codes. - Tests: new `output-limit.test.ts` (limit then answer, limit twice, normal), and a driver test for a runaway reply after an accepted `paperclip_finish`, with a new fake-server scenario. ## Verification - `npx vitest run src/drivers/opencode` in `packages/paperclip-runner`: 13 files, 166 tests passed. - Negative controls: with the old driver, the two new limit cases and the post-completion case fail. - `native-provider-failure.test.ts`: 57 tests passed. - The live QA staging journey is not verified yet. That needs a staging deploy of this change. ## Risks - A long but valid answer that ends exactly at the output limit is now discarded and gets one correction request. The provider itself marks such an answer as cut off, so it was already incomplete. - When pre-tool commentary exists and the only post-tool text is truncated, existing selection rules can pick that commentary. This is the same as the existing behavior when no post-tool text exists. - Only the OpenCode driver changes. Other drivers are not affected. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, browser, code editing). Extended reasoning was used. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 --- .../opencode/opencode-server-driver.test.ts | 47 ++++++++++ .../opencode/opencode-server-driver.ts | 32 ++++++- .../src/drivers/opencode/output-limit.test.ts | 93 +++++++++++++++++++ .../test/fixtures/fake-opencode-server.mjs | 50 ++++++++++ .../native-runtime/native-provider-failure.ts | 2 +- 5 files changed, 219 insertions(+), 5 deletions(-) create mode 100644 packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 6d03d051c3a..dff8d69d3e2 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -2600,6 +2600,53 @@ describe("OpenCodeServerDriver", () => { await session.close({ reason: "test" }); }); + it("never settles a post-completion response truncated at the output limit", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); + const workspace = await mkdtemp( + join(tmpdir(), "paperclip-opencode-workspace-"), + ); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { + PATH: process.env.PATH, + OPENROUTER_API_KEY: "fixture-key", + }, + }); + const session = await driver.openSession({ + runId: "run-output-limit-after-tool", + normalizedSessionId: "output-limit-after-tool", + workingDirectory: workspace, + }); + await session.startTurn({ + message: { role: "user", text: "output-limit-after-tool" }, + }); + const events = await collectTurnEvents(session.events()); + // The accepted completion report's summary resolves the durable reply; + // the runaway text is incomplete by the provider's own finish reason. + expect(events.some((event) => event.eventType === "run.result.proposed")).toBe(true); + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect( + events.filter( + (event) => + event.eventType === "item.completed" && + event.payload.kind === "agentMessage" && + event.payload.channel === "final", + ), + ).toHaveLength(0); + expect( + events.some( + (event) => + event.eventType === "harness.diagnostic" && + event.payload.code === "opencode_output_limit_reached", + ), + ).toBe(true); + await session.close({ reason: "test" }); + }); + it("uses provider structure rather than prose length to select a post-tool answer", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 94808372eaf..bf623db3f9d 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -512,6 +512,8 @@ class OpenCodeHarnessSession implements HarnessSession { observedSourceSeq: number; }> = []; readonly #messageUsageFingerprints = new Map(); + /** Assistant messages OpenCode stopped at the model output limit. */ + readonly #outputLimitedMessageIds = new Set(); readonly #workspaceChangesByTurn = new Map>(); #reportedSseCompaction = false; readonly #emittedFileReferences = new Set(); @@ -655,6 +657,7 @@ class OpenCodeHarnessSession implements HarnessSession { this.#completedTextPartIds.clear(); this.#completedReasoningPartIds.clear(); this.#completedTextParts.length = 0; + this.#outputLimitedMessageIds.clear(); // `#terminalTurns` clears here for its own persisted-snapshot bookkeeping. // The late-frame gate in `#emit` does not depend on this map: it compares // against `#activeTurnId` directly, so a frame for the just-finished turn @@ -771,11 +774,23 @@ class OpenCodeHarnessSession implements HarnessSession { #failEmptyResponse(turnId: string, message: string): void { if (this.#closed || this.#activeTurnId !== turnId || this.#interruptedTurnId === turnId || this.#result !== null || this.#pendingToolCalls > 0 || this.#runningNativeToolParts.size > 0) return; - this.#emit("turn.failed", { status: "failed", error: { code: "opencode_empty_response", message, retryable: false } }, { turnId }); + const code = this.#turnHitOutputLimit(turnId) ? "opencode_output_limit_reached" : "opencode_empty_response"; + this.#emit("turn.failed", { status: "failed", error: { code, message, retryable: false } }, { turnId }); this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" })); this.#activeTurnId = null; } + /** A response cut off at the output limit is incomplete, so it is never a final answer. */ + #isOutputLimited(part: { messageId: string | null }): boolean { + return part.messageId !== null && this.#outputLimitedMessageIds.has(part.messageId); + } + + #turnHitOutputLimit(turnId: string): boolean { + for (const messageId of this.#outputLimitedMessageIds) + if (this.#messageTurnIds.get(messageId) === turnId) return true; + return false; + } + #recoverEmptyResponse(turnId: string): boolean { if (this.#result !== null || this.#pendingRuntimeRequests.size > 0) return false; if (this.#pendingToolCalls > 0 || this.#runningNativeToolParts.size > 0) return true; @@ -784,7 +799,7 @@ class OpenCodeHarnessSession implements HarnessSession { if (this.#interruptedTurnId === turnId) return false; const recovery = this.#emptyResponseRecovery; const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq); - if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary)) return false; + if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary && !this.#isOutputLimited(part))) return false; // Both idle event forms can remain queued from the empty response. Wait // for correction evidence. A completed answer part is sufficient even // when its message header omits completion metadata. An idle response @@ -802,7 +817,10 @@ class OpenCodeHarnessSession implements HarnessSession { this.#emptyResponseRecovery = { turnId, priorMessages: new Set(this.#messageRoles.keys()), responseStarted: false, responseFinished: false }; this.#emit("harness.diagnostic", { code: "opencode_empty_response_correction", message: "Requesting one final response after an empty OpenCode completion." }, { turnId }); this.#armEmptyResponseAckTimer(turnId, "OpenCode did not start its requested final response."); - void this.#submitPrompt("Your response ended without a user-visible final answer or an accepted completion report. Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.") + const opening = this.#turnHitOutputLimit(turnId) + ? "Your previous response reached the output limit and was discarded as incomplete." + : "Your response ended without a user-visible final answer or an accepted completion report."; + void this.#submitPrompt(`${opening} Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.`) .catch(() => this.#failEmptyResponse(turnId, "OpenCode could not request a final response after an empty completion.")); return true; } @@ -1823,6 +1841,12 @@ class OpenCodeHarnessSession implements HarnessSession { // legitimately reclaims ownership; a stale message never sees this // branch again, so its recorded owner never changes. if (messageId && role && turnId) this.#messageTurnIds.set(messageId, turnId); + if (role === "assistant" && messageId && text(info.finish) === "length" && !this.#outputLimitedMessageIds.has(messageId)) { + this.#outputLimitedMessageIds.add(messageId); + const limitedTurnId = this.#messageTurnIds.get(messageId) ?? turnId; + if (limitedTurnId) this.#emit("harness.diagnostic", { code: "opencode_output_limit_reached", + message: "OpenCode stopped a response at the model output limit; its truncated text is not used as a final answer." }, { turnId: limitedTurnId }); + } const owningTurnId = messageId ? (this.#messageTurnIds.get(messageId) ?? turnId) : turnId; @@ -2168,7 +2192,7 @@ class OpenCodeHarnessSession implements HarnessSession { // and let the accepted semantic summary resolve the durable reply. const afterLastWorkTool = (part: { observedSourceSeq: number }) => part.observedSourceSeq > this.#lastNonTerminalToolSourceSeq; - const eligibleText = this.#completedTextParts.filter(afterLastWorkTool); + const eligibleText = this.#completedTextParts.filter(part => afterLastWorkTool(part) && !this.#isOutputLimited(part)); // OpenCode exposes no prose channel, so final selection must use provider // structure rather than text length or content. When OpenCode reports the // terminal tool part, prefer prose completed afterward in that exact diff --git a/packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts b/packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts new file mode 100644 index 00000000000..b3fbe9cf6b7 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts @@ -0,0 +1,93 @@ +import { chmod, copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { OpenCodeServerDriver } from "./opencode-server-driver.js"; +import type { PrpEvent } from "../../protocol/replay-contract.js"; + +const RUNAWAY = `E2E_${"QUIP_".repeat(400)}`; +const ANSWER = "Here is the actual final answer."; + +// A provider message that OpenCode finished with `length` was cut off at the +// model output limit. Its text is incomplete and must never become the final +// answer; the turn gets the same single correction as an empty response. +it.each(["limit-then-answer", "limit-twice", "normal"] as const)("does not settle a truncated OpenCode response: %s", async mode => { + const root = await mkdtemp(join(tmpdir(), "opencode-output-limit-")); + const command = join(root, "fake-opencode.mjs"); + await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command); + await chmod(command, 0o755); + const prompts: Record[] = []; + let stream: ReadableStreamDefaultController | undefined; + let streamOpened!: () => void; + const opened = new Promise(resolve => { streamOpened = resolve; }); + const queued: Uint8Array[] = []; + const emit = (value: unknown) => { const frame = new TextEncoder().encode(`data: ${JSON.stringify(value)}\n\n`); if (stream) stream.enqueue(frame); else queued.push(frame); }; + const driver = new OpenCodeServerDriver({ + model: "openrouter/controlled-model", runtimeDirectory: join(root, "runtime"), command, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "controlled-fixture-key" }, + fetch: async (input, init) => { + const url = new URL(String(input)); + if (url.pathname === "/event") return new Response(new ReadableStream({ start(controller) { stream = controller; for (const frame of queued.splice(0)) controller.enqueue(frame); streamOpened(); } }), { headers: { "Content-Type": "text/event-stream" } }); + if (url.pathname.endsWith("/prompt_async")) { + const prompt = JSON.parse(String(init?.body)); prompts.push(prompt); + const sessionID = decodeURIComponent(url.pathname.split("/")[2]!); + const number = prompts.length, messageID = `assistant-${number}`; + const limited = mode === "limit-twice" || mode === "limit-then-answer" && number === 1; + queueMicrotask(() => { + emit({ type: "message.updated", properties: { info: { id: messageID, sessionID, role: "assistant" } } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID, + type: "text", text: limited ? RUNAWAY : ANSWER, time: { start: 1, end: 2 } } } }); + emit({ type: "message.updated", properties: { info: { id: messageID, sessionID, role: "assistant", + finish: limited ? "length" : "stop", time: { completed: Date.now() }, tokens: { output: limited ? 32_000 : 12 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + }); + return new Response(null, { status: 204 }); + } + return fetch(input, init); + }, + }); + const session = await driver.openSession({ runId: "owned-run", normalizedSessionId: "owned-session", workingDirectory: root }); + let deadline: ReturnType | undefined; + try { + await opened; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const collect = async () => { const rows: PrpEvent[] = []; for await (const event of session.events()) { + rows.push(event); if (["turn.completed", "turn.failed"].includes(event.eventType)) break; + } return rows; }; + const collected = collect(); + await session.startTurn({ message: { role: "user", text: "Answer the current request." }, reasoningMode: "disabled" }); + const completed = Promise.race([collected, new Promise((_, reject) => { + deadline = setTimeout(() => reject(new Error("Controlled provider produced no terminal event")), 5_000); + })]); + void completed.catch(() => {}); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5_001); + const events = await completed; + const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final"); + const diagnostics = events.filter(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_output_limit_reached"); + if (mode === "normal") { + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect(final.map(event => event.payload.text)).toEqual([ANSWER]); + expect(diagnostics).toHaveLength(0); + expect(prompts).toHaveLength(1); + return; + } + expect(prompts).toHaveLength(2); + expect(JSON.stringify(prompts[1]!.parts)).toContain("reached the output limit and was discarded"); + expect(JSON.stringify(final)).not.toContain("QUIP"); + expect(diagnostics.length).toBeGreaterThanOrEqual(1); + if (mode === "limit-then-answer") { + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect(final.map(event => event.payload.text)).toEqual([ANSWER]); + } else { + expect(events.at(-1)?.eventType).toBe("turn.failed"); + expect(events.at(-1)?.payload).toMatchObject({ error: { code: "opencode_output_limit_reached" } }); + expect(final).toHaveLength(0); + } + } finally { + clearTimeout(deadline); + vi.useRealTimers(); + await session.close({ reason: "controlled test complete" }); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs index 74f483c4790..f9891b9ec4a 100644 --- a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs +++ b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs @@ -714,6 +714,9 @@ const server = createServer(async (request, response) => { const finalAfterToolCommentary = String( parsedPrompt.message ?? "", ).includes("final-after-tool-commentary"); + const outputLimitAfterTool = String( + parsedPrompt.message ?? "", + ).includes("output-limit-after-tool"); const commentaryOnlyBeforeWork = String( parsedPrompt.message ?? "", ).includes("commentary-only-before-work"); @@ -806,6 +809,53 @@ const server = createServer(async (request, response) => { }, }); await callTerminalTool(promptPayload); + } else if (outputLimitAfterTool) { + await callTerminalTool(promptPayload); + emit({ + type: "message.part.updated", + id: "event-limit-tool-part", + properties: { + sessionID: session.id, + part: { + id: "part-limit-tool", + messageID: "message-assistant", + type: "tool", + tool: "paperclip_paperclip_finish", + state: { status: "completed", output: "accepted" }, + }, + }, + }); + emit({ + type: "message.updated", + id: "event-limit-message", + properties: { + sessionID: session.id, + info: { id: "message-limit", sessionID: session.id, role: "assistant" }, + }, + }); + emit({ + type: "message.part.updated", + id: "event-limit-text", + properties: { + sessionID: session.id, + part: { + id: "part-limit-text", + messageID: "message-limit", + type: "text", + text: `E2E_${"QUIP_".repeat(400)}`, + time: { start: 3, end: 4 }, + }, + }, + }); + emit({ + type: "message.updated", + id: "event-limit-message-finished", + properties: { + sessionID: session.id, + info: { id: "message-limit", sessionID: session.id, role: "assistant", + finish: "length", time: { completed: 5 } }, + }, + }); } else if (finalAfterToolCommentary) { emit({ type: "message.part.updated", diff --git a/server/src/services/native-runtime/native-provider-failure.ts b/server/src/services/native-runtime/native-provider-failure.ts index de2ef11c21c..71d040566bb 100644 --- a/server/src/services/native-runtime/native-provider-failure.ts +++ b/server/src/services/native-runtime/native-provider-failure.ts @@ -79,7 +79,7 @@ export interface NativeCodexTurnFailureDiagnostic { */ const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet = new Set([ "opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing", - "opencode_empty_response", + "opencode_empty_response", "opencode_output_limit_reached", ]); const PROVIDER_FAILURE_CATEGORIES: ReadonlySet = new Set([ "rate_limit", "quota_exhausted", "authentication", "permission", "model_unavailable", "invalid_request",