diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 6d03d051c3a..dff8d69d3e2 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -2600,6 +2600,53 @@ describe("OpenCodeServerDriver", () => { await session.close({ reason: "test" }); }); + it("never settles a post-completion response truncated at the output limit", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); + const workspace = await mkdtemp( + join(tmpdir(), "paperclip-opencode-workspace-"), + ); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { + PATH: process.env.PATH, + OPENROUTER_API_KEY: "fixture-key", + }, + }); + const session = await driver.openSession({ + runId: "run-output-limit-after-tool", + normalizedSessionId: "output-limit-after-tool", + workingDirectory: workspace, + }); + await session.startTurn({ + message: { role: "user", text: "output-limit-after-tool" }, + }); + const events = await collectTurnEvents(session.events()); + // The accepted completion report's summary resolves the durable reply; + // the runaway text is incomplete by the provider's own finish reason. + expect(events.some((event) => event.eventType === "run.result.proposed")).toBe(true); + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect( + events.filter( + (event) => + event.eventType === "item.completed" && + event.payload.kind === "agentMessage" && + event.payload.channel === "final", + ), + ).toHaveLength(0); + expect( + events.some( + (event) => + event.eventType === "harness.diagnostic" && + event.payload.code === "opencode_output_limit_reached", + ), + ).toBe(true); + await session.close({ reason: "test" }); + }); + it("uses provider structure rather than prose length to select a post-tool answer", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 94808372eaf..bf623db3f9d 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -512,6 +512,8 @@ class OpenCodeHarnessSession implements HarnessSession { observedSourceSeq: number; }> = []; readonly #messageUsageFingerprints = new Map(); + /** Assistant messages OpenCode stopped at the model output limit. */ + readonly #outputLimitedMessageIds = new Set(); readonly #workspaceChangesByTurn = new Map>(); #reportedSseCompaction = false; readonly #emittedFileReferences = new Set(); @@ -655,6 +657,7 @@ class OpenCodeHarnessSession implements HarnessSession { this.#completedTextPartIds.clear(); this.#completedReasoningPartIds.clear(); this.#completedTextParts.length = 0; + this.#outputLimitedMessageIds.clear(); // `#terminalTurns` clears here for its own persisted-snapshot bookkeeping. // The late-frame gate in `#emit` does not depend on this map: it compares // against `#activeTurnId` directly, so a frame for the just-finished turn @@ -771,11 +774,23 @@ class OpenCodeHarnessSession implements HarnessSession { #failEmptyResponse(turnId: string, message: string): void { if (this.#closed || this.#activeTurnId !== turnId || this.#interruptedTurnId === turnId || this.#result !== null || this.#pendingToolCalls > 0 || this.#runningNativeToolParts.size > 0) return; - this.#emit("turn.failed", { status: "failed", error: { code: "opencode_empty_response", message, retryable: false } }, { turnId }); + const code = this.#turnHitOutputLimit(turnId) ? "opencode_output_limit_reached" : "opencode_empty_response"; + this.#emit("turn.failed", { status: "failed", error: { code, message, retryable: false } }, { turnId }); this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" })); this.#activeTurnId = null; } + /** A response cut off at the output limit is incomplete, so it is never a final answer. */ + #isOutputLimited(part: { messageId: string | null }): boolean { + return part.messageId !== null && this.#outputLimitedMessageIds.has(part.messageId); + } + + #turnHitOutputLimit(turnId: string): boolean { + for (const messageId of this.#outputLimitedMessageIds) + if (this.#messageTurnIds.get(messageId) === turnId) return true; + return false; + } + #recoverEmptyResponse(turnId: string): boolean { if (this.#result !== null || this.#pendingRuntimeRequests.size > 0) return false; if (this.#pendingToolCalls > 0 || this.#runningNativeToolParts.size > 0) return true; @@ -784,7 +799,7 @@ class OpenCodeHarnessSession implements HarnessSession { if (this.#interruptedTurnId === turnId) return false; const recovery = this.#emptyResponseRecovery; const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq); - if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary)) return false; + if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary && !this.#isOutputLimited(part))) return false; // Both idle event forms can remain queued from the empty response. Wait // for correction evidence. A completed answer part is sufficient even // when its message header omits completion metadata. An idle response @@ -802,7 +817,10 @@ class OpenCodeHarnessSession implements HarnessSession { this.#emptyResponseRecovery = { turnId, priorMessages: new Set(this.#messageRoles.keys()), responseStarted: false, responseFinished: false }; this.#emit("harness.diagnostic", { code: "opencode_empty_response_correction", message: "Requesting one final response after an empty OpenCode completion." }, { turnId }); this.#armEmptyResponseAckTimer(turnId, "OpenCode did not start its requested final response."); - void this.#submitPrompt("Your response ended without a user-visible final answer or an accepted completion report. Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.") + const opening = this.#turnHitOutputLimit(turnId) + ? "Your previous response reached the output limit and was discarded as incomplete." + : "Your response ended without a user-visible final answer or an accepted completion report."; + void this.#submitPrompt(`${opening} Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.`) .catch(() => this.#failEmptyResponse(turnId, "OpenCode could not request a final response after an empty completion.")); return true; } @@ -1823,6 +1841,12 @@ class OpenCodeHarnessSession implements HarnessSession { // legitimately reclaims ownership; a stale message never sees this // branch again, so its recorded owner never changes. if (messageId && role && turnId) this.#messageTurnIds.set(messageId, turnId); + if (role === "assistant" && messageId && text(info.finish) === "length" && !this.#outputLimitedMessageIds.has(messageId)) { + this.#outputLimitedMessageIds.add(messageId); + const limitedTurnId = this.#messageTurnIds.get(messageId) ?? turnId; + if (limitedTurnId) this.#emit("harness.diagnostic", { code: "opencode_output_limit_reached", + message: "OpenCode stopped a response at the model output limit; its truncated text is not used as a final answer." }, { turnId: limitedTurnId }); + } const owningTurnId = messageId ? (this.#messageTurnIds.get(messageId) ?? turnId) : turnId; @@ -2168,7 +2192,7 @@ class OpenCodeHarnessSession implements HarnessSession { // and let the accepted semantic summary resolve the durable reply. const afterLastWorkTool = (part: { observedSourceSeq: number }) => part.observedSourceSeq > this.#lastNonTerminalToolSourceSeq; - const eligibleText = this.#completedTextParts.filter(afterLastWorkTool); + const eligibleText = this.#completedTextParts.filter(part => afterLastWorkTool(part) && !this.#isOutputLimited(part)); // OpenCode exposes no prose channel, so final selection must use provider // structure rather than text length or content. When OpenCode reports the // terminal tool part, prefer prose completed afterward in that exact diff --git a/packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts b/packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts new file mode 100644 index 00000000000..b3fbe9cf6b7 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/output-limit.test.ts @@ -0,0 +1,93 @@ +import { chmod, copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it, vi } from "vitest"; +import { OpenCodeServerDriver } from "./opencode-server-driver.js"; +import type { PrpEvent } from "../../protocol/replay-contract.js"; + +const RUNAWAY = `E2E_${"QUIP_".repeat(400)}`; +const ANSWER = "Here is the actual final answer."; + +// A provider message that OpenCode finished with `length` was cut off at the +// model output limit. Its text is incomplete and must never become the final +// answer; the turn gets the same single correction as an empty response. +it.each(["limit-then-answer", "limit-twice", "normal"] as const)("does not settle a truncated OpenCode response: %s", async mode => { + const root = await mkdtemp(join(tmpdir(), "opencode-output-limit-")); + const command = join(root, "fake-opencode.mjs"); + await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command); + await chmod(command, 0o755); + const prompts: Record[] = []; + let stream: ReadableStreamDefaultController | undefined; + let streamOpened!: () => void; + const opened = new Promise(resolve => { streamOpened = resolve; }); + const queued: Uint8Array[] = []; + const emit = (value: unknown) => { const frame = new TextEncoder().encode(`data: ${JSON.stringify(value)}\n\n`); if (stream) stream.enqueue(frame); else queued.push(frame); }; + const driver = new OpenCodeServerDriver({ + model: "openrouter/controlled-model", runtimeDirectory: join(root, "runtime"), command, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "controlled-fixture-key" }, + fetch: async (input, init) => { + const url = new URL(String(input)); + if (url.pathname === "/event") return new Response(new ReadableStream({ start(controller) { stream = controller; for (const frame of queued.splice(0)) controller.enqueue(frame); streamOpened(); } }), { headers: { "Content-Type": "text/event-stream" } }); + if (url.pathname.endsWith("/prompt_async")) { + const prompt = JSON.parse(String(init?.body)); prompts.push(prompt); + const sessionID = decodeURIComponent(url.pathname.split("/")[2]!); + const number = prompts.length, messageID = `assistant-${number}`; + const limited = mode === "limit-twice" || mode === "limit-then-answer" && number === 1; + queueMicrotask(() => { + emit({ type: "message.updated", properties: { info: { id: messageID, sessionID, role: "assistant" } } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID, + type: "text", text: limited ? RUNAWAY : ANSWER, time: { start: 1, end: 2 } } } }); + emit({ type: "message.updated", properties: { info: { id: messageID, sessionID, role: "assistant", + finish: limited ? "length" : "stop", time: { completed: Date.now() }, tokens: { output: limited ? 32_000 : 12 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + }); + return new Response(null, { status: 204 }); + } + return fetch(input, init); + }, + }); + const session = await driver.openSession({ runId: "owned-run", normalizedSessionId: "owned-session", workingDirectory: root }); + let deadline: ReturnType | undefined; + try { + await opened; + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const collect = async () => { const rows: PrpEvent[] = []; for await (const event of session.events()) { + rows.push(event); if (["turn.completed", "turn.failed"].includes(event.eventType)) break; + } return rows; }; + const collected = collect(); + await session.startTurn({ message: { role: "user", text: "Answer the current request." }, reasoningMode: "disabled" }); + const completed = Promise.race([collected, new Promise((_, reject) => { + deadline = setTimeout(() => reject(new Error("Controlled provider produced no terminal event")), 5_000); + })]); + void completed.catch(() => {}); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(5_001); + const events = await completed; + const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final"); + const diagnostics = events.filter(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_output_limit_reached"); + if (mode === "normal") { + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect(final.map(event => event.payload.text)).toEqual([ANSWER]); + expect(diagnostics).toHaveLength(0); + expect(prompts).toHaveLength(1); + return; + } + expect(prompts).toHaveLength(2); + expect(JSON.stringify(prompts[1]!.parts)).toContain("reached the output limit and was discarded"); + expect(JSON.stringify(final)).not.toContain("QUIP"); + expect(diagnostics.length).toBeGreaterThanOrEqual(1); + if (mode === "limit-then-answer") { + expect(events.at(-1)?.eventType).toBe("turn.completed"); + expect(final.map(event => event.payload.text)).toEqual([ANSWER]); + } else { + expect(events.at(-1)?.eventType).toBe("turn.failed"); + expect(events.at(-1)?.payload).toMatchObject({ error: { code: "opencode_output_limit_reached" } }); + expect(final).toHaveLength(0); + } + } finally { + clearTimeout(deadline); + vi.useRealTimers(); + await session.close({ reason: "controlled test complete" }); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs index 74f483c4790..f9891b9ec4a 100644 --- a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs +++ b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs @@ -714,6 +714,9 @@ const server = createServer(async (request, response) => { const finalAfterToolCommentary = String( parsedPrompt.message ?? "", ).includes("final-after-tool-commentary"); + const outputLimitAfterTool = String( + parsedPrompt.message ?? "", + ).includes("output-limit-after-tool"); const commentaryOnlyBeforeWork = String( parsedPrompt.message ?? "", ).includes("commentary-only-before-work"); @@ -806,6 +809,53 @@ const server = createServer(async (request, response) => { }, }); await callTerminalTool(promptPayload); + } else if (outputLimitAfterTool) { + await callTerminalTool(promptPayload); + emit({ + type: "message.part.updated", + id: "event-limit-tool-part", + properties: { + sessionID: session.id, + part: { + id: "part-limit-tool", + messageID: "message-assistant", + type: "tool", + tool: "paperclip_paperclip_finish", + state: { status: "completed", output: "accepted" }, + }, + }, + }); + emit({ + type: "message.updated", + id: "event-limit-message", + properties: { + sessionID: session.id, + info: { id: "message-limit", sessionID: session.id, role: "assistant" }, + }, + }); + emit({ + type: "message.part.updated", + id: "event-limit-text", + properties: { + sessionID: session.id, + part: { + id: "part-limit-text", + messageID: "message-limit", + type: "text", + text: `E2E_${"QUIP_".repeat(400)}`, + time: { start: 3, end: 4 }, + }, + }, + }); + emit({ + type: "message.updated", + id: "event-limit-message-finished", + properties: { + sessionID: session.id, + info: { id: "message-limit", sessionID: session.id, role: "assistant", + finish: "length", time: { completed: 5 } }, + }, + }); } else if (finalAfterToolCommentary) { emit({ type: "message.part.updated", diff --git a/server/src/services/native-runtime/native-provider-failure.ts b/server/src/services/native-runtime/native-provider-failure.ts index de2ef11c21c..71d040566bb 100644 --- a/server/src/services/native-runtime/native-provider-failure.ts +++ b/server/src/services/native-runtime/native-provider-failure.ts @@ -79,7 +79,7 @@ export interface NativeCodexTurnFailureDiagnostic { */ const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet = new Set([ "opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing", - "opencode_empty_response", + "opencode_empty_response", "opencode_output_limit_reached", ]); const PROVIDER_FAILURE_CATEGORIES: ReadonlySet = new Set([ "rate_limit", "quota_exhausted", "authentication", "permission", "model_unavailable", "invalid_request",