diff --git a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs index 74a14c53002..6a7c12a8d3b 100644 --- a/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs +++ b/packages/paperclip-runner/runner/crates/runner-core/src/provider_backend.rs @@ -767,15 +767,26 @@ fn terminal_events( let Some(contract) = state.completion_contract.as_ref() else { return Vec::new(); }; - if rejected_completion { + let empty_opencode_completion = state.config.provider == "opencode" + && event_type == "turn.completed" + && state.active_provider_result_fingerprint.is_none() + && state + .last_agent_message + .as_deref() + .is_none_or(|text| text.trim().is_empty()); + if rejected_completion || empty_opencode_completion { return vec![ NormalizedProviderEvent { event_type: "session.failed".to_owned(), priority: EventPriority::P0, payload: json!({ "provider": state.config.provider, - "code": "semantic_completion_rejected", - "message": "The turn ended without correcting its rejected completion report.", + "code": if rejected_completion { "semantic_completion_rejected" } else { "opencode_empty_response" }, + "message": if rejected_completion { + "The turn ended without correcting its rejected completion report." + } else { + "OpenCode ended without a final answer or an accepted completion report." + }, "recoverable": false, }), }, @@ -5733,6 +5744,7 @@ mod tests { #[test] fn an_undeclared_finish_rejection_does_not_change_terminal_authority() { let mut state = opencode_result_state(); + state.last_agent_message = Some("The requested answer is ready.".to_owned()); state .admit_tool_call("undeclared-finish", "paperclip_finish", &json!({})) .unwrap_err(); @@ -6023,14 +6035,11 @@ mod tests { }), ProviderToolBridge::default(), ); - state.last_agent_message = None; + state.last_agent_message = Some("The actual final answer.".to_owned()); let events = terminal_events(&state, "turn.completed", None); - assert_eq!( - events[0].payload["summary"], - "OpenCode completed the requested work." - ); + assert_eq!(events[0].payload["summary"], "The actual final answer."); assert_eq!( events[0].payload["evidence"][0]["ref"], "provider:opencode:agent-message" @@ -6039,6 +6048,21 @@ mod tests { assert!(!events[0].payload.to_string().contains("Codex")); } + #[test] + fn opencode_empty_terminal_does_not_fabricate_a_successful_answer() { + for message in [None, Some(" \n".to_owned())] { + let mut state = opencode_result_state(); + state.last_agent_message = message; + let events = terminal_events(&state, "turn.completed", None); + assert!(!events + .iter() + .any(|event| event.event_type == "run.result.proposed")); + assert_eq!(events[0].event_type, "session.failed"); + assert_eq!(events[0].payload["code"], "opencode_empty_response"); + assert_eq!(events[1].payload["runTerminalState"], "failed"); + } + } + #[test] fn goal_timestamps_normalize_seconds_milliseconds_and_iso() { for value in [ diff --git a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts new file mode 100644 index 00000000000..cf4c0f9337b --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts @@ -0,0 +1,64 @@ +import { chmod, copyFile, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { expect, it } from "vitest"; +import { OpenCodeServerDriver } from "./opencode-server-driver.js"; +import type { PrpEvent } from "../../protocol/replay-contract.js"; + +it.each(["reply", "empty-again", "request-fails", "normal", "foreign-idle"] as const)("handles an empty OpenCode response: %s", async mode => { + const root = await mkdtemp(join(tmpdir(), "opencode-empty-response-")); + const command = join(root, "fake-opencode.mjs"); + await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command); + await chmod(command, 0o755); + const prompts: Record[] = []; + let stream: ReadableStreamDefaultController | undefined; + const queued: Uint8Array[] = []; + const emit = (value: unknown) => { const frame = new TextEncoder().encode(`data: ${JSON.stringify(value)}\n\n`); if (stream) stream.enqueue(frame); else queued.push(frame); }; + const driver = new OpenCodeServerDriver({ + model: "openrouter/controlled-model", runtimeDirectory: join(root, "runtime"), command, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "controlled-fixture-key" }, + fetch: async (input, init) => { + const url = new URL(String(input)); + if (url.pathname === "/event") return new Response(new ReadableStream({ start(controller) { stream = controller; for (const frame of queued.splice(0)) controller.enqueue(frame); } }), { headers: { "Content-Type": "text/event-stream" } }); + if (url.pathname.endsWith("/prompt_async")) { + const prompt = JSON.parse(String(init?.body)); prompts.push(prompt); + if (prompts.length === 2 && mode === "request-fails") throw new Error("controlled correction request failed"); + const sessionID = decodeURIComponent(url.pathname.split("/")[2]!); + const number = prompts.length, messageID = `assistant-${number}`; + queueMicrotask(() => { + if (mode === "foreign-idle") emit({ type: "session.idle", properties: { sessionID: "foreign-session" } }); + emit({ type: "message.updated", properties: { info: { id: messageID, sessionID, role: "assistant", finish: "stop", time: { completed: Date.now() } } } }); + const answer = mode === "normal" || mode === "foreign-idle" || number === 2 && mode === "reply"; + emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID, + type: answer ? "text" : "reasoning", text: answer ? "Here is the actual final answer." : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); + }); + return new Response(null, { status: 204 }); + } + return fetch(input, init); + }, + }); + const session = await driver.openSession({ runId: "owned-run", normalizedSessionId: "owned-session", workingDirectory: root }); + try { + const collected = (async () => { const rows: PrpEvent[] = []; for await (const event of session.events()) { + rows.push(event); if (["turn.completed", "turn.failed"].includes(event.eventType)) break; + } return rows; })(); + const { turnId } = await session.startTurn({ message: { role: "user", text: "Answer the current request." }, reasoningMode: "disabled" }); + const events = await collected; + const succeeds = ["reply", "normal", "foreign-idle"].includes(mode); + expect(events.at(-1)?.eventType).toBe(succeeds ? "turn.completed" : "turn.failed"); + expect(events.filter(event => event.eventType === "turn.started")).toHaveLength(1); + expect(events.every(event => !event.turnId || event.turnId === turnId)).toBe(true); + const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final"); + expect(final.map(event => event.payload.text)).toEqual(succeeds ? ["Here is the actual final answer."] : []); + expect(prompts).toHaveLength(["normal", "foreign-idle"].includes(mode) ? 1 : 2); + if (prompts.length === 2) { + expect(prompts[1]).toMatchObject({ providerID: prompts[0]!.providerID, modelID: prompts[0]!.modelID, system: prompts[0]!.system, variant: "paperclip-no-reasoning" }); + expect(JSON.stringify(prompts[1]!.parts)).toContain("Do not repeat completed actions"); + } + } finally { + await session.close({ reason: "controlled test complete" }); + await rm(root, { recursive: true, force: true }); + } +}, 30_000); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 58af69bad4b..1824f4e4bb2 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -2750,6 +2750,10 @@ describe("OpenCodeServerDriver", () => { id: "prt_oversized", messageID: "msg_oversized", sessionID, type: "tool", callID: "call_oversized", tool: "bash", state: { status: "completed", input: { command: "pnpm test" }, output, title: "pnpm test", metadata: { exit: 1, output }, time: { start: 1, end: 2 } }, } } }); + send({ type: "message.part.updated", properties: { part: { + id: "prt_final", messageID: "msg_oversized", sessionID, type: "text", + text: "The suite completed: one test failed; 812 passed.", time: { start: 3, end: 4 }, + } } }); send({ type: "session.idle", properties: { sessionID } }); const observed = await events; expect(observed.at(-1)?.eventType).toBe("turn.completed"); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 6576fe894e7..e7e83d43758 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -540,6 +540,10 @@ class OpenCodeHarnessSession implements HarnessSession { readonly #companyObservability: ReturnType; #closed = false; #completionSettlement: Promise | null = null; + #turnStartSourceSeq = 0; + #turnReasoningMode: OpenCodeReasoningMode = "default"; + #emptyResponseRecovery: { turnId: string; priorMessages: Set; responseFinished: boolean } | null = null; + #emptyResponseAckTimer: ReturnType | undefined; #abort = new AbortController(); constructor(input: { @@ -686,14 +690,15 @@ class OpenCodeHarnessSession implements HarnessSession { const turnId = `turn-${randomBytes(12).toString("hex")}`; this.#streamingParts.clear(); this.#activeTurnId = turnId; + this.#turnStartSourceSeq = this.#sourceSequence; + this.#turnReasoningMode = reasoningMode; + this.#emptyResponseRecovery = null; this.#emit("turn.submitted", { envelopeSchema: this.#taskEnvelope.schema, text: input.message.text, }, { turnId }); this.#emit("turn.accepted", { turnId }, { turnId }); this.#emit("turn.started", { status: "inProgress" }, { turnId }); - const [providerID, ...modelParts] = this.#model.split("/"); - const modelID = modelParts.join("/"); // Keep the task envelope only on the initial task-mode wake. OpenCode // rebuilds system instructions from the latest user message, so system // instructions must still accompany every prompt (including recovery). @@ -703,12 +708,21 @@ class OpenCodeHarnessSession implements HarnessSession { message: input.message.text, }) : input.message.text; + await this.#submitPrompt(prompt); + this.#sendFullContext = false; + return { turnId }; + } + + async #submitPrompt(prompt: string): Promise { + const [providerID, ...modelParts] = this.#model.split("/"); + const modelID = modelParts.join("/"); await api( this.#fetch, this.#runtime, `/session/${encodeURIComponent(this.#providerSessionId)}/prompt_async`, { method: "POST", + signal: this.#abort.signal, body: JSON.stringify({ // OpenCode 1.18's PromptPayload carries the provider and model at the // top level. Older examples used a nested `model` object; 1.18 @@ -721,7 +735,7 @@ class OpenCodeHarnessSession implements HarnessSession { // Always select the empty default variant on an ordinary turn so a // previous disabled turn cannot change this turn's provider defaults. ...(providerID === "openrouter" - ? { variant: reasoningMode === "disabled" ? "paperclip-no-reasoning" : "paperclip-default" } + ? { variant: this.#turnReasoningMode === "disabled" ? "paperclip-no-reasoning" : "paperclip-default" } : {}), // Keep question enabled in the isolated config. OpenCode 1.18.32 // turns a prompt's deprecated `tools` map into replacement session @@ -731,8 +745,39 @@ class OpenCodeHarnessSession implements HarnessSession { }), }, ); - this.#sendFullContext = false; - return { turnId }; + } + + #clearEmptyResponseAckTimer(): void { + clearTimeout(this.#emptyResponseAckTimer); + this.#emptyResponseAckTimer = undefined; + } + + #failEmptyResponse(turnId: string, message: string): void { + if (this.#closed || this.#activeTurnId !== turnId || this.#result !== null) return; + this.#emit("turn.failed", { status: "failed", error: { code: "opencode_empty_response", message, retryable: false } }, { turnId }); + this.#terminalTurns.set(turnId, canonicalJson({ status: "failed" })); + this.#activeTurnId = null; + } + + #recoverEmptyResponse(turnId: string): boolean { + if (this.#result !== null || this.#pendingRuntimeRequests.size > 0) return false; + const recovery = this.#emptyResponseRecovery; + // Both idle event forms can remain queued from the empty response. Wait + // for the correction's own completed assistant message before settling. + if (recovery && !recovery.responseFinished) return true; + const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq); + if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary)) return false; + if (recovery) { + this.#failEmptyResponse(turnId, "OpenCode ended without a final answer or accepted completion report after one correction."); + return true; + } + this.#emptyResponseRecovery = { turnId, priorMessages: new Set(this.#messageRoles.keys()), responseFinished: false }; + this.#emit("harness.diagnostic", { code: "opencode_empty_response_correction", message: "Requesting one final response after an empty OpenCode completion." }, { turnId }); + this.#emptyResponseAckTimer = setTimeout(() => this.#failEmptyResponse(turnId, "OpenCode did not start its requested final response."), 20_000); + this.#emptyResponseAckTimer.unref(); + void this.#submitPrompt("Your response ended without a user-visible final answer or an accepted completion report. Continue this same request from the work already done. Give the user a concise final answer and submit paperclip_finish (or paperclip_block when blocked) with the current completion contract. Do not repeat completed actions.") + .catch(() => this.#failEmptyResponse(turnId, "OpenCode could not request a final response after an empty completion.")); + return true; } async interrupt(input: { turnId?: string; reason?: string }): Promise { @@ -992,6 +1037,7 @@ class OpenCodeHarnessSession implements HarnessSession { await this.#completionSettlement; if (this.#closed) return; this.#closed = true; + this.#clearEmptyResponseAckTimer(); this.#abort.abort(); // Settle whatever is still pending, including a request whose turn // already went terminal without the driver observing it (see @@ -1583,7 +1629,8 @@ class OpenCodeHarnessSession implements HarnessSession { // adjacent tokens. Content deduplication would silently lose the second. // Key id-less events by digest: a retained canonical copy of each recent // event would hold up to 10,000 large tool payloads in memory. - const eventId = text(event.id) || (type === "message.part.delta" + const idle = type === "session.idle" || type === "session.status" && text(record(properties.status).type) === "idle"; + const eventId = idle ? null : text(event.id) || (type === "message.part.delta" ? null : `sha256:${createHash("sha256").update(canonicalJson(value)).digest("hex")}`); if (eventId !== null && this.#seenProviderEvents.has(eventId)) return; @@ -1722,6 +1769,12 @@ class OpenCodeHarnessSession implements HarnessSession { text(info.messageID, text(info.messageId)), ); const role = text(info.role); + const recovery = this.#emptyResponseRecovery; + if (recovery?.turnId === turnId && role === "assistant" && messageId && !recovery.priorMessages.has(messageId)) { + this.#clearEmptyResponseAckTimer(); + if ((Number.isFinite(record(info.time).completed) || text(info.finish)) && !["tool-calls", "tool_calls"].includes(text(info.finish))) + recovery.responseFinished = true; + } // Record the message's owning turn at the moment OpenCode first // reports it. A later turn that reuses the same native message id // legitimately reclaims ownership; a stale message never sees this @@ -1775,6 +1828,7 @@ class OpenCodeHarnessSession implements HarnessSession { text(record(properties.status).type) === "idle")) && turnId ) { + if (this.#recoverEmptyResponse(turnId)) return; const fingerprint = canonicalJson({ status: "completed", semanticResult: this.#resultFingerprint, @@ -2147,7 +2201,11 @@ class OpenCodeHarnessSession implements HarnessSession { }); return; } + if (eventType === "run.result.proposed" || eventType === "runtime_request.created") { + this.#clearEmptyResponseAckTimer(); + } if (["turn.completed", "turn.failed", "turn.cancelled", "turn.interrupted"].includes(eventType)) { + this.#clearEmptyResponseAckTimer(); const performance = this.#runtime.performanceDiagnostics?.take(this.#performanceBoundaryMs); this.#performanceBoundaryMs = this.#now().getTime(); if (performance) this.#emit("harness.diagnostic", performance, refs);