diff --git a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts index 4edda24e38e..b9fb24ff1ac 100644 --- a/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/empty-response.test.ts @@ -5,12 +5,16 @@ import { expect, it, vi } from "vitest"; import { OpenCodeServerDriver } from "./opencode-server-driver.js"; import type { PrpEvent } from "../../protocol/replay-contract.js"; -it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool"] as const)("handles an empty OpenCode response: %s", async mode => { +it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "idle-stall", "tool-idle-stall", "empty-again", "request-fails", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches"] as const)("handles an empty OpenCode response: %s", async mode => { const root = await mkdtemp(join(tmpdir(), "opencode-empty-response-")); const command = join(root, "fake-opencode.mjs"); await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), command); await chmod(command, 0o755); const prompts: Record[] = []; + // Recorded from chat-eval cohort 5 (tasks.website-task, DeepSeek v4.1 flash + // through OpenRouter): the step's reasoning, then the same text again as an + // ordinary text part, and no completion call. + const recordedReasoning = "Task created and already executing (status in_progress, run dafb...). Good. Reply with link. Use finish. Keep concise.\n\nCompletion contract revision 1, criterion \"objective\". I should call paperclip_finish with done. Include evidence refs maybe task link. Let's do."; let toolTimer: ReturnType | undefined; let stream: ReadableStreamDefaultController | undefined; let streamOpened!: () => void; @@ -51,9 +55,37 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow return; } } - const answer = mode === "normal" || mode === "foreign-idle" || mode === "queued-tool" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle"].includes(mode); + if (mode === "reasoning-echo-reasoning-after" && number === 1) { + emit({ type: "message.part.updated", properties: { sessionID, part: { id: "part-1-text", messageID, sessionID, type: "text", text: recordedReasoning, time: { start: 1, end: 2 } } } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { id: "part-1-reasoning", messageID, sessionID, type: "reasoning", text: recordedReasoning, time: { start: 1, end: 3 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); + return; + } + if (mode === "reasoning-echo-text-first" && number === 1) { + const reasoningPart = { id: "part-1-reasoning", messageID, sessionID, type: "reasoning", text: recordedReasoning, time: { start: 1 } }; + emit({ type: "message.part.updated", properties: { sessionID, part: reasoningPart } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { id: "part-1-text", messageID, sessionID, type: "text", text: recordedReasoning, time: { start: 3, end: 4 } } } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { ...reasoningPart, time: { start: 1, end: 5 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); + return; + } + if ((mode === "reasoning-echo" || mode === "reasoning-echo-correction-matches") && number === 1) { + const reasoningPart = { id: "part-1-reasoning", messageID, sessionID, type: "reasoning", text: recordedReasoning, time: { start: 1, end: 2 } }; + const echo = { id: "part-1-text", messageID, sessionID, type: "text", text: "", time: { start: 3 } }; + emit({ type: "message.part.updated", properties: { sessionID, part: reasoningPart } }); + emit({ type: "message.part.updated", properties: { sessionID, part: echo } }); + emit({ type: "message.part.delta", properties: { sessionID, messageID, partID: echo.id, field: "text", delta: recordedReasoning.slice(0, 40) } }); + emit({ type: "message.part.delta", properties: { sessionID, messageID, partID: echo.id, field: "text", delta: `${recordedReasoning.slice(40)}\n` } }); + emit({ type: "message.part.updated", properties: { sessionID, part: { ...echo, text: `${recordedReasoning}\n`, time: { start: 3, end: 4 } } } }); + emit({ type: "session.idle", properties: { sessionID } }); + emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); + return; + } + const answer = mode === "normal" || mode === "foreign-idle" || mode === "queued-tool" || number === 2 && ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches"].includes(mode); emit({ type: "message.part.updated", properties: { sessionID, part: { id: `part-${number}`, messageID, - type: answer ? "text" : "reasoning", text: answer ? "Here is the actual final answer." : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } }); + type: answer ? "text" : "reasoning", text: answer ? (mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer.") : "Synthetic reasoning must not become the answer.", time: { start: 1, end: 2 } } } }); emit({ type: "session.idle", properties: { sessionID } }); emit({ type: "session.status", properties: { sessionID, status: { type: "idle" } } }); }); @@ -82,7 +114,7 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow return result; }; const collected = collect(); - const { turnId } = await session.startTurn({ message: { role: "user", text: "Answer the current request." }, reasoningMode: mode === "empty-again" || mode === "reply" ? "default" : "disabled" }); + const { turnId } = await session.startTurn({ message: { role: "user", text: "Answer the current request." }, reasoningMode: mode === "empty-again" || mode === "reply" || mode.startsWith("reasoning-echo") ? "default" : "disabled" }); const completed = guard(collected, mode.endsWith("idle-stall") || mode === "slow-native-tool" ? 26_000 : 5_000); await vi.advanceTimersByTimeAsync(0); if (mode === "slow-native-tool") { @@ -91,18 +123,24 @@ it.each(["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow } await vi.advanceTimersByTimeAsync(mode.endsWith("idle-stall") ? 20_001 : mode === "slow-native-tool" ? 1_999 : 5_001); const events = await completed; - const succeeds = ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool"].includes(mode); + const succeeds = ["reply", "reply-no-finish", "reply-tool-finish", "two-turn-idle", "slow-native-tool", "normal", "foreign-idle", "queued-tool", "reasoning-echo", "reasoning-echo-text-first", "reasoning-echo-reasoning-after", "reasoning-echo-correction-matches"].includes(mode); expect(events.at(-1)?.eventType).toBe(succeeds ? "turn.completed" : "turn.failed"); expect(events.filter(event => event.eventType === "turn.started")).toHaveLength(1); expect(events.every(event => !event.turnId || event.turnId === turnId)).toBe(true); const final = events.filter(event => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final"); - expect(final.map(event => event.payload.text)).toEqual(succeeds ? ["Here is the actual final answer."] : []); + expect(final.map(event => event.payload.text)).toEqual(succeeds ? [mode === "reasoning-echo-correction-matches" ? recordedReasoning : "Here is the actual final answer."] : []); expect(prompts).toHaveLength(["normal", "foreign-idle", "queued-tool"].includes(mode) ? 1 : 2); if (prompts.length === 2) { - expect(prompts[0]!.variant).toBe(mode === "empty-again" || mode === "reply" ? "paperclip-default" : "paperclip-no-reasoning"); + expect(prompts[0]!.variant).toBe(mode === "empty-again" || mode === "reply" || mode.startsWith("reasoning-echo") ? "paperclip-default" : "paperclip-no-reasoning"); expect(prompts[1]).toMatchObject({ providerID: prompts[0]!.providerID, modelID: prompts[0]!.modelID, system: prompts[0]!.system, variant: "paperclip-no-reasoning" }); expect(JSON.stringify(prompts[1]!.parts)).toContain("Do not repeat completed actions"); } + if (mode === "reasoning-echo" || mode === "reasoning-echo-text-first") { + // The repeated reasoning never reaches the person, as progress or as the reply. + const assistantText = events.filter(event => event.payload.kind === "agentMessage").map(event => String(event.payload.text ?? "")); + expect(assistantText.join("")).not.toContain(recordedReasoning.slice(0, 20)); + expect(events.some(event => event.eventType === "harness.diagnostic" && event.payload.code === "opencode_reasoning_echo_discarded")).toBe(true); + } if (mode === "two-turn-idle") { const nextEvents = collect(); const next = await session.startTurn({ message: { role: "user", text: "Answer the next request." }, reasoningMode: "disabled" }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index f04ca959a30..b2f40a73a5d 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -3062,6 +3062,89 @@ describe("OpenCodeServerDriver", () => { await session.close({ reason: "test" }); }); + it("keeps a short answer written before the completion call when a later message only refers back to it", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + }); + const session = await driver.openSession({ + runId: "run-short-answer-then-reference", + normalizedSessionId: "short-answer-then-reference", + workingDirectory: workspace, + }); + await session.startTurn({ message: { role: "user", text: "short-answer-before-finish-then-reference" } }); + const events = await collectTurnEvents(session.events()); + const finalMessages = events.filter( + (event) => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final", + ); + expect(finalMessages).toHaveLength(1); + expect(finalMessages[0]?.payload.text).toBe("The result is 42.\n\nAnswered above."); + await session.close({ reason: "test" }); + }); + + it("keeps a full answer written before the completion call when the later message is only \"Done.\"", async () => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + }); + const session = await driver.openSession({ + runId: "run-long-answer-then-done", + normalizedSessionId: "long-answer-then-done", + workingDirectory: workspace, + }); + await session.startTurn({ message: { role: "user", text: "long-answer-before-finish-then-done" } }); + const events = await collectTurnEvents(session.events()); + const finalMessages = events.filter( + (event) => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final", + ); + expect(finalMessages).toHaveLength(1); + expect(finalMessages[0]?.payload.text).toBe( + "Morgan's run on [COM-1](/COM/issues/COM-1) finished: the landing page is saved as `index.html` on the task, with the hero, features and signup sections from the brief.\n\nDone.", + ); + await session.close({ reason: "test" }); + }); + + it.each([ + ["short-status-before-finish-then-short-answer", "Updated the Launch blockers status card."], + ["short-answer-repeated-after-finish", "The result is 42."], + ])("saves only a short complete answer written after the completion call: %s", async (prompt, expected) => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + }); + const session = await driver.openSession({ + runId: `run-${prompt}`, + normalizedSessionId: prompt, + workingDirectory: workspace, + }); + await session.startTurn({ message: { role: "user", text: prompt } }); + const events = await collectTurnEvents(session.events()); + const finalMessages = events.filter( + (event) => event.eventType === "item.completed" && event.payload.kind === "agentMessage" && event.payload.channel === "final", + ); + expect(finalMessages).toHaveLength(1); + expect(finalMessages[0]?.payload.text).toBe(expected); + await session.close({ reason: "test" }); + }); + it("saves an answer once when the model restates it after the completion call", async () => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); @@ -3101,6 +3184,40 @@ describe("OpenCodeServerDriver", () => { await session.close({ reason: "test" }); }); + it.each([ + ["answer-before-finish-then-rewrite", "Work on [COM-1 Website task](/COM/issues/COM-1) stopped at 06:57 UTC, about 90 seconds after starting: its run's local runner process exited unexpectedly (`runnerd exited unexpectedly`), all 3 automatic recovery attempts were exhausted, and Paperclip marked the task **blocked** with cause `native_session_retry_exhausted`. The human-supplied website brief had also not arrived, so no brief was saved.\n\n[The assigned agent, Morgan,](https://example.invalid) is in an error state. To resume, open COM-1 and use **Retry** on that task; it rechecks gates and preserves the failed attempt's evidence. Agents cannot retry it for you."], + ["status-before-finish-then-answer", "Updated the [Launch blockers status card](/COM/status/status-card-launch) with current Synthetic Launch Notes verification.\n\n- **COM-1 \u2014 Mobile Safari checkout regression blocks launch** \u2014 `blocked`\n- Payment approval is now recorded as received; the mobile Safari checkout regression remains the blocker.\n- Only COM-1 appears in the bounded changed-issue set; no other Launch blockers were added."], + ])("saves only the later message when %s writes its own answer after the completion call", async (prompt, expected) => { + await chmod(fixture, 0o755); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-driver-")); + const workspace = await mkdtemp(join(tmpdir(), "paperclip-opencode-workspace-")); + roots.push(root, workspace); + const driver = new OpenCodeServerDriver({ + model: "openrouter/deepseek/deepseek-v4-flash-0731", + runtimeDirectory: root, + command: fixture, + environment: { PATH: process.env.PATH, OPENROUTER_API_KEY: "fixture-key" }, + }); + const session = await driver.openSession({ + runId: `run-${prompt}`, + normalizedSessionId: prompt, + workingDirectory: workspace, + }); + await session.startTurn({ message: { role: "user", text: prompt } }); + const events = await collectTurnEvents(session.events()); + const finalMessages = events.filter( + (event) => + event.eventType === "item.completed" && + event.payload.kind === "agentMessage" && + event.payload.channel === "final", + ); + // The completion tool names the later message as the final response; a + // planning note or an earlier copy of the answer is not saved with it. + expect(finalMessages.map((event) => event.payload.text)).toEqual([expected]); + expect(finalMessages[0]?.itemId).toBe("part-reference"); + await session.close({ reason: "test" }); + }); + it.each(["session.idle", "session.status", "session.error", "aborted"])("settles accepted completion before a racing %s event", async terminalType => { await chmod(fixture, 0o755); const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-completion-race-")); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 4ccf82c8df1..1667f037b01 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -23,7 +23,6 @@ import { getCACertificates } from "node:tls"; import { pipeline } from "node:stream/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; -import { restatesEarlierReply } from "./restated-reply.js"; import { parseOpenCodeReasoningMode, type OpenCodeReasoningMode } from "./reasoning-mode.js"; import { attributeUnsupportedToolMedia } from "./tool-media-attribution.js"; import { openCodeProviderFailure } from "./provider-failure.js"; @@ -526,6 +525,10 @@ class OpenCodeHarnessSession implements HarnessSession { readonly #streamingParts = new Map>(); readonly #completedTextPartIds = new Set(); readonly #completedReasoningPartIds = new Set(); + /** Latest trimmed text of each reasoning part in the current turn, recorded while it streams. */ + readonly #reasoningTexts = new Map(); + /** Completed text parts rejected as reasoning copies; they stay rejected after the reasoning is cleared. */ + readonly #reasoningEchoPartIds = new Set(); readonly #completedTextParts: Array<{ partId: string; messageId: string | null; @@ -698,6 +701,8 @@ class OpenCodeHarnessSession implements HarnessSession { this.#lastNonTerminalToolSourceSeq = 0; this.#completedTextPartIds.clear(); this.#completedReasoningPartIds.clear(); + this.#reasoningTexts.clear(); + this.#reasoningEchoPartIds.clear(); this.#completedTextParts.length = 0; this.#outputLimitedMessageIds.clear(); // `#terminalTurns` clears here for its own persisted-snapshot bookkeeping. @@ -822,6 +827,22 @@ class OpenCodeHarnessSession implements HarnessSession { this.#activeTurnId = null; } + /** + * Whether assistant text repeats a reasoning part: the whole part once the + * text is complete, or the start of one while it still streams. + */ + #echoesReasoning(content: string, completed: boolean): boolean { + const value = content.trim(); + if (!value) return false; + for (const reasoning of this.#reasoningTexts.values()) + if (completed ? reasoning === value : reasoning.startsWith(value)) return true; + return false; + } + + #isReasoningEcho(part: { partId: string; text: string }): boolean { + return this.#reasoningEchoPartIds.has(part.partId) || this.#echoesReasoning(part.text, true); + } + /** A response cut off at the output limit is incomplete, so it is never a final answer. */ #isOutputLimited(part: { messageId: string | null }): boolean { return part.messageId !== null && this.#outputLimitedMessageIds.has(part.messageId); @@ -841,7 +862,7 @@ class OpenCodeHarnessSession implements HarnessSession { if (this.#interruptedTurnId === turnId) return false; const recovery = this.#emptyResponseRecovery; const boundary = Math.max(this.#turnStartSourceSeq, this.#lastNonTerminalToolSourceSeq); - if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary && !this.#isOutputLimited(part))) return false; + if (this.#completedTextParts.some(part => part.observedSourceSeq > boundary && !this.#isOutputLimited(part) && !this.#isReasoningEcho(part))) return false; // Both idle event forms can remain queued from the empty response. Wait // for correction evidence. A completed answer part is sufficient even // when its message header omits completion metadata. An idle response @@ -857,6 +878,9 @@ class OpenCodeHarnessSession implements HarnessSession { return true; } this.#emptyResponseRecovery = { turnId, priorMessages: new Set(this.#messageRoles.keys()), responseStarted: false, responseFinished: false }; + for (const part of this.#completedTextParts) + if (this.#echoesReasoning(part.text, true)) this.#reasoningEchoPartIds.add(part.partId); + this.#reasoningTexts.clear(); this.#emit("harness.diagnostic", { code: "opencode_empty_response_correction", message: "Requesting one final response after an empty OpenCode completion." }, { turnId }); this.#armEmptyResponseAckTimer(turnId, "OpenCode did not start its requested final response."); const opening = this.#turnHitOutputLimit(turnId) @@ -2132,8 +2156,17 @@ class OpenCodeHarnessSession implements HarnessSession { } const content = text(part.text, text(part.output)); const previous = this.#partText.get(partId) ?? ""; - this.#partText.set(partId, content); - if (partType === "text" && content.length > 0) { + const completedAt = record(part.time).end; + const completed = Number.isFinite(completedAt); + // Some OpenRouter models (DeepSeek v4.1 flash, for one) end a step by + // repeating their reasoning as an ordinary text part. That copy is + // planning, not prose for the person, so it never streams as progress + // and never becomes the reply. Hold a text part while it still matches + // the start of a completed reasoning part, and release it once it differs. + if (partType === "reasoning" && content.trim()) this.#reasoningTexts.set(partId, content.trim()); + const reasoningEcho = partType === "text" && this.#echoesReasoning(content, completed); + if (!reasoningEcho) this.#partText.set(partId, content); + if (partType === "text" && content.length > 0 && !reasoningEcho) { for (const reference of paperclipWorkspaceFileReferencesFromText( this.#workingDirectory, content, @@ -2151,7 +2184,7 @@ class OpenCodeHarnessSession implements HarnessSession { const delta = content.startsWith(previous) ? content.slice(previous.length) : content; - if (delta) { + if (delta && !reasoningEcho) { // OpenCode calls assistant prose a `text` part, but `text` is not a PRP // item identity and is consequently ignored by the task projection. // Until settlement selects one completed part as the final response, @@ -2195,11 +2228,10 @@ class OpenCodeHarnessSession implements HarnessSession { { turnId, itemId: partId }, ); } - const completedAt = record(part.time).end; if ( partType === "reasoning" && content.trim().length > 0 && - Number.isFinite(completedAt) && + completed && !this.#completedReasoningPartIds.has(partId) ) { this.#completedReasoningPartIds.add(partId); @@ -2224,10 +2256,20 @@ class OpenCodeHarnessSession implements HarnessSession { if ( partType === "text" && content.trim().length > 0 && - Number.isFinite(completedAt) && + completed && !this.#completedTextPartIds.has(partId) ) { this.#completedTextPartIds.add(partId); + if (reasoningEcho) { + // Without this copy the turn may hold no answer; the empty-response + // correction then asks the model for one with reasoning disabled. + this.#emit( + "harness.diagnostic", + { code: "opencode_reasoning_echo_discarded", message: "OpenCode repeated a reasoning part as assistant text; the copy was not used as a response." }, + { turnId }, + ); + return; + } this.#completedTextParts.push({ partId, messageId, @@ -2254,7 +2296,7 @@ class OpenCodeHarnessSession implements HarnessSession { // and let the accepted semantic summary resolve the durable reply. const afterLastWorkTool = (part: { observedSourceSeq: number }) => part.observedSourceSeq > this.#lastNonTerminalToolSourceSeq; - const eligibleText = this.#completedTextParts.filter(part => afterLastWorkTool(part) && !this.#isOutputLimited(part)); + const eligibleText = this.#completedTextParts.filter(part => afterLastWorkTool(part) && !this.#isOutputLimited(part) && !this.#isReasoningEcho(part)); // OpenCode exposes no prose channel, so final selection must use provider // structure rather than text length or content. When OpenCode reports the // terminal tool part, prefer prose completed afterward in that exact @@ -2297,22 +2339,27 @@ class OpenCodeHarnessSession implements HarnessSession { afterResult[0]?.part ?? null; if (!selected) return; - // A later message after the terminal call is only the closing part of the - // reply when the model already wrote prose after its last work tool and - // before that call: the person saw both, and the later message often just - // refers back ("Answered above"). Structure cannot tell an answer from - // commentary, so keep every eligible part rather than drop the answer. + // The completion tool asks for the final response after the call, so a + // later message is the reply. The prose written between the last work tool + // and that call stays in the reply only when the later message is a short + // closing line that leans on it ("Answered above", "Commands above are + // ready"): a short line with no link of its own that points back to it, or + // one under half that prose's length. A later message that repeats the + // prose, or any other later message, is the model's own answer: keeping the + // earlier prose too would save a planning note or a second copy. const candidatePreface = this.#semanticResultProviderMessageId !== null && correlatedAfterResult.length === 0 && uncorrelatedAfterResult[0]?.part === selected ? beforeResult.map(({ part }) => part) : []; - const preface = - candidatePreface.length && - restatesEarlierReply(candidatePreface.map((part) => part.text.trim()).join("\n\n"), selected.text) - ? [] - : candidatePreface; + const prefaceText = candidatePreface.map((part) => part.text.trim()).join("\n\n"); + const selectedText = selected.text.trim(); + const normalized = (value: string) => value.replace(/\s+/g, " ").trim().toLowerCase(); + const repeatsPreface = normalized(selectedText) === normalized(prefaceText); + const pointsBack = selectedText.length <= 120 && !/\]\(|https?:\/\//.test(selectedText) + && /\b(above|earlier|previous|preceding)\b/i.test(selectedText); + const preface = !repeatsPreface && (pointsBack || selectedText.length * 2 < prefaceText.length) ? candidatePreface : []; const text = preface.length ? [...preface, selected].map((part) => part.text.trim()).filter(Boolean).join("\n\n") : selected.text; diff --git a/packages/paperclip-runner/src/drivers/opencode/restated-reply.test.ts b/packages/paperclip-runner/src/drivers/opencode/restated-reply.test.ts deleted file mode 100644 index 116cda28980..00000000000 --- a/packages/paperclip-runner/src/drivers/opencode/restated-reply.test.ts +++ /dev/null @@ -1,100 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { restatesEarlierReply } from "./restated-reply.js"; - -const recordedBefore = - "I can't set this from chat — changing an agent's monthly budget is a board-only action (agent credentials get 403). There are also two matching agents, so pick the intended one in the web app and set its monthly limit to $50 on the budget tab: [Morgan budgets](/COM/agents/a1/budgets) or [Morgan Research budgets](/COM/agents/a2/budgets). Tell me which Morgan you meant and I'll track the right one."; -const recordedAfter = - "I can't change agent budgets from chat — that's board-only, and agent credentials get 403. Two agents also match \"Morgan,\" so choose the intended one and set $50/month on its budget tab: [Morgan budgets](/COM/agents/a1/budgets) or [Morgan Research budgets](/COM/agents/a2/budgets)."; - -describe("restatesEarlierReply", () => { - it("treats an identical later message as a restatement", () => { - expect(restatesEarlierReply(recordedBefore, `${recordedBefore}\n`)).toBe(true); - }); - - it("treats the recorded reworded answer as a restatement", () => { - expect(restatesEarlierReply(recordedBefore, recordedAfter)).toBe(true); - }); - - it("keeps an answer whose later message only refers back to it", () => { - expect( - restatesEarlierReply( - "Two inbox items need you: [COM-2](/COM/issues/COM-2) is blocked and [COM-1](/COM/issues/COM-1) is waiting for review.", - "Answered above: two inbox items need your attention.", - ), - ).toBe(false); - }); - - it("keeps an earlier link the later message drops", () => { - expect( - restatesEarlierReply( - recordedBefore, - recordedAfter.replace(" or [Morgan Research budgets](/COM/agents/a2/budgets)", " or Morgan Research's budget tab"), - ), - ).toBe(false); - }); - - it("keeps an earlier code block the later message drops", () => { - expect( - restatesEarlierReply( - "Run this in your terminal:\n\n```shell\nnpx paperclipai auth login\n```\n\nThen confirm with whoami.", - "Run the sign-in command in your terminal, then confirm with whoami before any write.", - ), - ).toBe(false); - }); - - const reportBefore = "The full launch report is ready at https://example.com/reports/launch and covers the signup funnel, the pricing page and the onboarding emails for review."; - const reportAfter = "The launch report is ready and covers the signup funnel, the pricing page and the onboarding emails, so you can review it."; - - it("keeps an earlier bare URL the later message drops", () => { - expect(restatesEarlierReply(reportBefore, reportAfter)).toBe(false); - }); - - it("treats a later message that keeps the bare URL as a restatement", () => { - expect(restatesEarlierReply(reportBefore, `${reportAfter} Link: https://example.com/reports/launch.`)).toBe(true); - }); - - it("keeps an earlier autolink the later message drops", () => { - expect( - restatesEarlierReply(reportBefore.replace("https://example.com/reports/launch", ""), reportAfter), - ).toBe(false); - }); - - it("keeps an earlier reference-style link the later message drops", () => { - expect( - restatesEarlierReply( - "The full launch [report][launch] is ready and covers the signup funnel, the pricing page and the onboarding emails for review.\n\n[launch]: /COM/issues/COM-7#document-report", - reportAfter, - ), - ).toBe(false); - }); - - const codeAnswer = (code: string) => - `Run this script to check the export before you send it to the team:\n\n\`\`\`python\n${code}\n\`\`\`\n\nIt prints the row count and exits.`; - - it("keeps an earlier code block whose string contents differ only in spacing", () => { - expect(restatesEarlierReply(codeAnswer('print("a b")'), codeAnswer('print("a b")'))).toBe(false); - }); - - it("keeps an earlier code block whose indentation differs", () => { - expect( - restatesEarlierReply( - codeAnswer("for row in rows:\n if row:\n print(row)"), - codeAnswer("for row in rows:\n if row:\n print(row)"), - ), - ).toBe(false); - }); - - it("treats a later message with the same code block as a restatement", () => { - const code = "for row in rows:\n print(row)"; - expect(restatesEarlierReply(codeAnswer(code), codeAnswer(code).replace("Run this script", "Use this script"))).toBe(true); - }); - - it("keeps earlier prose that says something different", () => { - expect( - restatesEarlierReply( - "The deploy failed because the database migration timed out after 30 seconds on the orders table.", - "I opened a follow-up task for Morgan to split the migration and will report back once it lands.", - ), - ).toBe(false); - }); -}); diff --git a/packages/paperclip-runner/src/drivers/opencode/restated-reply.ts b/packages/paperclip-runner/src/drivers/opencode/restated-reply.ts deleted file mode 100644 index 8f4225b1466..00000000000 --- a/packages/paperclip-runner/src/drivers/opencode/restated-reply.ts +++ /dev/null @@ -1,43 +0,0 @@ -export function restatesEarlierReply(earlier: string, later: string): boolean { - const before = earlier.trim(); - const after = later.trim(); - if (!before || !after) return false; - if (before === after) return true; - if (after.length * 2 < before.length) return false; - const laterTargets = linkTargets(after); - if (![...linkTargets(before)].every((target) => laterTargets.has(target))) return false; - const laterBlocks = new Set(fencedBlocks(after)); - if (!fencedBlocks(before).every((block) => laterBlocks.has(block))) return false; - const earlierWords = proseWords(before); - if (earlierWords.size === 0) return true; - const laterWords = proseWords(after); - let repeated = 0; - for (const word of earlierWords) if (laterWords.has(word)) repeated += 1; - return repeated * 2 >= earlierWords.size; -} - -const INLINE_LINK_TARGET = /\]\(\s*]+)>?(?:\s+"[^"]*")?\s*\)/g; -const REFERENCE_DEFINITION = /^[ \t]{0,3}\[[^\]\n]+\]:[ \t]*]+)>?/gm; -const AUTOLINK = /<([a-z][a-z\d+.-]*:[^\s<>]+)>/gi; -const BARE_URL = /\b(?:https?|ftp|mailto):[^\s<>()[\]`"']+/gi; -const FENCED_BLOCK = /```[^\n]*\n([\s\S]*?)```/g; - -function linkTargets(text: string): Set { - const targets = new Set(); - for (const pattern of [INLINE_LINK_TARGET, REFERENCE_DEFINITION, AUTOLINK]) { - for (const match of text.matchAll(pattern)) targets.add(match[1]!); - } - for (const match of text.replace(FENCED_BLOCK, " ").matchAll(BARE_URL)) { - targets.add(match[0].replace(/[.,;:!?]+$/, "")); - } - return targets; -} - -function fencedBlocks(text: string): string[] { - return [...text.matchAll(FENCED_BLOCK)].map((match) => match[1]!.replace(/\n$/, "")); -} - -function proseWords(text: string): Set { - const prose = text.replace(FENCED_BLOCK, " ").replace(INLINE_LINK_TARGET, "]"); - return new Set(prose.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []); -} diff --git a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs index 346e305899e..100ac75fca1 100644 --- a/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs +++ b/packages/paperclip-runner/test/fixtures/fake-opencode-server.mjs @@ -803,10 +803,41 @@ const server = createServer(async (request, response) => { const answerBeforeFinishThenRestatement = String( parsedPrompt.message ?? "", ).includes("answer-before-finish-then-restatement"); - const answerBeforeFinishText = answerBeforeFinishThenRestatement + // Recorded in chat-eval cohort 5: a rewritten answer after the call + // (recovery.explain-stall) and a status note before it + // (status-cards.refresh-card). + const answerBeforeFinishThenRewrite = String(parsedPrompt.message ?? "").includes("answer-before-finish-then-rewrite"); + const statusBeforeFinishThenAnswer = String(parsedPrompt.message ?? "").includes("status-before-finish-then-answer"); + const shortAnswerBeforeFinishThenReference = String(parsedPrompt.message ?? "").includes("short-answer-before-finish-then-reference"); + const shortStatusBeforeFinishThenShortAnswer = String(parsedPrompt.message ?? "").includes("short-status-before-finish-then-short-answer"); + const shortAnswerRepeatedAfterFinish = String(parsedPrompt.message ?? "").includes("short-answer-repeated-after-finish"); + const longAnswerBeforeFinishThenDone = String(parsedPrompt.message ?? "").includes("long-answer-before-finish-then-done"); + const answerBeforeFinishText = longAnswerBeforeFinishThenDone + ? "Morgan's run on [COM-1](/COM/issues/COM-1) finished: the landing page is saved as `index.html` on the task, with the hero, features and signup sections from the brief." + : shortAnswerBeforeFinishThenReference || shortAnswerRepeatedAfterFinish + ? "The result is 42." + : shortStatusBeforeFinishThenShortAnswer + ? "STATUS: Status card summary written successfully (revision 2)." + : answerBeforeFinishThenRewrite + ? "Work on [COM-1 Website task](/COM/issues/COM-1) stopped at 06:57 UTC, about 90 seconds after starting: its run's local runner process exited unexpectedly (`runnerd exited unexpectedly`), all 3 automatic recovery attempts were exhausted, and Paperclip marked the task **blocked** with cause `native_session_retry_exhausted`. The human-supplied website brief had also not arrived, so no brief document was saved.\n\nThe assignee, [Morgan the frontend engineer](/COM/agents/agent-morgan), is showing an error status. To resume, open COM-1 and use **Retry** on that task; it will recheck gates and preserve the failed attempt's evidence. Agents cannot retry it for you." + : statusBeforeFinishThenAnswer + ? "STATUS: Status card summary written successfully (revision 2)." + : answerBeforeFinishThenRestatement ? "I can't set this from chat — changing an agent's monthly budget is a board-only action (agent credentials get 403). There are also two matching agents, so pick the intended one in the web app and set its monthly limit to $50 on the budget tab: [Morgan budgets](/COM/agents/agent-morgan/budgets) or [Morgan Research budgets](/COM/agents/agent-morgan-research/budgets). Tell me which Morgan you meant and I'll track the right one." : "Two inbox items need you: [COM-2](/COM/issues/COM-2) is blocked and [COM-1](/COM/issues/COM-1) is waiting for review."; - const answerAfterFinishText = answerBeforeFinishThenRestatement + const answerAfterFinishText = longAnswerBeforeFinishThenDone + ? "Done." + : shortAnswerBeforeFinishThenReference + ? "Answered above." + : shortAnswerRepeatedAfterFinish + ? "The result is 42." + : shortStatusBeforeFinishThenShortAnswer + ? "Updated the Launch blockers status card." + : answerBeforeFinishThenRewrite + ? "Work on [COM-1 Website task](/COM/issues/COM-1) stopped at 06:57 UTC, about 90 seconds after starting: its run's local runner process exited unexpectedly (`runnerd exited unexpectedly`), all 3 automatic recovery attempts were exhausted, and Paperclip marked the task **blocked** with cause `native_session_retry_exhausted`. The human-supplied website brief had also not arrived, so no brief was saved.\n\n[The assigned agent, Morgan,](https://example.invalid) is in an error state. To resume, open COM-1 and use **Retry** on that task; it rechecks gates and preserves the failed attempt's evidence. Agents cannot retry it for you." + : statusBeforeFinishThenAnswer + ? "Updated the [Launch blockers status card](/COM/status/status-card-launch) with current Synthetic Launch Notes verification.\n\n- **COM-1 \u2014 Mobile Safari checkout regression blocks launch** \u2014 `blocked`\n- Payment approval is now recorded as received; the mobile Safari checkout regression remains the blocker.\n- Only COM-1 appears in the bounded changed-issue set; no other Launch blockers were added." + : answerBeforeFinishThenRestatement ? "I can't change agent budgets from chat — that's board-only, and agent credentials get 403. Two agents also match \"Morgan,\" so choose the intended one and set $50/month on its budget tab: [Morgan budgets](/COM/agents/agent-morgan/budgets) or [Morgan Research budgets](/COM/agents/agent-morgan-research/budgets)." : "Answered above: two inbox items need your attention."; const finalAfterToolCommentary = String( @@ -1010,7 +1041,7 @@ const server = createServer(async (request, response) => { }, }, }); - } else if (answerBeforeFinishThenReference || answerBeforeFinishThenRestatement) { + } else if (answerBeforeFinishThenReference || answerBeforeFinishThenRestatement || answerBeforeFinishThenRewrite || statusBeforeFinishThenAnswer || shortStatusBeforeFinishThenShortAnswer || shortAnswerRepeatedAfterFinish || longAnswerBeforeFinishThenDone) { emit({ type: "message.part.updated", id: "event-answer-before-finish",