From 60a60a0cd11dd7f06e30b16418e395132ca548b6 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 9 Nov 2026 18:11:50 -0600 Subject: [PATCH] fix(opencode): attribute unreadable tool-result images to the tool (#20399) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - The Paperclip runner drives OpenCode agents. It sends their OpenRouter and Paperclip-connection model calls through a local provider proxy. > - An agent often takes a screenshot of its own work and reads it to verify the result. > - OpenCode 1.18.34 cannot put images in OpenAI-compatible tool results. It moves them into a synthetic user message ("Attached media from tool result:"). For a text-only model, it replaces the image with "ERROR: Cannot read image (this model does not support image input). Inform the user." > - The model reads that text as a message from the person. In a chat eval, a DeepSeek worker redid its verification for about 5 minutes, registered its deliverable twice, and ran out of the 15-minute attempt clock. > - This pull request moves that notice into the tool result that produced the image, worded as a runner note, and removes the synthetic user message. > - The benefit is that a text-only agent sees a clear tool outcome and continues its work. It does not mistake the notice for the person. ## Linked Issues or Issue Description **What happened?** An OpenCode agent on `openrouter/deepseek/deepseek-v4.1-flash-1017` read a PNG screenshot that it made. The next model request contained a user-role message with the text "Attached media from tool result: ERROR: Cannot read image (this model does not support image input). Inform the user." The model reasoned "The previous input from the user was ..." and repeated finished work. **Steps to reproduce** 1. Run OpenCode 1.18.34 with an `@ai-sdk/openai-compatible` provider and a model that has no image input. 2. Have the model call the `read` tool on a PNG file. 3. Inspect the next chat-completions request. The last message is the synthetic user message above. **Expected behavior** The model must get the notice as part of the tool result, not as a message from the person. **Environment** Paperclip runner, OpenCode 1.18.34 (qualified pin), OpenRouter and Paperclip-connection model routes. ## What Changed - Add `packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts`. It finds the OpenCode synthetic attachment message only when it directly follows tool results. It moves each "Cannot read ..." or "empty or corrupted" notice into the last tool result of that step as a runner note. It keeps readable media in the synthetic message. It removes the message when no media remains. - The OpenCode provider proxy applies this rewrite before it forwards a request. It re-serializes the body only when something changed. - Messages written by a person are never changed. A person who attaches an image to a text-only model still gets "Inform the user", which is correct for that case. - Add unit tests for the rewrite and an integration test through the real provider proxy. ## Verification - Red: the new proxy test failed before the change (`expected ... to have a length of 4 but got 5`). - Green: `npx vitest run src/drivers/opencode/opencode-server-driver.test.ts src/drivers/opencode/tool-media-attribution.test.ts` (82 tests pass). `tsc --noEmit -p packages/paperclip-runner` is clean. - Real binary: I ran the real OpenCode 1.18.34 binary with a mock OpenAI-compatible server and had the model read a PNG. The captured request has exactly the message shape that the test fixture uses. - `pnpm verify` returned exit 2 (test files outside the resolved graphs). Required CI supplies broad coverage. - A paid live run of `chat-features.runner-opencode.local.task-coordination.delegate-with-context` will follow in a PR comment. ## Risks - Low risk. The rewrite matches the exact OpenCode 1.18.34 strings. If a future OpenCode version changes them, the rewrite does nothing and the old behavior returns. It does not fail the request. - The rewrite is deterministic, so prompt-cache prefixes stay stable across requests. - No persisted state changes. OpenCode history stays as it is. Sessions saved before this change also get the rewrite when they resume. - Providers that do not use the runner provider proxy are not covered by this change. ## Model Used - Claude Opus 5.5 (Anthropic), model ID `claude-opus-5-5`, used through Claude Code with tool use (shell, file edits, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../opencode/opencode-server-driver.test.ts | 67 +++++++++++++ .../opencode/opencode-server-driver.ts | 6 +- .../opencode/tool-media-attribution.test.ts | 75 ++++++++++++++ .../opencode/tool-media-attribution.ts | 98 +++++++++++++++++++ 4 files changed, 245 insertions(+), 1 deletion(-) create mode 100644 packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts create mode 100644 packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index a1e31515dcf..89e6caf2d46 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -58,6 +58,8 @@ beforeAll(async () => { fixtureRoot = await mkdtemp(join(tmpdir(), "paperclip-opencode-fixture-")); fixture = join(fixtureRoot, "fake-opencode-server.mjs"); await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), fixture); + // Tests launch the fixture directly; it must not depend on run order. + await chmod(fixture, 0o755); }); const TURN_TERMINAL_EVENT_TYPES = new Set([ @@ -939,6 +941,71 @@ describe("OpenCodeServerDriver", () => { } }); + it("forwards an unreadable tool-result image as a tool outcome, not as a user message", async () => { + // Shape OpenCode 1.18.34 sends through @ai-sdk/openai-compatible after a + // text-only model reads a screenshot (chat-eval run 42704461144). + const received: unknown[] = []; + const upstream = createHttpServer((request, response) => { + void (async () => { + const chunks = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + received.push(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("data: [DONE]\n\n"); + })().catch(() => response.destroy()); + }); + await new Promise(resolve => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture address"); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-media-")); + roots.push(root); + let session: Awaited> | undefined; + try { + const driver = new OpenCodeServerDriver({ + model: "paperclip/team/model-alias", runtimeDirectory: root, command: fixture, + environment: { PATH: process.env.PATH, PAPERCLIP_AI_PROVIDER_URL: `http://127.0.0.1:${address.port}/v1`, PAPERCLIP_AI_PROVIDER_KEY: "gateway-key" }, + }); + session = await driver.openSession({ runId: "media", normalizedSessionId: "media", workingDirectory: root }); + const config = JSON.parse(await readFile(join(root, "media", "config", "opencode", "opencode.json"), "utf8")); + const { baseURL, apiKey } = config.provider.paperclip.options; + const prefix = [ + { role: "system", content: "system" }, + { role: "user", content: "Build the launch page and verify it." }, + { role: "assistant", content: "", tool_calls: [{ id: "call_read", type: "function", function: { name: "read", arguments: "{\"filePath\":\"/tmp/opencode/website-preview.png\"}" } }] }, + { role: "tool", tool_call_id: "call_read", content: "Image read successfully" }, + ]; + const payload = { + model: "team/model-alias", + stream: true, + messages: [...prefix, { + role: "user", + content: [ + { type: "text", text: "Attached media from tool result:" }, + { type: "text", text: "ERROR: Cannot read image (this model does not support image input). Inform the user." }, + ], + }], + }; + const result = await fetch(`${baseURL}/chat/completions`, { + method: "POST", + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + expect(result.status).toBe(200); + await result.text(); + const forwarded = received[0] as { messages: Array<{ role: string; content: unknown }> }; + expect(forwarded.messages.slice(0, 3)).toEqual(prefix.slice(0, 3)); + expect(forwarded.messages).toHaveLength(prefix.length); + expect(forwarded.messages.at(-1)?.role).toBe("tool"); + const toolContent = String(forwarded.messages.at(-1)?.content); + expect(toolContent).toMatch(/^Image read successfully\n\n\[Paperclip runner: This tool result included an image attachment/); + expect(toolContent).toContain("not from the user"); + expect(JSON.stringify(forwarded)).not.toContain("Inform the user"); + } finally { + await session?.close({ reason: "test" }); + await new Promise(resolve => { upstream.close(() => resolve()); upstream.closeAllConnections(); }); + } + }); + it.each(["gateway-key", ""])("forwards selected-model streams without exposing the reusable key and revokes the proxy on close (%s)", async key => { const received: Array<{ path: string; authorization?: string; body: unknown }> = []; const upstream = createHttpServer((request, response) => { diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 8d536ccaa78..0f9aea629ec 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -24,6 +24,7 @@ import { pipeline } from "node:stream/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { parseOpenCodeReasoningMode, type OpenCodeReasoningMode } from "./reasoning-mode.js"; +import { attributeUnsupportedToolMedia } from "./tool-media-attribution.js"; import { openCodeProviderFailure } from "./provider-failure.js"; import { createOpenCodeCompanyObservability } from "./company-observability.js"; import { startOpenCodePerformanceDiagnostics } from "./performance-diagnostics.js"; @@ -2411,7 +2412,7 @@ async function startOpenCodeProviderProxy(baseUrl: string, key: string, model: s } chunks.push(Buffer.from(chunk)); } - const body = Buffer.concat(chunks); + let body = Buffer.concat(chunks); let payload: Record; try { payload = JSON.parse(body.toString("utf8")); } catch { response.writeHead(400).end(); return; } @@ -2420,6 +2421,9 @@ async function startOpenCodeProviderProxy(baseUrl: string, key: string, model: s response.writeHead(400).end(); return; } + // Keep OpenCode's unreadable-media notice attributed to the tool result + // instead of a synthetic user message the model mistakes for the person. + if (attributeUnsupportedToolMedia(payload)) body = Buffer.from(JSON.stringify(payload), "utf8"); const upstream = await new Promise((resolve, reject) => { const outgoing = (upstreamUrl.protocol === "https:" ? requestHttps : requestHttp)(upstreamUrl, { method: "POST", diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts new file mode 100644 index 00000000000..1a1b8197695 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { attributeUnsupportedToolMedia, OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT } from "./tool-media-attribution.js"; + +const header = { type: "text", text: OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT }; +const imageError = { type: "text", text: "ERROR: Cannot read image (this model does not support image input). Inform the user." }; +const assistantCall = (ids: string[]) => ({ + role: "assistant", + content: "", + tool_calls: ids.map(id => ({ id, type: "function", function: { name: "read", arguments: "{}" } })), +}); + +describe("attributeUnsupportedToolMedia", () => { + it("moves the unreadable-image notice into the producing tool result", () => { + const payload = { messages: [ + { role: "user", content: "verify the page" }, + assistantCall(["a"]), + { role: "tool", tool_call_id: "a", content: "Image read successfully" }, + { role: "user", content: [header, imageError] }, + ] }; + expect(attributeUnsupportedToolMedia(payload)).toBe(true); + expect(payload.messages).toHaveLength(3); + expect(payload.messages[2]).toEqual({ + role: "tool", + tool_call_id: "a", + content: expect.stringMatching(/^Image read successfully\n\n\[Paperclip runner: This tool result included an image attachment that this model cannot read because it does not accept image input, so it was not shown to you\. .* not from the user, and needs no reply\.\]$/), + }); + }); + + it("names the file and keeps readable media in the synthetic message", () => { + const pdf = { type: "file", file: { filename: "brief.pdf", file_data: "data:application/pdf;base64,AA==" } }; + const payload = { messages: [ + { role: "user", content: "go" }, + assistantCall(["a", "b"]), + { role: "tool", tool_call_id: "a", content: "PDF read successfully" }, + { role: "tool", tool_call_id: "b", content: "Image read successfully" }, + { role: "user", content: [header, pdf, { type: "text", text: "ERROR: Cannot read \"shot.png\" (this model does not support image input). Inform the user." }] }, + { role: "assistant", content: "next step" }, + ] }; + expect(attributeUnsupportedToolMedia(payload)).toBe(true); + expect(payload.messages[2].content).toBe("PDF read successfully"); + expect(payload.messages[3].content).toContain("A tool result in this step included an image attachment (shot.png)"); + expect(payload.messages[4]).toEqual({ role: "user", content: [header, pdf] }); + expect(payload.messages[5]).toEqual({ role: "assistant", content: "next step" }); + }); + + it("is deterministic across repeated requests for the same history", () => { + const build = () => ({ messages: [ + assistantCall(["a"]), + { role: "tool", tool_call_id: "a", content: "Image read successfully" }, + { role: "user", content: [header, imageError] }, + ] }); + const first = build(); + const second = build(); + attributeUnsupportedToolMedia(first); + attributeUnsupportedToolMedia(second); + expect(JSON.stringify(first)).toBe(JSON.stringify(second)); + expect(attributeUnsupportedToolMedia(first)).toBe(false); + }); + + it("leaves person-authored messages and unrelated payloads untouched", () => { + const personImage = { messages: [ + { role: "user", content: [{ type: "text", text: "what is in this?" }, imageError] }, + ] }; + const unanchored = { messages: [ + { role: "assistant", content: "hi" }, + { role: "user", content: [header, imageError] }, + ] }; + const snapshot = JSON.stringify([personImage, unanchored]); + expect(attributeUnsupportedToolMedia(personImage)).toBe(false); + expect(attributeUnsupportedToolMedia(unanchored)).toBe(false); + expect(attributeUnsupportedToolMedia({ messages: [{ role: "user", content: "hi" }] })).toBe(false); + expect(attributeUnsupportedToolMedia({})).toBe(false); + expect(JSON.stringify([personImage, unanchored])).toBe(snapshot); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts new file mode 100644 index 00000000000..d7e5ddb4e21 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts @@ -0,0 +1,98 @@ +/** + * OpenCode (qualified 1.18.34) cannot place images in OpenAI-compatible tool + * results, so it moves tool-result media into a synthetic *user* message that + * starts with "Attached media from tool result:". When the selected model has + * no image input, OpenCode then swaps each image for + * "ERROR: Cannot read image (this model does not support image input). Inform + * the user." The model sees that as the person speaking and derails: it redoes + * finished work or tells the person about an error they never caused. + * + * Every OpenCode request for an OpenRouter or Paperclip-connection model goes + * through the runner's provider proxy, so the proxy moves that notice back + * into the tool result it came from, worded as a tool outcome. Real user + * messages (including images a person attached) are never touched, and the + * rewrite is deterministic so prompt-cache prefixes stay stable. + */ + +export const OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:"; + +const UNSUPPORTED_MEDIA = /^ERROR: Cannot read (.+) \(this model does not support ([a-z]+) input\)\. Inform the user\.$/; +const EMPTY_IMAGE = "ERROR: Image file is empty or corrupted. Please provide a valid image."; + +type Message = Record; + +function textOf(part: unknown): string | null { + if (!part || typeof part !== "object") return null; + const record = part as Record; + return record.type === "text" && typeof record.text === "string" ? record.text : null; +} + +function describe(text: string): string | null { + if (text === EMPTY_IMAGE) return "an image attachment that was empty or corrupted"; + const match = UNSUPPORTED_MEDIA.exec(text); + if (!match) return null; + const [, name, modality] = match; + const article = /^[aeiou]/.test(modality) ? "an" : "a"; + const file = name === modality ? "" : ` (${name.replace(/^"|"$/g, "")})`; + const subject = `${article} ${modality} attachment${file}`; + return `${subject} that this model cannot read because it does not accept ${modality} input`; +} + +function toolNote(descriptions: string[], toolResults: number): string { + const unique = [...new Set(descriptions)]; + const source = toolResults === 1 ? "This tool result" : "A tool result in this step"; + return [ + `[Paperclip runner: ${source} included ${unique.join("; ")}, so it was not shown to you.`, + "The tool call itself completed. Do not retry viewing it; verify with text-based checks instead.", + "This note comes from the runner, not from the user, and needs no reply.]", + ].join(" "); +} + +function appendToToolMessage(message: Message, note: string): boolean { + if (typeof message.content === "string") { + message.content = message.content ? `${message.content}\n\n${note}` : note; + return true; + } + if (Array.isArray(message.content)) { + message.content = [...message.content, { type: "text", text: note }]; + return true; + } + return false; +} + +/** + * Rewrites an OpenAI-compatible chat-completions payload in place. Returns + * true when anything changed, so the caller can keep the original bytes + * otherwise. + */ +export function attributeUnsupportedToolMedia(payload: Record): boolean { + const messages = payload.messages; + if (!Array.isArray(messages)) return false; + let changed = false; + for (let index = messages.length - 1; index > 0; index -= 1) { + const message = messages[index] as Message | null; + if (!message || message.role !== "user" || !Array.isArray(message.content)) continue; + const [header, ...rest] = message.content as unknown[]; + if (textOf(header) !== OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT) continue; + const descriptions: string[] = []; + const kept: unknown[] = []; + for (const part of rest) { + const text = textOf(part); + const description = text === null ? null : describe(text); + if (description) descriptions.push(description); + else kept.push(part); + } + if (descriptions.length === 0) continue; + // OpenCode places the synthetic message directly after the tool results + // of the step that produced the media. Anything else is not ours to edit. + let firstTool = index; + while (firstTool > 0 && (messages[firstTool - 1] as Message | null)?.role === "tool") firstTool -= 1; + const toolResults = index - firstTool; + if (toolResults === 0) continue; + if (!appendToToolMessage(messages[index - 1] as Message, toolNote(descriptions, toolResults))) continue; + if (kept.length === 0) messages.splice(index, 1); + else message.content = [header, ...kept]; + changed = true; + } + return changed; +}