diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index a1e31515dcf..89e6caf2d46 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -58,6 +58,8 @@ beforeAll(async () => { fixtureRoot = await mkdtemp(join(tmpdir(), "paperclip-opencode-fixture-")); fixture = join(fixtureRoot, "fake-opencode-server.mjs"); await copyFile(resolve("test/fixtures/fake-opencode-server.mjs"), fixture); + // Tests launch the fixture directly; it must not depend on run order. + await chmod(fixture, 0o755); }); const TURN_TERMINAL_EVENT_TYPES = new Set([ @@ -939,6 +941,71 @@ describe("OpenCodeServerDriver", () => { } }); + it("forwards an unreadable tool-result image as a tool outcome, not as a user message", async () => { + // Shape OpenCode 1.18.34 sends through @ai-sdk/openai-compatible after a + // text-only model reads a screenshot (chat-eval run 42704461144). + const received: unknown[] = []; + const upstream = createHttpServer((request, response) => { + void (async () => { + const chunks = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + received.push(JSON.parse(Buffer.concat(chunks).toString("utf8"))); + response.writeHead(200, { "content-type": "text/event-stream" }); + response.end("data: [DONE]\n\n"); + })().catch(() => response.destroy()); + }); + await new Promise(resolve => upstream.listen(0, "127.0.0.1", resolve)); + const address = upstream.address(); + if (!address || typeof address === "string") throw new Error("Missing fixture address"); + const root = await mkdtemp(join(tmpdir(), "paperclip-opencode-media-")); + roots.push(root); + let session: Awaited> | undefined; + try { + const driver = new OpenCodeServerDriver({ + model: "paperclip/team/model-alias", runtimeDirectory: root, command: fixture, + environment: { PATH: process.env.PATH, PAPERCLIP_AI_PROVIDER_URL: `http://127.0.0.1:${address.port}/v1`, PAPERCLIP_AI_PROVIDER_KEY: "gateway-key" }, + }); + session = await driver.openSession({ runId: "media", normalizedSessionId: "media", workingDirectory: root }); + const config = JSON.parse(await readFile(join(root, "media", "config", "opencode", "opencode.json"), "utf8")); + const { baseURL, apiKey } = config.provider.paperclip.options; + const prefix = [ + { role: "system", content: "system" }, + { role: "user", content: "Build the launch page and verify it." }, + { role: "assistant", content: "", tool_calls: [{ id: "call_read", type: "function", function: { name: "read", arguments: "{\"filePath\":\"/tmp/opencode/website-preview.png\"}" } }] }, + { role: "tool", tool_call_id: "call_read", content: "Image read successfully" }, + ]; + const payload = { + model: "team/model-alias", + stream: true, + messages: [...prefix, { + role: "user", + content: [ + { type: "text", text: "Attached media from tool result:" }, + { type: "text", text: "ERROR: Cannot read image (this model does not support image input). Inform the user." }, + ], + }], + }; + const result = await fetch(`${baseURL}/chat/completions`, { + method: "POST", + headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" }, + body: JSON.stringify(payload), + }); + expect(result.status).toBe(200); + await result.text(); + const forwarded = received[0] as { messages: Array<{ role: string; content: unknown }> }; + expect(forwarded.messages.slice(0, 3)).toEqual(prefix.slice(0, 3)); + expect(forwarded.messages).toHaveLength(prefix.length); + expect(forwarded.messages.at(-1)?.role).toBe("tool"); + const toolContent = String(forwarded.messages.at(-1)?.content); + expect(toolContent).toMatch(/^Image read successfully\n\n\[Paperclip runner: This tool result included an image attachment/); + expect(toolContent).toContain("not from the user"); + expect(JSON.stringify(forwarded)).not.toContain("Inform the user"); + } finally { + await session?.close({ reason: "test" }); + await new Promise(resolve => { upstream.close(() => resolve()); upstream.closeAllConnections(); }); + } + }); + it.each(["gateway-key", ""])("forwards selected-model streams without exposing the reusable key and revokes the proxy on close (%s)", async key => { const received: Array<{ path: string; authorization?: string; body: unknown }> = []; const upstream = createHttpServer((request, response) => { diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 8d536ccaa78..0f9aea629ec 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -24,6 +24,7 @@ import { pipeline } from "node:stream/promises"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { parseOpenCodeReasoningMode, type OpenCodeReasoningMode } from "./reasoning-mode.js"; +import { attributeUnsupportedToolMedia } from "./tool-media-attribution.js"; import { openCodeProviderFailure } from "./provider-failure.js"; import { createOpenCodeCompanyObservability } from "./company-observability.js"; import { startOpenCodePerformanceDiagnostics } from "./performance-diagnostics.js"; @@ -2411,7 +2412,7 @@ async function startOpenCodeProviderProxy(baseUrl: string, key: string, model: s } chunks.push(Buffer.from(chunk)); } - const body = Buffer.concat(chunks); + let body = Buffer.concat(chunks); let payload: Record; try { payload = JSON.parse(body.toString("utf8")); } catch { response.writeHead(400).end(); return; } @@ -2420,6 +2421,9 @@ async function startOpenCodeProviderProxy(baseUrl: string, key: string, model: s response.writeHead(400).end(); return; } + // Keep OpenCode's unreadable-media notice attributed to the tool result + // instead of a synthetic user message the model mistakes for the person. + if (attributeUnsupportedToolMedia(payload)) body = Buffer.from(JSON.stringify(payload), "utf8"); const upstream = await new Promise((resolve, reject) => { const outgoing = (upstreamUrl.protocol === "https:" ? requestHttps : requestHttp)(upstreamUrl, { method: "POST", diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts new file mode 100644 index 00000000000..1a1b8197695 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { attributeUnsupportedToolMedia, OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT } from "./tool-media-attribution.js"; + +const header = { type: "text", text: OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT }; +const imageError = { type: "text", text: "ERROR: Cannot read image (this model does not support image input). Inform the user." }; +const assistantCall = (ids: string[]) => ({ + role: "assistant", + content: "", + tool_calls: ids.map(id => ({ id, type: "function", function: { name: "read", arguments: "{}" } })), +}); + +describe("attributeUnsupportedToolMedia", () => { + it("moves the unreadable-image notice into the producing tool result", () => { + const payload = { messages: [ + { role: "user", content: "verify the page" }, + assistantCall(["a"]), + { role: "tool", tool_call_id: "a", content: "Image read successfully" }, + { role: "user", content: [header, imageError] }, + ] }; + expect(attributeUnsupportedToolMedia(payload)).toBe(true); + expect(payload.messages).toHaveLength(3); + expect(payload.messages[2]).toEqual({ + role: "tool", + tool_call_id: "a", + content: expect.stringMatching(/^Image read successfully\n\n\[Paperclip runner: This tool result included an image attachment that this model cannot read because it does not accept image input, so it was not shown to you\. .* not from the user, and needs no reply\.\]$/), + }); + }); + + it("names the file and keeps readable media in the synthetic message", () => { + const pdf = { type: "file", file: { filename: "brief.pdf", file_data: "data:application/pdf;base64,AA==" } }; + const payload = { messages: [ + { role: "user", content: "go" }, + assistantCall(["a", "b"]), + { role: "tool", tool_call_id: "a", content: "PDF read successfully" }, + { role: "tool", tool_call_id: "b", content: "Image read successfully" }, + { role: "user", content: [header, pdf, { type: "text", text: "ERROR: Cannot read \"shot.png\" (this model does not support image input). Inform the user." }] }, + { role: "assistant", content: "next step" }, + ] }; + expect(attributeUnsupportedToolMedia(payload)).toBe(true); + expect(payload.messages[2].content).toBe("PDF read successfully"); + expect(payload.messages[3].content).toContain("A tool result in this step included an image attachment (shot.png)"); + expect(payload.messages[4]).toEqual({ role: "user", content: [header, pdf] }); + expect(payload.messages[5]).toEqual({ role: "assistant", content: "next step" }); + }); + + it("is deterministic across repeated requests for the same history", () => { + const build = () => ({ messages: [ + assistantCall(["a"]), + { role: "tool", tool_call_id: "a", content: "Image read successfully" }, + { role: "user", content: [header, imageError] }, + ] }); + const first = build(); + const second = build(); + attributeUnsupportedToolMedia(first); + attributeUnsupportedToolMedia(second); + expect(JSON.stringify(first)).toBe(JSON.stringify(second)); + expect(attributeUnsupportedToolMedia(first)).toBe(false); + }); + + it("leaves person-authored messages and unrelated payloads untouched", () => { + const personImage = { messages: [ + { role: "user", content: [{ type: "text", text: "what is in this?" }, imageError] }, + ] }; + const unanchored = { messages: [ + { role: "assistant", content: "hi" }, + { role: "user", content: [header, imageError] }, + ] }; + const snapshot = JSON.stringify([personImage, unanchored]); + expect(attributeUnsupportedToolMedia(personImage)).toBe(false); + expect(attributeUnsupportedToolMedia(unanchored)).toBe(false); + expect(attributeUnsupportedToolMedia({ messages: [{ role: "user", content: "hi" }] })).toBe(false); + expect(attributeUnsupportedToolMedia({})).toBe(false); + expect(JSON.stringify([personImage, unanchored])).toBe(snapshot); + }); +}); diff --git a/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts new file mode 100644 index 00000000000..d7e5ddb4e21 --- /dev/null +++ b/packages/paperclip-runner/src/drivers/opencode/tool-media-attribution.ts @@ -0,0 +1,98 @@ +/** + * OpenCode (qualified 1.18.34) cannot place images in OpenAI-compatible tool + * results, so it moves tool-result media into a synthetic *user* message that + * starts with "Attached media from tool result:". When the selected model has + * no image input, OpenCode then swaps each image for + * "ERROR: Cannot read image (this model does not support image input). Inform + * the user." The model sees that as the person speaking and derails: it redoes + * finished work or tells the person about an error they never caused. + * + * Every OpenCode request for an OpenRouter or Paperclip-connection model goes + * through the runner's provider proxy, so the proxy moves that notice back + * into the tool result it came from, worded as a tool outcome. Real user + * messages (including images a person attached) are never touched, and the + * rewrite is deterministic so prompt-cache prefixes stay stable. + */ + +export const OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT = "Attached media from tool result:"; + +const UNSUPPORTED_MEDIA = /^ERROR: Cannot read (.+) \(this model does not support ([a-z]+) input\)\. Inform the user\.$/; +const EMPTY_IMAGE = "ERROR: Image file is empty or corrupted. Please provide a valid image."; + +type Message = Record; + +function textOf(part: unknown): string | null { + if (!part || typeof part !== "object") return null; + const record = part as Record; + return record.type === "text" && typeof record.text === "string" ? record.text : null; +} + +function describe(text: string): string | null { + if (text === EMPTY_IMAGE) return "an image attachment that was empty or corrupted"; + const match = UNSUPPORTED_MEDIA.exec(text); + if (!match) return null; + const [, name, modality] = match; + const article = /^[aeiou]/.test(modality) ? "an" : "a"; + const file = name === modality ? "" : ` (${name.replace(/^"|"$/g, "")})`; + const subject = `${article} ${modality} attachment${file}`; + return `${subject} that this model cannot read because it does not accept ${modality} input`; +} + +function toolNote(descriptions: string[], toolResults: number): string { + const unique = [...new Set(descriptions)]; + const source = toolResults === 1 ? "This tool result" : "A tool result in this step"; + return [ + `[Paperclip runner: ${source} included ${unique.join("; ")}, so it was not shown to you.`, + "The tool call itself completed. Do not retry viewing it; verify with text-based checks instead.", + "This note comes from the runner, not from the user, and needs no reply.]", + ].join(" "); +} + +function appendToToolMessage(message: Message, note: string): boolean { + if (typeof message.content === "string") { + message.content = message.content ? `${message.content}\n\n${note}` : note; + return true; + } + if (Array.isArray(message.content)) { + message.content = [...message.content, { type: "text", text: note }]; + return true; + } + return false; +} + +/** + * Rewrites an OpenAI-compatible chat-completions payload in place. Returns + * true when anything changed, so the caller can keep the original bytes + * otherwise. + */ +export function attributeUnsupportedToolMedia(payload: Record): boolean { + const messages = payload.messages; + if (!Array.isArray(messages)) return false; + let changed = false; + for (let index = messages.length - 1; index > 0; index -= 1) { + const message = messages[index] as Message | null; + if (!message || message.role !== "user" || !Array.isArray(message.content)) continue; + const [header, ...rest] = message.content as unknown[]; + if (textOf(header) !== OPENCODE_SYNTHETIC_ATTACHMENT_PROMPT) continue; + const descriptions: string[] = []; + const kept: unknown[] = []; + for (const part of rest) { + const text = textOf(part); + const description = text === null ? null : describe(text); + if (description) descriptions.push(description); + else kept.push(part); + } + if (descriptions.length === 0) continue; + // OpenCode places the synthetic message directly after the tool results + // of the step that produced the media. Anything else is not ours to edit. + let firstTool = index; + while (firstTool > 0 && (messages[firstTool - 1] as Message | null)?.role === "tool") firstTool -= 1; + const toolResults = index - firstTool; + if (toolResults === 0) continue; + if (!appendToToolMessage(messages[index - 1] as Message, toolNote(descriptions, toolResults))) continue; + if (kept.length === 0) messages.splice(index, 1); + else message.content = [header, ...kept]; + changed = true; + } + return changed; +}