From 7941f6b05161386ca0b280ca3509d78410c5d1d9 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:08:04 -0600 Subject: [PATCH] fix(runner): show OpenRouter's invalid model ID as a model rejection (#19548) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - Native runs use the Paperclip runner with drivers such as OpenCode > - When a provider rejects the selected model, the product shows "Model unavailable" and offers a model choice > - OpenCode reports provider HTTP errors as `APIError` with the message under `data`, not at the top level > - The runner then reported "Provider turn failed", so an invalid model ended as a generic adapter failure > - This pull request classifies OpenRouter's invalid model ID with a closed provider-failure category, and makes the server keep that classification > - The benefit is that a person sees "Model unavailable" and the real provider message, and can choose another model ## Linked Issues or Issue Description **What happened?** A chat evaluation selected a model ID that OpenRouter does not know. OpenCode reported `turn.failed` with `error: { name: "APIError", data: { message: " is not a valid model ID", statusCode: 400, isRetryable: false } }`. The run was saved with `errorCode: adapter_failed`. The task chat showed "The OpenCode run failed before it completed." and no "Model unavailable" marker. The provider notice said only "OpenCode session failed.". **Expected behavior** A non-retryable rejection of the selected model is saved as `native_provider_model_rejected`, so the product shows "Model unavailable" and the model choice. The notice shows the provider's message. **Steps to reproduce** 1. Configure an OpenCode agent with an OpenRouter model ID that does not exist. 2. Send the agent a message. 3. Before this change, the run fails as `adapter_failed` and the notice says "OpenCode session failed.". ## What Changed - `drivers/opencode/provider-failure.ts`: a non-retryable OpenCode provider error whose message says the model is not valid (`not a valid model`, `model_not_found`, model not found or not supported) gets the closed category `model_unavailable`. A provider that allows a retry keeps the existing category. - `native-session-runtime.ts`: a non-recoverable failure with `model_unavailable` is a model rejection. The runtime reads the closed category, not provider text. - `native-session-executor.ts`: a non-recoverable `NativeProviderTerminalFailure` that the runner marked as a model rejection is saved as `native_provider_model_rejected`. Before, every such terminal was saved as `native_provider_terminal_failed` before the marker was read. - `drivers/opencode/opencode-server-driver.ts`: the provider notice summary shows the provider message from `data` when there is no top-level message. Provider text is not used as the turn failure message, so it does not reach `run.error` or Sentry. - Tests: provider-failure mapping (retryable and non-retryable), runtime classification by category, driver notice summary with no failure message, and an executor test for the saved code and terminal disposition. ## Verification - `npx vitest run src/native-session-runtime.test.ts src/drivers/provider-failure.test.ts src/drivers/opencode` in `packages/paperclip-runner`: 16 files, 412 tests passed. - `npx vitest run src/services/native-runtime/native-session-executor.test.ts` in `server`: 1,152 tests passed. - Negative controls: reverting the provider-failure mapping, the runtime check, or the executor branch each makes its new test fail. ## Risks - Low risk. Classification uses a closed category and a runner-set marker, not provider text. - The model check on the provider message is text-based inside the OpenCode mapper. A provider that uses "not a valid model" for another reason with a non-retryable error will show "Model unavailable". - Provider text appears in the notice summary, as provider notices already did for top-level messages. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../opencode/opencode-server-driver.test.ts | 3 +++ .../opencode/opencode-server-driver.ts | 10 ++++---- .../src/drivers/opencode/provider-failure.ts | 7 +++++- .../src/drivers/provider-failure.test.ts | 8 +++++++ .../src/native-session-runtime.test.ts | 24 +++++++++++++++---- .../src/native-session-runtime.ts | 12 ++++++---- .../native-session-executor.test.ts | 12 ++++++++++ .../native-runtime/native-session-executor.ts | 4 ++++ 8 files changed, 67 insertions(+), 13 deletions(-) diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts index 840d36f97c7..9afbe5c5a51 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.test.ts @@ -3084,6 +3084,9 @@ describe("OpenCodeServerDriver", () => { const observed = await events; const failure = observed.find(event => event.eventType === "turn.failed"); expect(failure?.payload.error).toMatchObject({ providerFailure: { provider: "openrouter", category: "overloaded", message: "Provider is busy." } }); + // Provider text stays in display fields: the notice summary, never the failure message. + expect((failure?.payload.error as Record).message).toBeUndefined(); + expect(observed.find(event => event.eventType === "provider.notice.recorded")?.payload.summary).toBe("Provider is busy."); expect(JSON.stringify(failure)).not.toContain("private-body"); } finally { await session.close({ reason: "test complete" }); } }); diff --git a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts index 9010a7a9646..4b2f24b79e5 100644 --- a/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts +++ b/packages/paperclip-runner/src/drivers/opencode/opencode-server-driver.ts @@ -1972,6 +1972,11 @@ class OpenCodeHarnessSession implements HarnessSession { this.#activeTurnId = null; return; } + // OpenCode's provider errors (`APIError`) carry the message under `data`. + const providerMessage = redact( + text(providerError.message, text(providerErrorData.message, "OpenCode session failed.")), + this.#runtime.sensitiveValues, + ).slice(0, 4000); this.#emit( "provider.notice.recorded", { @@ -1982,10 +1987,7 @@ class OpenCodeHarnessSession implements HarnessSession { scope: "turn", recoverable: false, userActionable: true, - summary: redact( - text(record(properties.error).message, "OpenCode session failed."), - this.#runtime.sensitiveValues, - ).slice(0, 4000), + summary: providerMessage, }, { turnId, itemId: `${turnId}:session-error` }, ); diff --git a/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts b/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts index 03ffecfac84..920c1d60a3e 100644 --- a/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts +++ b/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts @@ -22,6 +22,9 @@ function providerErrorEnvelope(message: string): { message?: string; httpStatus? }; } +// OpenRouter reports an unknown model ID as " is not a valid model ID". +const MODEL_UNAVAILABLE = /not a valid model|model_not_found|model[^\n]*(?:does not exist|not found|not supported)/i; + export function openCodeProviderFailure(error: Record, provider: string, environment: NodeJS.ProcessEnv = {}) { const data = error.data && typeof error.data === "object" ? error.data as Record : {}; @@ -31,5 +34,7 @@ export function openCodeProviderFailure(error: Record, provider return createProviderFailure({ provider, phase: "turn", message: envelope.message ?? message, httpStatus: data.statusCode ?? envelope.httpStatus, providerCode: data.code ?? envelope.providerCode ?? error.name, requestId: data.requestID, - category: error.name === "ProviderAuthError" ? "authentication" : undefined, environment }); + category: error.name === "ProviderAuthError" ? "authentication" + : data.isRetryable !== true && MODEL_UNAVAILABLE.test(envelope.message ?? message) ? "model_unavailable" : undefined, + environment }); } diff --git a/packages/paperclip-runner/src/drivers/provider-failure.test.ts b/packages/paperclip-runner/src/drivers/provider-failure.test.ts index fb128eaa21a..f0e48ca19a4 100644 --- a/packages/paperclip-runner/src/drivers/provider-failure.test.ts +++ b/packages/paperclip-runner/src/drivers/provider-failure.test.ts @@ -95,6 +95,14 @@ describe("provider failure boundary", () => { .toMatchObject({ category: "rate_limit", retryability: "unknown" }); }); + it("classifies OpenRouter's invalid model ID as an unavailable model unless OpenCode allows a retry", () => { + const data = { message: "openrouter/unavailable-model is not a valid model ID", statusCode: 400 }; + expect(openCodeProviderFailure({ name: "APIError", data: { ...data, isRetryable: false } }, "openrouter")) + .toMatchObject({ category: "model_unavailable", retryability: "requires_action", httpStatus: 400 }); + expect(openCodeProviderFailure({ name: "APIError", data: { ...data, isRetryable: true } }, "openrouter")?.category) + .not.toBe("model_unavailable"); + }); + it("extracts OpenCode API error fields without copying response bodies or headers", () => { const failure = openCodeProviderFailure({ name: "APIError", data: { message: "Service is busy.", statusCode: 503, responseBody: "private request", responseHeaders: { authorization: "secret" } } }, "openrouter"); diff --git a/packages/paperclip-runner/src/native-session-runtime.test.ts b/packages/paperclip-runner/src/native-session-runtime.test.ts index 4ffdcc26909..e77ab6e17ed 100644 --- a/packages/paperclip-runner/src/native-session-runtime.test.ts +++ b/packages/paperclip-runner/src/native-session-runtime.test.ts @@ -1,4 +1,5 @@ import { validatePrpStructuredRunResult } from "./protocol/replay-contract.js"; +import { createProviderFailure } from "./drivers/provider-failure.js"; import { createHash } from "node:crypto"; import { writeFile } from "node:fs/promises"; import { join } from "node:path"; @@ -1501,9 +1502,22 @@ describe("executeNativeSession recovery", () => { message: "The model service failed while processing output.", modelRejected: false, }, - ])( - "preserves structured provider failure and model retry classification ($recoverable, $modelRejected)", - async ({ recoverable, message, modelRejected }) => { + { + // A driver's closed category marks the rejection; no provider text is needed. + recoverable: false, + message: "Provider turn failed", + modelRejected: true, + category: "model_unavailable" as const, + }, + { + recoverable: true, + message: "Provider turn failed", + modelRejected: false, + category: "model_unavailable" as const, + }, + ] as Array<{ recoverable: boolean; message: string; modelRejected: boolean; category?: "model_unavailable" }>)( + "preserves structured provider failure and model retry classification ($recoverable, $modelRejected, $category)", + async ({ recoverable, message, modelRejected, category }) => { const capabilities = { resume: true, typedEvents: true, @@ -1519,7 +1533,9 @@ describe("executeNativeSession recovery", () => { }, async *events() { yield runnerEvent(1, "turn.failed", { - error: { code: "RUNTIME", recoverable, message }, + error: { code: "RUNTIME", recoverable, message, ...(category ? { providerFailure: createProviderFailure({ + provider: "openrouter", phase: "turn", message: "openrouter/unavailable-model is not a valid model ID", httpStatus: 400, category, + }) } : {}) }, }); }, async startTurn() { diff --git a/packages/paperclip-runner/src/native-session-runtime.ts b/packages/paperclip-runner/src/native-session-runtime.ts index a03ac2c6aee..c2ed976c24c 100644 --- a/packages/paperclip-runner/src/native-session-runtime.ts +++ b/packages/paperclip-runner/src/native-session-runtime.ts @@ -2860,11 +2860,15 @@ export async function executeNativeSession( // Retain the older consumer's permanent-model classification while // preserving structured provider metadata. A provider explicitly // permitting retry must not become permanent merely from its text. + // A driver's closed provider-failure category also marks a model + // rejection, without putting provider text in the failure message. + const providerFailure = readProviderFailure(failure.providerFailure); const modelRejected = !recoverable && - /issue with the selected model|model_not_found|invalid model|model[^\n]*(?:does not exist|not found|not supported)/i.test( - message, - ); + (providerFailure?.category === "model_unavailable" || + /issue with the selected model|model_not_found|invalid model|model[^\n]*(?:does not exist|not found|not supported)/i.test( + message, + )); throw new NativeProviderTerminalFailure( typeof failure.code === "string" ? failure.code @@ -2873,7 +2877,7 @@ export async function executeNativeSession( modelRejected ? `native_provider_model_rejected: ${message}` : message, - readProviderFailure(failure.providerFailure), + providerFailure, ); } if (settledCompletion === null && options.resolveMissingResult) { diff --git a/server/src/services/native-runtime/native-session-executor.test.ts b/server/src/services/native-runtime/native-session-executor.test.ts index 5ce7ab0c1d3..d140b35a031 100644 --- a/server/src/services/native-runtime/native-session-executor.test.ts +++ b/server/src/services/native-runtime/native-session-executor.test.ts @@ -18791,6 +18791,18 @@ it.each([ ).toBe("native_event_replay_conflict"); }); +it("saves a runner-marked model rejection from a provider terminal as a model choice", () => { + const rejected = new NativeProviderTerminalFailure("provider_turn_failed", false, + "native_provider_model_rejected: openrouter/unavailable-model is not a valid model ID"); + const code = nativeSessionFailureSourceCode(rejected); + expect(code).toBe("native_provider_model_rejected"); + expect(nativeSessionFailureDisposition(1, new Date(), code)).toEqual({ phase: "terminal_failure", failureCode: code, nextAttemptAt: null }); + // A recoverable terminal or an unmarked message stays a generic provider failure. + expect(nativeSessionFailureSourceCode(new NativeProviderTerminalFailure("provider_turn_failed", true, + "native_provider_model_rejected: openrouter/unavailable-model is not a valid model ID"))).toBe("native_provider_terminal_failed"); + expect(nativeSessionFailureSourceCode(new NativeProviderTerminalFailure("provider_turn_failed", false, + "openrouter/unavailable-model is not a valid model ID"))).toBe("native_provider_terminal_failed"); +}); it("keeps an ACPX missing runtime package failure actionable and stops automatic retries", () => { const failure = new Error('PRP command session.open failed: {"result":{"code":"command_execution_failed","message":"failed to start ACPX provider: ACPX sidecar command session.open was rejected (retryable=false, classification=runtime_package_missing)"},"status":"failed"}'); const code = nativeSessionFailureSourceCode(failure); diff --git a/server/src/services/native-runtime/native-session-executor.ts b/server/src/services/native-runtime/native-session-executor.ts index 5f8201f8ab7..acc9ace7e60 100644 --- a/server/src/services/native-runtime/native-session-executor.ts +++ b/server/src/services/native-runtime/native-session-executor.ts @@ -7437,6 +7437,10 @@ export function nativeSessionFailureSourceCode( ) ) return "native_event_replay_conflict"; + // The runner marks a non-recoverable rejection of the selected model in the + // terminal message; it needs a model choice, not a generic provider failure. + if (!error.recoverable && /^native_provider_model_rejected:/.test(error.message)) + return "native_provider_model_rejected"; return "native_provider_terminal_failed"; } if (error instanceof NativeSessionProtocolIntegrityError)