diff --git a/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts b/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts index f6532b999aa..03ffecfac84 100644 --- a/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts +++ b/packages/paperclip-runner/src/drivers/opencode/provider-failure.ts @@ -1,11 +1,35 @@ import { createProviderFailure } from "../provider-failure.js"; +function record(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +/** + * OpenCode reports some upstream failures as an `UnknownError` whose message is + * the provider's JSON error, for example OpenRouter's + * `{"code":502,"message":"...","metadata":{"error_type":"provider_unavailable"}}`. + * Read its status, type and message instead of presenting the raw body. + */ +function providerErrorEnvelope(message: string): { message?: string; httpStatus?: unknown; providerCode?: unknown } { + if (message.length > 16_000) return {}; + let response: Record; + try { response = record(JSON.parse(message)); } catch { return {}; } + const detail = Object.keys(record(response.error)).length ? record(response.error) : response; + return { + message: typeof detail.message === "string" && detail.message.trim() ? detail.message : undefined, + httpStatus: detail.code, + providerCode: record(detail.metadata).error_type, + }; +} + export function openCodeProviderFailure(error: Record, provider: string, environment: NodeJS.ProcessEnv = {}) { const data = error.data && typeof error.data === "object" ? error.data as Record : {}; const message = typeof data.message === "string" ? data.message : error.message; if (typeof message !== "string") return null; - return createProviderFailure({ provider, phase: "turn", message, httpStatus: data.statusCode, - providerCode: data.code ?? error.name, requestId: data.requestID, + const envelope = providerErrorEnvelope(message); + return createProviderFailure({ provider, phase: "turn", message: envelope.message ?? message, + httpStatus: data.statusCode ?? envelope.httpStatus, + providerCode: data.code ?? envelope.providerCode ?? error.name, requestId: data.requestID, category: error.name === "ProviderAuthError" ? "authentication" : undefined, environment }); } diff --git a/packages/paperclip-runner/src/drivers/provider-failure.test.ts b/packages/paperclip-runner/src/drivers/provider-failure.test.ts index cda14560019..27f28d6f757 100644 --- a/packages/paperclip-runner/src/drivers/provider-failure.test.ts +++ b/packages/paperclip-runner/src/drivers/provider-failure.test.ts @@ -79,6 +79,24 @@ describe("provider failure boundary", () => { expect(JSON.stringify(failure)).not.toMatch(/private request|secret|responseHeaders/); }); + it("reads the upstream status and type from an OpenRouter error OpenCode reports as UnknownError", () => { + const upstream = "Upstream error from Fireworks: Grammar processing took too much time."; + const body = { code: 502, message: upstream, metadata: { error_type: "provider_unavailable" } }; + expect(openCodeProviderFailure({ name: "UnknownError", data: { message: JSON.stringify(body) } }, "openrouter")) + .toEqual({ schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn", category: "unknown", + message: upstream, providerCode: "provider_unavailable", httpStatus: 502, retryability: "unknown" }); + // The wrapped `{ error: ... }` form, with OpenCode's own status code taking precedence. + expect(openCodeProviderFailure({ name: "UnknownError", data: { statusCode: 503, + message: JSON.stringify({ error: body }) } }, "openrouter")) + .toMatchObject({ httpStatus: 503, category: "overloaded", message: upstream, providerCode: "provider_unavailable" }); + // Plain text, and JSON without a usable message or status, keep the original fields. + expect(openCodeProviderFailure({ name: "UnknownError", data: { message: "Plain failure." } }, "openrouter")) + .toMatchObject({ message: "Plain failure.", providerCode: "UnknownError", category: "unknown" }); + const invalid = openCodeProviderFailure({ name: "UnknownError", data: { message: "{\"code\":999}" } }, "openrouter"); + expect(invalid).toMatchObject({ message: "{\"code\":999}", providerCode: "UnknownError" }); + expect(invalid).not.toHaveProperty("httpStatus"); + }); + it("redacts credentials before truncation and preserves ordinary words and Unicode", () => { const secret = "fixture-unusual-secret"; const failure = createProviderFailure({ provider: "anthropic", phase: "startup", diff --git a/server/src/services/native-runtime/native-provider-failure.test.ts b/server/src/services/native-runtime/native-provider-failure.test.ts index 9406db76a27..ef1f17e40d4 100644 --- a/server/src/services/native-runtime/native-provider-failure.test.ts +++ b/server/src/services/native-runtime/native-provider-failure.test.ts @@ -197,6 +197,45 @@ describe("native provider failure observation", () => { expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGTERM" })).toEqual({ ...expected, signal: "SIGTERM" }); }); + it("records an OpenCode upstream provider failure's closed fields without its message or code", () => { + const observed = createNativeProviderFailureObservation({ kind: "opencode", model }); + const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn", + category: "unknown", message: "Private upstream explanation", providerCode: "private_provider_type", + httpStatus: 502, retryability: "unknown" }; + observed.observe({ ...event, payload: { status: "failed", error: { + name: "UnknownError", data: { message: "Private upstream body" }, providerFailure, + } } }); + expect(observed.forTerminal("turn", terminal)).toBeNull(); + expect(observed.providerFailureForTerminal("turn", terminal)).toEqual(providerFailure); + const diagnostic = observed.unclassifiedTurnFailure("turn", terminal); + expect(diagnostic).toEqual({ + provider: "opencode", category: "turn_failed", + providerFailureCategory: "unknown", providerFailureRetryability: "unknown", httpStatus: 502, + }); + expect(JSON.stringify(diagnostic)).not.toMatch(/private/i); + expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic); + // A malformed provider failure adds nothing, and a persisted record keeps only known values. + const malformed = createNativeProviderFailureObservation({ kind: "opencode", model }); + malformed.observe({ ...event, payload: { status: "failed", error: { providerFailure: { ...providerFailure, category: "private" } } } }); + expect(malformed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" }); + expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "turn_failed", + providerFailureCategory: "private", providerFailureRetryability: "unknown", httpStatus: 502 })) + .toEqual({ provider: "opencode", category: "turn_failed" }); + expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "turn_failed", + providerFailureCategory: "overloaded", providerFailureRetryability: "private", httpStatus: 42 })) + .toEqual({ provider: "opencode", category: "turn_failed", providerFailureCategory: "overloaded" }); + }); + + it("names the OpenCode empty-response failure the driver writes", () => { + const observed = createNativeProviderFailureObservation({ kind: "opencode", model }); + observed.observe({ ...event, payload: { status: "failed", error: { + code: "opencode_empty_response", message: "OpenCode ended the turn without a final response.", retryable: false, + } } }); + const diagnostic = observed.unclassifiedTurnFailure("turn", terminal); + expect(diagnostic).toEqual({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" }); + expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic); + }); + it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])( "does not apply a rejection to another selected provider/model (%j)", (provider) => { const observed = createNativeProviderFailureObservation(provider); diff --git a/server/src/services/native-runtime/native-provider-failure.ts b/server/src/services/native-runtime/native-provider-failure.ts index ef858593b97..de2ef11c21c 100644 --- a/server/src/services/native-runtime/native-provider-failure.ts +++ b/server/src/services/native-runtime/native-provider-failure.ts @@ -79,6 +79,14 @@ export interface NativeCodexTurnFailureDiagnostic { */ const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet = new Set([ "opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing", + "opencode_empty_response", +]); +const PROVIDER_FAILURE_CATEGORIES: ReadonlySet = new Set([ + "rate_limit", "quota_exhausted", "authentication", "permission", "model_unavailable", "invalid_request", + "overloaded", "timeout", "unknown", +]); +const PROVIDER_FAILURE_RETRYABILITY: ReadonlySet = new Set([ + "transient", "requires_action", "unknown", ]); const OPENCODE_STREAM_FAILURE_KINDS: ReadonlySet = new Set([ "server_exited", "connection_failed", "http_error", "stream_closed", "stream_error", @@ -107,7 +115,11 @@ export interface NativeOpenCodeTurnFailureDiagnostic { exitCode?: number; signal?: string; connectionError?: string; + /** The stream failure's HTTP status, or else the provider error's status. */ httpStatus?: number; + /** The provider error's category and retryability, when the turn carried one. */ + providerFailureCategory?: ProviderFailure["category"]; + providerFailureRetryability?: ProviderFailure["retryability"]; } export type NativeTurnFailureDiagnostic = NativeCodexTurnFailureDiagnostic | NativeOpenCodeTurnFailureDiagnostic; @@ -136,11 +148,27 @@ function openCodeStreamFailureFields(value: unknown): Omit { + if (typeof category !== "string" || !PROVIDER_FAILURE_CATEGORIES.has(category)) return {}; + return { + providerFailureCategory: category as ProviderFailure["category"], + ...(typeof retryability === "string" && PROVIDER_FAILURE_RETRYABILITY.has(retryability) + ? { providerFailureRetryability: retryability as ProviderFailure["retryability"] } : {}), + ...httpStatusField(httpStatus), + }; +} + function openCodeTurnFailure(failure: Record): NativeOpenCodeTurnFailureDiagnostic { + const providerFailure = readProviderFailure(failure.providerFailure); return { provider: "opencode", category: "turn_failed", ...runnerFailureCodeField(failure.code, OPENCODE_TURN_FAILURE_RUNNER_CODES), + ...(providerFailure ? openCodeProviderFailureFields( + providerFailure.category, providerFailure.retryability, providerFailure.httpStatus) : {}), ...openCodeStreamFailureFields(failure.opencodeFailure), }; } @@ -164,6 +192,7 @@ export function readNativeTurnFailureDiagnostic(value: unknown): NativeTurnFailu provider: "opencode", category: "turn_failed", ...runnerFailureCodeField(entry.runnerFailureCode, OPENCODE_TURN_FAILURE_RUNNER_CODES), + ...openCodeProviderFailureFields(entry.providerFailureCategory, entry.providerFailureRetryability, entry.httpStatus), ...openCodeStreamFailureFields(entry), }; }