fix(runner): report the upstream cause of OpenCode provider failures (#19166)

OpenCode reports some OpenRouter failures as an UnknownError whose message is the provider's JSON error body. The provider failure now reads its HTTP status, error type and message from that body, so the run shows the upstream message instead of raw JSON.

The OpenCode nativeTurnFailure diagnostic now carries the provider failure's category, retryability and HTTP status (closed values only; no provider text or code), and recognizes the driver's opencode_empty_response code. Run status, error codes and retries do not change.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

packages/paperclip-runner/src/drivers/opencode/provider-failure.ts

@@ -1,11 +1,35 @@
 import { createProviderFailure } from "../provider-failure.js";
 
+function record(value: unknown): Record<string, unknown> {
+  return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : {};
+}
+
+/**
+ * OpenCode reports some upstream failures as an `UnknownError` whose message is
+ * the provider's JSON error, for example OpenRouter's
+ * `{"code":502,"message":"...","metadata":{"error_type":"provider_unavailable"}}`.
+ * Read its status, type and message instead of presenting the raw body.
+ */
+function providerErrorEnvelope(message: string): { message?: string; httpStatus?: unknown; providerCode?: unknown } {
+  if (message.length > 16_000) return {};
+  let response: Record<string, unknown>;
+  try { response = record(JSON.parse(message)); } catch { return {}; }
+  const detail = Object.keys(record(response.error)).length ? record(response.error) : response;
+  return {
+    message: typeof detail.message === "string" && detail.message.trim() ? detail.message : undefined,
+    httpStatus: detail.code,
+    providerCode: record(detail.metadata).error_type,
+  };
+}
+
 export function openCodeProviderFailure(error: Record<string, unknown>, provider: string,
   environment: NodeJS.ProcessEnv = {}) {
   const data = error.data && typeof error.data === "object" ? error.data as Record<string, unknown> : {};
   const message = typeof data.message === "string" ? data.message : error.message;
   if (typeof message !== "string") return null;
-  return createProviderFailure({ provider, phase: "turn", message, httpStatus: data.statusCode,
-    providerCode: data.code ?? error.name, requestId: data.requestID,
+  const envelope = providerErrorEnvelope(message);
+  return createProviderFailure({ provider, phase: "turn", message: envelope.message ?? message,
+    httpStatus: data.statusCode ?? envelope.httpStatus,
+    providerCode: data.code ?? envelope.providerCode ?? error.name, requestId: data.requestID,
     category: error.name === "ProviderAuthError" ? "authentication" : undefined, environment });
 }

packages/paperclip-runner/src/drivers/provider-failure.test.ts

@@ -79,6 +79,24 @@ describe("provider failure boundary", () => {
     expect(JSON.stringify(failure)).not.toMatch(/private request|secret|responseHeaders/);
   });
 
+  it("reads the upstream status and type from an OpenRouter error OpenCode reports as UnknownError", () => {
+    const upstream = "Upstream error from Fireworks: Grammar processing took too much time.";
+    const body = { code: 502, message: upstream, metadata: { error_type: "provider_unavailable" } };
+    expect(openCodeProviderFailure({ name: "UnknownError", data: { message: JSON.stringify(body) } }, "openrouter"))
+      .toEqual({ schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn", category: "unknown",
+        message: upstream, providerCode: "provider_unavailable", httpStatus: 502, retryability: "unknown" });
+    // The wrapped `{ error: ... }` form, with OpenCode's own status code taking precedence.
+    expect(openCodeProviderFailure({ name: "UnknownError", data: { statusCode: 503,
+      message: JSON.stringify({ error: body }) } }, "openrouter"))
+      .toMatchObject({ httpStatus: 503, category: "overloaded", message: upstream, providerCode: "provider_unavailable" });
+    // Plain text, and JSON without a usable message or status, keep the original fields.
+    expect(openCodeProviderFailure({ name: "UnknownError", data: { message: "Plain failure." } }, "openrouter"))
+      .toMatchObject({ message: "Plain failure.", providerCode: "UnknownError", category: "unknown" });
+    const invalid = openCodeProviderFailure({ name: "UnknownError", data: { message: "{\"code\":999}" } }, "openrouter");
+    expect(invalid).toMatchObject({ message: "{\"code\":999}", providerCode: "UnknownError" });
+    expect(invalid).not.toHaveProperty("httpStatus");
+  });
+
   it("redacts credentials before truncation and preserves ordinary words and Unicode", () => {
     const secret = "fixture-unusual-secret";
     const failure = createProviderFailure({ provider: "anthropic", phase: "startup",

server/src/services/native-runtime/native-provider-failure.test.ts

@@ -197,6 +197,45 @@ describe("native provider failure observation", () => {
     expect(readNativeTurnFailureDiagnostic({ ...expected, signal: "SIGTERM" })).toEqual({ ...expected, signal: "SIGTERM" });
   });
 
+  it("records an OpenCode upstream provider failure's closed fields without its message or code", () => {
+    const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
+    const providerFailure = { schema: "paperclip.provider-failure.v1", provider: "openrouter", phase: "turn",
+      category: "unknown", message: "Private upstream explanation", providerCode: "private_provider_type",
+      httpStatus: 502, retryability: "unknown" };
+    observed.observe({ ...event, payload: { status: "failed", error: {
+      name: "UnknownError", data: { message: "Private upstream body" }, providerFailure,
+    } } });
+    expect(observed.forTerminal("turn", terminal)).toBeNull();
+    expect(observed.providerFailureForTerminal("turn", terminal)).toEqual(providerFailure);
+    const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
+    expect(diagnostic).toEqual({
+      provider: "opencode", category: "turn_failed",
+      providerFailureCategory: "unknown", providerFailureRetryability: "unknown", httpStatus: 502,
+    });
+    expect(JSON.stringify(diagnostic)).not.toMatch(/private/i);
+    expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
+    // A malformed provider failure adds nothing, and a persisted record keeps only known values.
+    const malformed = createNativeProviderFailureObservation({ kind: "opencode", model });
+    malformed.observe({ ...event, payload: { status: "failed", error: { providerFailure: { ...providerFailure, category: "private" } } } });
+    expect(malformed.unclassifiedTurnFailure("turn", terminal)).toEqual({ provider: "opencode", category: "turn_failed" });
+    expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "turn_failed",
+      providerFailureCategory: "private", providerFailureRetryability: "unknown", httpStatus: 502 }))
+      .toEqual({ provider: "opencode", category: "turn_failed" });
+    expect(readNativeTurnFailureDiagnostic({ provider: "opencode", category: "turn_failed",
+      providerFailureCategory: "overloaded", providerFailureRetryability: "private", httpStatus: 42 }))
+      .toEqual({ provider: "opencode", category: "turn_failed", providerFailureCategory: "overloaded" });
+  });
+
+  it("names the OpenCode empty-response failure the driver writes", () => {
+    const observed = createNativeProviderFailureObservation({ kind: "opencode", model });
+    observed.observe({ ...event, payload: { status: "failed", error: {
+      code: "opencode_empty_response", message: "OpenCode ended the turn without a final response.", retryable: false,
+    } } });
+    const diagnostic = observed.unclassifiedTurnFailure("turn", terminal);
+    expect(diagnostic).toEqual({ provider: "opencode", category: "turn_failed", runnerFailureCode: "opencode_empty_response" });
+    expect(readNativeTurnFailureDiagnostic(diagnostic)).toEqual(diagnostic);
+  });
+
   it.each([{ kind: "claude", model }, { kind: "codex" }, { kind: "codex", model: "other" }])(
     "does not apply a rejection to another selected provider/model (%j)", (provider) => {
       const observed = createNativeProviderFailureObservation(provider);

server/src/services/native-runtime/native-provider-failure.ts

@@ -79,6 +79,14 @@ export interface NativeCodexTurnFailureDiagnostic {
  */
 const OPENCODE_TURN_FAILURE_RUNNER_CODES: ReadonlySet<string> = new Set([
   "opencode_server_exited", "opencode_event_stream_lost", "provider_terminal_error_missing",
+  "opencode_empty_response",
+]);
+const PROVIDER_FAILURE_CATEGORIES: ReadonlySet<string> = new Set<ProviderFailure["category"]>([
+  "rate_limit", "quota_exhausted", "authentication", "permission", "model_unavailable", "invalid_request",
+  "overloaded", "timeout", "unknown",
+]);
+const PROVIDER_FAILURE_RETRYABILITY: ReadonlySet<string> = new Set<ProviderFailure["retryability"]>([
+  "transient", "requires_action", "unknown",
 ]);
 const OPENCODE_STREAM_FAILURE_KINDS: ReadonlySet<string> = new Set([
   "server_exited", "connection_failed", "http_error", "stream_closed", "stream_error",
@@ -107,7 +115,11 @@ export interface NativeOpenCodeTurnFailureDiagnostic {
   exitCode?: number;
   signal?: string;
   connectionError?: string;
+  /** The stream failure's HTTP status, or else the provider error's status. */
   httpStatus?: number;
+  /** The provider error's category and retryability, when the turn carried one. */
+  providerFailureCategory?: ProviderFailure["category"];
+  providerFailureRetryability?: ProviderFailure["retryability"];
 }
 
 export type NativeTurnFailureDiagnostic = NativeCodexTurnFailureDiagnostic | NativeOpenCodeTurnFailureDiagnostic;
@@ -136,11 +148,27 @@ function openCodeStreamFailureFields(value: unknown): Omit<NativeOpenCodeTurnFai
   };
 }
 
+/** Only the provider error's closed fields; its message and code stay out of diagnostics. */
+function openCodeProviderFailureFields(
+  category: unknown, retryability: unknown, httpStatus: unknown,
+): Pick<NativeOpenCodeTurnFailureDiagnostic, "providerFailureCategory" | "providerFailureRetryability" | "httpStatus"> {
+  if (typeof category !== "string" || !PROVIDER_FAILURE_CATEGORIES.has(category)) return {};
+  return {
+    providerFailureCategory: category as ProviderFailure["category"],
+    ...(typeof retryability === "string" && PROVIDER_FAILURE_RETRYABILITY.has(retryability)
+      ? { providerFailureRetryability: retryability as ProviderFailure["retryability"] } : {}),
+    ...httpStatusField(httpStatus),
+  };
+}
+
 function openCodeTurnFailure(failure: Record<string, unknown>): NativeOpenCodeTurnFailureDiagnostic {
+  const providerFailure = readProviderFailure(failure.providerFailure);
   return {
     provider: "opencode",
     category: "turn_failed",
     ...runnerFailureCodeField(failure.code, OPENCODE_TURN_FAILURE_RUNNER_CODES),
+    ...(providerFailure ? openCodeProviderFailureFields(
+      providerFailure.category, providerFailure.retryability, providerFailure.httpStatus) : {}),
     ...openCodeStreamFailureFields(failure.opencodeFailure),
   };
 }
@@ -164,6 +192,7 @@ export function readNativeTurnFailureDiagnostic(value: unknown): NativeTurnFailu
         provider: "opencode",
         category: "turn_failed",
         ...runnerFailureCodeField(entry.runnerFailureCode, OPENCODE_TURN_FAILURE_RUNNER_CODES),
+        ...openCodeProviderFailureFields(entry.providerFailureCategory, entry.providerFailureRetryability, entry.httpStatus),
         ...openCodeStreamFailureFields(entry),
       };
     }