diff --git a/server/src/__tests__/openrouter-catalog-error-handler.test.ts b/server/src/__tests__/openrouter-catalog-error-handler.test.ts new file mode 100644 index 00000000000..b8d5199bd34 --- /dev/null +++ b/server/src/__tests__/openrouter-catalog-error-handler.test.ts @@ -0,0 +1,51 @@ +import express from "express"; +import request from "supertest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { HttpError } from "../errors.js"; +import { errorHandler } from "../middleware/error-handler.js"; +import { listOpenRouterModels } from "../services/openrouter-models.js"; + +const captureExceptionMock = vi.hoisted(() => vi.fn()); + +vi.mock("../sentry.js", () => ({ captureException: captureExceptionMock })); +vi.mock("../telemetry.js", () => ({ getTelemetryClient: () => null })); + +function createApp() { + const app = express(); + app.get("/models", async (_req, res) => { res.json(await listOpenRouterModels(true)); }); + app.get("/other-outage", async () => { throw new HttpError(503, "Durable agent files require shared object storage"); }); + app.use(errorHandler); + return app; +} + +describe("OpenRouter catalog outages", () => { + beforeEach(() => captureExceptionMock.mockReset()); + afterEach(() => { vi.unstubAllGlobals(); vi.restoreAllMocks(); }); + + it("answers a stalled public catalog with retry guidance and no crash report", async () => { + // PAPERCLIP-BACKEND-8D: the catalog request's own 10s deadline used to + // reach this handler as a raw TimeoutError, answering 500 and paging Sentry. + const timeout = AbortSignal.timeout.bind(AbortSignal); + vi.spyOn(AbortSignal, "timeout").mockImplementation(() => timeout(5)); + vi.stubGlobal("fetch", vi.fn((_url: string, init: { signal: AbortSignal }) => new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(init.signal.reason)); + }))); + + const res = await request(createApp()).get("/models"); + + expect(res.status).toBe(503); + expect(res.body).toEqual({ + error: "Could not load OpenRouter models. Retry or enter a model ID manually.", + code: "openrouter_catalog_unavailable", + details: { code: "openrouter_catalog_unavailable" }, + }); + expect(captureExceptionMock).not.toHaveBeenCalled(); + }); + + it("still reports other server-side 503 failures", async () => { + const res = await request(createApp()).get("/other-outage"); + + expect(res.status).toBe(503); + expect(captureExceptionMock).toHaveBeenCalledTimes(1); + }); +}); diff --git a/server/src/middleware/error-handler.ts b/server/src/middleware/error-handler.ts index d374e5f37d1..e311dd4e377 100644 --- a/server/src/middleware/error-handler.ts +++ b/server/src/middleware/error-handler.ts @@ -20,6 +20,7 @@ import { import { recordResponsibleUserDenialOnActiveRun } from "../services/responsible-user-denial-run-outcomes.js"; import { isObjectJournalBusyError } from "../services/object-byte-journal.js"; import { isExpectedMcpConnectionFailure } from "../services/mcp-connection-failure.js"; +import { isOpenRouterCatalogUnavailable } from "../services/openrouter-models.js"; import { isEndedDbPoolError } from "../database-retry.js"; import { ToolGatewayHttpError } from "../services/tool-gateway.js"; @@ -214,7 +215,9 @@ export function errorHandler( }, reportableError, ); - reportCrash(reportableError, req, !isExpectedMcpConnectionFailure(err)); + // An unreachable public provider catalog still answers 503 with its + // retry guidance, but it is the provider's outage, not a server fault. + reportCrash(reportableError, req, !isExpectedMcpConnectionFailure(err) && !isOpenRouterCatalogUnavailable(err)); } const secretSensitiveServerError = err.status >= 500 && diff --git a/server/src/services/openrouter-models.test.ts b/server/src/services/openrouter-models.test.ts index cb47ee32236..68b18dacc84 100644 --- a/server/src/services/openrouter-models.test.ts +++ b/server/src/services/openrouter-models.test.ts @@ -27,3 +27,55 @@ it("allows retry after a public catalog failure", async () => { await expect(listOpenRouterModels()).rejects.toThrow("Retry or enter a model ID manually"); await expect(listOpenRouterModels()).resolves.toEqual([]); }); + +/** A fetch that only settles when its signal fires, as a stalled upstream does. */ +function stalledFetch() { + return vi.fn((_url: string, init: { signal: AbortSignal }) => new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(init.signal.reason)); + })); +} + +function shortCatalogDeadline() { + const timeout = AbortSignal.timeout.bind(AbortSignal); + return vi.spyOn(AbortSignal, "timeout").mockImplementation(() => timeout(5)); +} + +it("answers its own catalog deadline as an unavailable provider, not a raw TimeoutError", async () => { + const deadline = shortCatalogDeadline(); + vi.stubGlobal("fetch", stalledFetch()); + const { listOpenRouterModels, isOpenRouterCatalogUnavailable } = await import("./openrouter-models.js"); + const error = await listOpenRouterModels().catch((err: unknown) => err); + deadline.mockRestore(); + expect(error).toMatchObject({ status: 503, message: "Could not load OpenRouter models. Retry or enter a model ID manually.", + details: { code: "openrouter_catalog_unavailable" } }); + expect(isOpenRouterCatalogUnavailable(error)).toBe(true); +}); + +it.each([ + ["a DNS failure", () => Promise.reject(Object.assign(new TypeError("fetch failed"), { cause: Object.assign(new Error("getaddrinfo ENOTFOUND openrouter.ai"), { code: "ENOTFOUND" }) }))], + ["a rate limit", () => Promise.resolve({ ok: false, status: 429 })], + ["a provider server error", () => Promise.resolve({ ok: false, status: 503 })], +])("marks %s as an unavailable catalog", async (_label, response) => { + vi.stubGlobal("fetch", vi.fn(response)); + const { listOpenRouterModels, isOpenRouterCatalogUnavailable } = await import("./openrouter-models.js"); + const error = await listOpenRouterModels().catch((err: unknown) => err); + expect(error).toMatchObject({ status: 503, details: { code: "openrouter_catalog_unavailable" } }); + expect(isOpenRouterCatalogUnavailable(error)).toBe(true); +}); + +it("keeps contract failures and unrelated errors reportable", async () => { + const { HttpError } = await import("../errors.js"); + vi.stubGlobal("fetch", vi.fn() + .mockResolvedValueOnce({ ok: false, status: 404 }) + .mockResolvedValueOnce({ ok: true, json: async () => { throw new SyntaxError("Unexpected token <"); } }) + .mockResolvedValueOnce({ ok: true, json: async () => ({ data: null }) })); + const { listOpenRouterModels, isOpenRouterCatalogUnavailable } = await import("./openrouter-models.js"); + const notFound = await listOpenRouterModels().catch((err: unknown) => err); + expect(notFound).toMatchObject({ status: 502, details: { code: "openrouter_catalog_error", status: 404 } }); + expect(isOpenRouterCatalogUnavailable(notFound)).toBe(false); + const malformed = await listOpenRouterModels().catch((err: unknown) => err); + expect(malformed).toBeInstanceOf(SyntaxError); + expect(isOpenRouterCatalogUnavailable(malformed)).toBe(false); + await expect(listOpenRouterModels()).rejects.toThrow("OpenRouter returned an invalid model catalog."); + expect(isOpenRouterCatalogUnavailable(new HttpError(503, "Could not load OpenRouter models. Retry or enter a model ID manually.", { code: "openrouter_catalog_unavailable" }))).toBe(false); +}); diff --git a/server/src/services/openrouter-models.ts b/server/src/services/openrouter-models.ts index 0e1ae87be8c..e224c8e3d82 100644 --- a/server/src/services/openrouter-models.ts +++ b/server/src/services/openrouter-models.ts @@ -1,16 +1,54 @@ import type { AdapterModel } from "@paperclipai/adapter-utils"; +import { HttpError } from "../errors.js"; +import { classifyRemoteConnectionError } from "./remote-connection-failure.js"; + +const CATALOG_URL = "https://openrouter.ai/api/v1/models?sort=most-popular"; +const CATALOG_TIMEOUT_MS = 10_000; +const CATALOG_UNAVAILABLE = "Could not load OpenRouter models. Retry or enter a model ID manually."; let cached: { until: number; models: AdapterModel[] } | undefined; let pending: Promise | undefined; +// Local provenance: only errors this module created for an actual outage of +// the public catalog (its own deadline, a classified transport failure, or a +// rate-limit/server status). They affect crash reporting only. +const unavailable = new WeakSet(); + +function catalogUnavailable(): HttpError { + const error = new HttpError(503, CATALOG_UNAVAILABLE, { code: "openrouter_catalog_unavailable" }); + unavailable.add(error); + return error; +} + +/** The public OpenRouter catalog was unreachable; this is not a Paperclip fault. */ +export function isOpenRouterCatalogUnavailable(error: unknown): boolean { + return error instanceof HttpError && unavailable.has(error); +} + +/** Map a failure of the outbound catalog request; rethrow anything else unchanged. */ +function requestFailure(error: unknown, signal: AbortSignal): unknown { + return signal.aborted || classifyRemoteConnectionError(error) ? catalogUnavailable() : error; +} + /** OpenRouter's public catalog does not require access to anyone's credentials. */ export async function listOpenRouterModels(refresh = false): Promise { if (!refresh && cached && cached.until > Date.now()) return cached.models; if (pending) return pending; pending = (async () => { - const response = await fetch("https://openrouter.ai/api/v1/models?sort=most-popular", { signal: AbortSignal.timeout(10_000) }); - if (!response.ok) throw new Error("Could not load OpenRouter models. Retry or enter a model ID manually."); - const body = await response.json() as { data?: Array<{ id?: unknown; name?: unknown }> }; + const signal = AbortSignal.timeout(CATALOG_TIMEOUT_MS); + let body: { data?: Array<{ id?: unknown; name?: unknown }> }; + try { + const response = await fetch(CATALOG_URL, { signal }); + if (!response.ok) { + await response.body?.cancel().catch(() => {}); + if (response.status === 408 || response.status === 429 || response.status >= 500) throw catalogUnavailable(); + throw new HttpError(502, CATALOG_UNAVAILABLE, { code: "openrouter_catalog_error", status: response.status }); + } + body = await response.json() as typeof body; + } catch (error) { + if (error instanceof HttpError) throw error; + throw requestFailure(error, signal); + } if (!Array.isArray(body.data)) throw new Error("OpenRouter returned an invalid model catalog."); const models = body.data.flatMap(model => typeof model.id === "string" && model.id.includes("/") ? [{ id: `openrouter/${model.id}`, label: typeof model.name === "string" ? model.name : model.id }]