From 2f319bdadd3bc6975e2adbf34baf38ef4100bb98 Mon Sep 17 00:00:00 2001 From: Paperclip Date: Sat, 14 Nov 2026 05:03:31 +0000 Subject: [PATCH] fix(server): pass a Cloud portfolio rate limit through as 429 Paperclip Cloud answers the tenant portfolio read with 429 when it throttles a stack. The proxy turned that into a 502 and reported it to Sentry as a server fault (PAPERCLIP-BACKEND-70). Return 429 with Cloud's bounded Retry-After hint instead, log the bounded diagnostics, and keep the throttled answer out of the portfolio cache. Other upstream failures keep the 502 contract and crash report. Fixes PAPERCLIP-BACKEND-70 Co-Authored-By: Claude Opus 5.5 --- server/src/__tests__/cloud-routes.test.ts | 33 ++++++++++++++++++++++- server/src/routes/cloud.ts | 27 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/server/src/__tests__/cloud-routes.test.ts b/server/src/__tests__/cloud-routes.test.ts index 9453453153e..d201b44c301 100644 --- a/server/src/__tests__/cloud-routes.test.ts +++ b/server/src/__tests__/cloud-routes.test.ts @@ -62,7 +62,38 @@ function jsonResponse(payload: unknown, status = 200) { } describe("GET /api/cloud/stacks", () => { - it.each([401, 403, 429, 503])("preserves the response contract for upstream HTTP %s without reading the body", async (status) => { + it.each([ + ["delta seconds", "7", "7"], + ["an HTTP date", new Date(Date.UTC(2026, 10, 10, 14, 25, 0)).toUTCString(), "34"], + ["an excessive delay", "86400", "300"], + ["a malformed hint", "soon", undefined], + ["no hint", null, undefined], + ])("passes an upstream 429 with %s through as a rate limit without reporting it", async (_case, hint, retryAfter) => { + const upstream = jsonResponse({ secret: "private upstream payload" }, 429); + if (hint !== null) upstream.headers.set("retry-after", hint); + const json = vi.spyOn(upstream, "json"); + const fetchImpl = vi.fn().mockResolvedValue(upstream); + const capture = vi.spyOn(sentry, "captureException").mockImplementation(() => {}); + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const app = createApp({ fetchImpl, now: () => Date.UTC(2026, 10, 10, 14, 24, 26, 209) }); + const response = await request(app).get("/api/cloud/stacks"); + expect(response.status).toBe(429); + expect(response.body).toEqual({ error: "Paperclip Cloud portfolio is rate limited", code: "cloud_portfolio_rate_limited", details: { code: "cloud_portfolio_rate_limited" } }); + expect(response.headers["retry-after"]).toBe(retryAfter); + expect(json).not.toHaveBeenCalled(); + expect(capture).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + { cloudPortfolio: expect.objectContaining({ phase: "http_response", upstreamStatus: 429, networkCode: "unknown" }) }, + "Paperclip Cloud portfolio request rate limited", + ); + expect(JSON.stringify(warn.mock.calls)).not.toContain("private"); + // The throttled answer is not cached: the next read asks Cloud again. + fetchImpl.mockResolvedValueOnce(jsonResponse({ stacks: [] })); + expect((await request(app).get("/api/cloud/stacks")).body).toEqual({ stacks: [] }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it.each([401, 403, 503])("preserves the response contract for upstream HTTP %s without reading the body", async (status) => { const upstream = jsonResponse({ secret: "private upstream payload" }, status); const json = vi.spyOn(upstream, "json"); const fetchImpl = vi.fn().mockResolvedValue(upstream); diff --git a/server/src/routes/cloud.ts b/server/src/routes/cloud.ts index 9e5b3246720..7ef28a948dc 100644 --- a/server/src/routes/cloud.ts +++ b/server/src/routes/cloud.ts @@ -11,11 +11,24 @@ const DEFAULT_PORTFOLIO_CACHE_TTL_MS = 30_000; const CLOUD_PORTFOLIO_PATH = "/v1/tenant/portfolio"; const RETRYABLE_FETCH_RESET_CODES = new Set(["ECONNRESET", "UND_ERR_SOCKET"]); +const MAX_RETRY_AFTER_SECONDS = 300; + type PortfolioCacheEntry = { expiresAt: number; payload: unknown; }; +/** Cloud's Retry-After as bounded whole seconds, or null when absent or malformed. */ +function retryAfterSeconds(value: string | null, currentTime: number): number | null { + const trimmed = value?.trim(); + if (!trimmed) return null; + const seconds = /^\d+$/.test(trimmed) + ? Number(trimmed) + : Math.ceil((Date.parse(trimmed) - currentTime) / 1000); + if (!Number.isFinite(seconds)) return null; + return Math.min(Math.max(seconds, 1), MAX_RETRY_AFTER_SECONDS); +} + export function cloudRoutes(opts: { runtimeEnv?: CloudInstanceEnv; fetchImpl?: typeof fetch; @@ -112,6 +125,20 @@ export function cloudRoutes(opts: { } phase = "http_response"; upstreamStatus = upstream.status; + if (upstream.status === 429) { + // Cloud is throttling portfolio reads. That is the control plane's + // backpressure, not a fault here: pass it on with Cloud's retry hint + // instead of answering 502 and reporting a crash. + logger.warn( + { cloudPortfolio: portfolioError("upstream").diagnostics }, + "Paperclip Cloud portfolio request rate limited", + ); + const retryAfter = retryAfterSeconds(upstream.headers.get("retry-after"), now()); + if (retryAfter !== null) res.setHeader("Retry-After", String(retryAfter)); + throw new HttpError(429, "Paperclip Cloud portfolio is rate limited", { + code: "cloud_portfolio_rate_limited", + }); + } if (!upstream.ok) { const error = portfolioError("upstream"); logger.warn(