mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-22 11:54:35 +01:00
fix(server): pass a Cloud portfolio rate limit through as 429
Paperclip Cloud answers the tenant portfolio read with 429 when it throttles a stack. The proxy turned that into a 502 and reported it to Sentry as a server fault (PAPERCLIP-BACKEND-70). Return 429 with Cloud's bounded Retry-After hint instead, log the bounded diagnostics, and keep the throttled answer out of the portfolio cache. Other upstream failures keep the 502 contract and crash report. Fixes PAPERCLIP-BACKEND-70 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
67652580ee
commit
2f319bdadd
2 files changed
+59
-1
No files matched your search
@@ -62,7 +62,38 @@ function jsonResponse(payload: unknown, status = 200) {
|
||||
}
|
||||
|
||||
describe("GET /api/cloud/stacks", () => {
|
||||
it.each([401, 403, 429, 503])("preserves the response contract for upstream HTTP %s without reading the body", async (status) => {
|
||||
it.each([
|
||||
["delta seconds", "7", "7"],
|
||||
["an HTTP date", new Date(Date.UTC(2026, 10, 10, 14, 25, 0)).toUTCString(), "34"],
|
||||
["an excessive delay", "86400", "300"],
|
||||
["a malformed hint", "soon", undefined],
|
||||
["no hint", null, undefined],
|
||||
])("passes an upstream 429 with %s through as a rate limit without reporting it", async (_case, hint, retryAfter) => {
|
||||
const upstream = jsonResponse({ secret: "private upstream payload" }, 429);
|
||||
if (hint !== null) upstream.headers.set("retry-after", hint);
|
||||
const json = vi.spyOn(upstream, "json");
|
||||
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(upstream);
|
||||
const capture = vi.spyOn(sentry, "captureException").mockImplementation(() => {});
|
||||
const warn = vi.spyOn(logger, "warn").mockImplementation(() => {});
|
||||
const app = createApp({ fetchImpl, now: () => Date.UTC(2026, 10, 10, 14, 24, 26, 209) });
|
||||
const response = await request(app).get("/api/cloud/stacks");
|
||||
expect(response.status).toBe(429);
|
||||
expect(response.body).toEqual({ error: "Paperclip Cloud portfolio is rate limited", code: "cloud_portfolio_rate_limited", details: { code: "cloud_portfolio_rate_limited" } });
|
||||
expect(response.headers["retry-after"]).toBe(retryAfter);
|
||||
expect(json).not.toHaveBeenCalled();
|
||||
expect(capture).not.toHaveBeenCalled();
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
{ cloudPortfolio: expect.objectContaining({ phase: "http_response", upstreamStatus: 429, networkCode: "unknown" }) },
|
||||
"Paperclip Cloud portfolio request rate limited",
|
||||
);
|
||||
expect(JSON.stringify(warn.mock.calls)).not.toContain("private");
|
||||
// The throttled answer is not cached: the next read asks Cloud again.
|
||||
fetchImpl.mockResolvedValueOnce(jsonResponse({ stacks: [] }));
|
||||
expect((await request(app).get("/api/cloud/stacks")).body).toEqual({ stacks: [] });
|
||||
expect(fetchImpl).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it.each([401, 403, 503])("preserves the response contract for upstream HTTP %s without reading the body", async (status) => {
|
||||
const upstream = jsonResponse({ secret: "private upstream payload" }, status);
|
||||
const json = vi.spyOn(upstream, "json");
|
||||
const fetchImpl = vi.fn<typeof fetch>().mockResolvedValue(upstream);
|
||||
|
||||
@@ -11,11 +11,24 @@ const DEFAULT_PORTFOLIO_CACHE_TTL_MS = 30_000;
|
||||
const CLOUD_PORTFOLIO_PATH = "/v1/tenant/portfolio";
|
||||
const RETRYABLE_FETCH_RESET_CODES = new Set(["ECONNRESET", "UND_ERR_SOCKET"]);
|
||||
|
||||
const MAX_RETRY_AFTER_SECONDS = 300;
|
||||
|
||||
type PortfolioCacheEntry = {
|
||||
expiresAt: number;
|
||||
payload: unknown;
|
||||
};
|
||||
|
||||
/** Cloud's Retry-After as bounded whole seconds, or null when absent or malformed. */
|
||||
function retryAfterSeconds(value: string | null, currentTime: number): number | null {
|
||||
const trimmed = value?.trim();
|
||||
if (!trimmed) return null;
|
||||
const seconds = /^\d+$/.test(trimmed)
|
||||
? Number(trimmed)
|
||||
: Math.ceil((Date.parse(trimmed) - currentTime) / 1000);
|
||||
if (!Number.isFinite(seconds)) return null;
|
||||
return Math.min(Math.max(seconds, 1), MAX_RETRY_AFTER_SECONDS);
|
||||
}
|
||||
|
||||
export function cloudRoutes(opts: {
|
||||
runtimeEnv?: CloudInstanceEnv;
|
||||
fetchImpl?: typeof fetch;
|
||||
@@ -112,6 +125,20 @@ export function cloudRoutes(opts: {
|
||||
}
|
||||
phase = "http_response";
|
||||
upstreamStatus = upstream.status;
|
||||
if (upstream.status === 429) {
|
||||
// Cloud is throttling portfolio reads. That is the control plane's
|
||||
// backpressure, not a fault here: pass it on with Cloud's retry hint
|
||||
// instead of answering 502 and reporting a crash.
|
||||
logger.warn(
|
||||
{ cloudPortfolio: portfolioError("upstream").diagnostics },
|
||||
"Paperclip Cloud portfolio request rate limited",
|
||||
);
|
||||
const retryAfter = retryAfterSeconds(upstream.headers.get("retry-after"), now());
|
||||
if (retryAfter !== null) res.setHeader("Retry-After", String(retryAfter));
|
||||
throw new HttpError(429, "Paperclip Cloud portfolio is rate limited", {
|
||||
code: "cloud_portfolio_rate_limited",
|
||||
});
|
||||
}
|
||||
if (!upstream.ok) {
|
||||
const error = portfolioError("upstream");
|
||||
logger.warn(
|
||||
|
||||
Reference in new issue
Block a user