From 21f4427b78fc3fe287f9f5905108c80971caed06 Mon Sep 17 00:00:00 2001 From: paperclipbugbot Date: Sat, 21 Nov 2026 04:22:41 -0800 Subject: [PATCH] fix(server): pass a Cloud portfolio rate limit through as 429 (#22488) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - A hosted stack reads its tenant portfolio from Paperclip Cloud through `GET /api/cloud/stacks` > - Paperclip Cloud can answer that read with 429 when it throttles a stack > - The proxy changed every non-OK upstream status into a 502, and the error handler reports every 5xx to Sentry > - So Cloud backpressure showed up as a server crash on a staging stack (`upstreamStatus=429`, `phase=http_response`) > - This pull request passes an upstream 429 through as a 429 with a bounded `Retry-After` hint > - The benefit is a correct status for clients and no false crash reports for throttling ## Linked Issues or Issue Description Refs #15340 (bounded Cloud portfolio diagnostics), Refs #19033 (retry one Cloud portfolio connection reset). **What happened?** A hosted staging stack reported "Paperclip Cloud portfolio request failed" to Sentry. The diagnostics show `upstreamStatus=429` after 44 ms. Cloud throttled the read, and the proxy answered 502 and reported a crash. **What did you expect to happen?** The proxy tells the client that Cloud is rate limiting the read. It does not report a server fault. **Steps to reproduce** Make the Cloud tenant portfolio endpoint answer 429, then call `GET /api/cloud/stacks`. Before this change the route answers 502 `cloud_portfolio_upstream_error` and reports to Sentry. ## What Changed - `server/src/routes/cloud.ts`: an upstream 429 now gives 429 `cloud_portfolio_rate_limited`. - The route copies Cloud's `Retry-After` as whole seconds. It accepts delta seconds or an HTTP-date and clamps the value to 1–300 seconds. It omits the header when the value is missing or malformed. - The route logs the bounded diagnostics at warn level. It does not report the 429 to Sentry and does not cache the throttled answer. - 401, 403, 503 and the other failure phases keep the 502 contract and the crash report. - `server/src/__tests__/cloud-routes.test.ts`: new cases for the 429 status, the `Retry-After` forms, no Sentry report, and no cache entry. - `docs/api/cloud.md` (new, in the API Reference nav): documents `GET /api/cloud/stacks`, its error codes, and the 429 with its bounded `Retry-After`. - `server/src/routes/openapi.ts`: the `/api/cloud/stacks` entry now lists the 429 response. ## Verification - Red: with the old `cloud.ts`, `cloud-routes.test.ts` gave 5 failed (the new 429 cases) and 23 passed. - Green: `cd server && npx vitest run src/__tests__/cloud-routes.test.ts src/__tests__/sentry.test.ts src/__tests__/cloud-portfolio-error.test.ts src/__tests__/run-failure-sentry-real-sdk.test.ts` gave 68 passed and 18 skipped. - `npx tsc --noEmit -p server/tsconfig.json` shows no errors in the changed files. - After review fixes (6a8f146c55): `cd server && npx vitest run src/__tests__/cloud-routes.test.ts src/__tests__/openapi-routes.test.ts` gave 80 passed. ## Risks - Low risk. Only the 429 path changes. - The two UI consumers (`useCloudInviteUrl`, `CompanyAccess`) treat any error the same way, so the UI needs no change. ## Model Used - Anthropic Claude Opus 5.5 (`claude-opus-5-5`) in Claude Code, with tool use and code execution. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Paperclip Co-authored-by: Claude Opus 5.5 Co-authored-by: paperclipbugbot <356415228+paperclipbugbot@users.noreply.github.com> --- docs/api/cloud.md | 35 +++++++++++++++++++++++ docs/docs.json | 3 +- server/src/__tests__/cloud-routes.test.ts | 32 ++++++++++++++++++++- server/src/routes/cloud.ts | 23 +++++++++++++++ server/src/routes/openapi.ts | 1 + 5 files changed, 92 insertions(+), 2 deletions(-) create mode 100644 docs/api/cloud.md diff --git a/docs/api/cloud.md b/docs/api/cloud.md new file mode 100644 index 00000000000..c53e406845d --- /dev/null +++ b/docs/api/cloud.md @@ -0,0 +1,35 @@ +--- +title: Cloud Stacks +summary: List the Paperclip Cloud stacks of the signed-in Cloud tenant user +--- + +This endpoint is available only on a Paperclip Cloud stack. It requires a +trusted Cloud tenant session. Board API keys and agent keys cannot use it. + +## List Stacks + +``` +GET /api/cloud/stacks +``` + +The server reads the user's stack portfolio from Paperclip Cloud and returns +Cloud's JSON payload unchanged. Responses use `Cache-Control: no-store`. A +successful portfolio is cached on the server for 30 seconds per user. + +## Errors + +Error bodies have the usual `error` message and a `code`. + +| Status | Code | Meaning | +|--------|------|---------| +| 403 | `cloud_tenant_required` | The caller is not a trusted Cloud tenant user | +| 404 | — | This server is not a Paperclip Cloud stack | +| 429 | `cloud_portfolio_rate_limited` | Paperclip Cloud is rate limiting portfolio reads | +| 502 | `cloud_portfolio_upstream_error` | Paperclip Cloud failed or could not be reached | +| 502 | `cloud_portfolio_invalid_response` | Paperclip Cloud returned invalid JSON | +| 503 | `cloud_portfolio_not_configured` | The stack's Cloud connection is not configured | + +A `429` response includes `Retry-After` in whole seconds when Paperclip Cloud +supplied a valid hint. The value is between 1 and 300. Wait at least that long +before you try again. A `429` is not cached, so the next request asks Paperclip +Cloud again. diff --git a/docs/docs.json b/docs/docs.json index 5c818919af0..9136f75fa7f 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -132,7 +132,8 @@ "api/tool-gateway-tokens", "api/tool-connections", "api/activity", - "api/dashboard" + "api/dashboard", + "api/cloud" ] } ] diff --git a/server/src/__tests__/cloud-routes.test.ts b/server/src/__tests__/cloud-routes.test.ts index 9453453153e..75dbc1f9e11 100644 --- a/server/src/__tests__/cloud-routes.test.ts +++ b/server/src/__tests__/cloud-routes.test.ts @@ -62,7 +62,37 @@ function jsonResponse(payload: unknown, status = 200) { } describe("GET /api/cloud/stacks", () => { - it.each([401, 403, 429, 503])("preserves the response contract for upstream HTTP %s without reading the body", async (status) => { + it.each([ + ["delta seconds", "7", "7"], + ["an HTTP date", new Date(Date.UTC(2026, 10, 10, 14, 25, 0)).toUTCString(), "34"], + ["an excessive delay", "86400", "300"], + ["a malformed hint", "soon", undefined], + ["no hint", null, undefined], + ])("passes an upstream 429 with %s through as a rate limit without reporting it", async (_case, hint, retryAfter) => { + const upstream = jsonResponse({ secret: "private upstream payload" }, 429); + if (hint !== null) upstream.headers.set("retry-after", hint); + const json = vi.spyOn(upstream, "json"); + const fetchImpl = vi.fn().mockResolvedValue(upstream); + const capture = vi.spyOn(sentry, "captureException").mockImplementation(() => {}); + const warn = vi.spyOn(logger, "warn").mockImplementation(() => {}); + const app = createApp({ fetchImpl, now: () => Date.UTC(2026, 10, 10, 14, 24, 26, 209) }); + const response = await request(app).get("/api/cloud/stacks"); + expect(response.status).toBe(429); + expect(response.body).toEqual({ error: "Paperclip Cloud portfolio is rate limited", code: "cloud_portfolio_rate_limited", details: { code: "cloud_portfolio_rate_limited" } }); + expect(response.headers["retry-after"]).toBe(retryAfter); + expect(json).not.toHaveBeenCalled(); + expect(capture).not.toHaveBeenCalled(); + expect(warn).toHaveBeenCalledWith( + { cloudPortfolio: expect.objectContaining({ phase: "http_response", upstreamStatus: 429, networkCode: "unknown" }) }, + "Paperclip Cloud portfolio request rate limited", + ); + expect(JSON.stringify(warn.mock.calls)).not.toContain("private"); + fetchImpl.mockResolvedValueOnce(jsonResponse({ stacks: [] })); + expect((await request(app).get("/api/cloud/stacks")).body).toEqual({ stacks: [] }); + expect(fetchImpl).toHaveBeenCalledTimes(2); + }); + + it.each([401, 403, 503])("preserves the response contract for upstream HTTP %s without reading the body", async (status) => { const upstream = jsonResponse({ secret: "private upstream payload" }, status); const json = vi.spyOn(upstream, "json"); const fetchImpl = vi.fn().mockResolvedValue(upstream); diff --git a/server/src/routes/cloud.ts b/server/src/routes/cloud.ts index 9e5b3246720..3b930ccddd2 100644 --- a/server/src/routes/cloud.ts +++ b/server/src/routes/cloud.ts @@ -11,11 +11,23 @@ const DEFAULT_PORTFOLIO_CACHE_TTL_MS = 30_000; const CLOUD_PORTFOLIO_PATH = "/v1/tenant/portfolio"; const RETRYABLE_FETCH_RESET_CODES = new Set(["ECONNRESET", "UND_ERR_SOCKET"]); +const MAX_RETRY_AFTER_SECONDS = 300; + type PortfolioCacheEntry = { expiresAt: number; payload: unknown; }; +function retryAfterSeconds(value: string | null, currentTime: number): number | null { + const trimmed = value?.trim(); + if (!trimmed) return null; + const seconds = /^\d+$/.test(trimmed) + ? Number(trimmed) + : Math.ceil((Date.parse(trimmed) - currentTime) / 1000); + if (!Number.isFinite(seconds)) return null; + return Math.min(Math.max(seconds, 1), MAX_RETRY_AFTER_SECONDS); +} + export function cloudRoutes(opts: { runtimeEnv?: CloudInstanceEnv; fetchImpl?: typeof fetch; @@ -112,6 +124,17 @@ export function cloudRoutes(opts: { } phase = "http_response"; upstreamStatus = upstream.status; + if (upstream.status === 429) { + logger.warn( + { cloudPortfolio: portfolioError("upstream").diagnostics }, + "Paperclip Cloud portfolio request rate limited", + ); + const retryAfter = retryAfterSeconds(upstream.headers.get("retry-after"), now()); + if (retryAfter !== null) res.setHeader("Retry-After", String(retryAfter)); + throw new HttpError(429, "Paperclip Cloud portfolio is rate limited", { + code: "cloud_portfolio_rate_limited", + }); + } if (!upstream.ok) { const error = portfolioError("upstream"); logger.warn( diff --git a/server/src/routes/openapi.ts b/server/src/routes/openapi.ts index aac0d81e584..7bc9228eb3c 100644 --- a/server/src/routes/openapi.ts +++ b/server/src/routes/openapi.ts @@ -2112,6 +2112,7 @@ registry.registerPath({ 200: r.ok(), 403: r.forbidden, 404: r.notFound, + 429: { description: "Paperclip Cloud is rate limiting portfolio reads (code cloud_portfolio_rate_limited). Retry-After, when present, gives whole seconds from 1 to 300. Not cached." }, 502: r.serverError, 503: r.serverError, 500: r.serverError,