From 11d777b80f019bd4d162eb0fe6871e2c313a226d Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Wed, 18 Nov 2026 18:05:52 -0600 Subject: [PATCH] feat(server): fit PostHog proxy limits to busy Cloud stacks and make them configurable (#22243) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - On Paperclip Cloud, the board UI sends browser analytics to PostHog through a same-origin proxy at `/_ph` > - The proxy limits requests in flight: 16 per client address and 256 in total > - Each open tab runs posthog-js, and one tab can have a few requests in flight at once > - On Railway without `TRUST_PROXY`, `req.ip` is the private edge hop, so all visitors share one 16-request limit > - This pull request raises the limits, makes them configurable, and stops the per-client limit when `req.ip` is a shared private hop > - The benefit is that busy stacks with many users and many tabs do not lose analytics to 429 answers, and ops can see when a limit is hit ## Linked Issues or Issue Description **Subsystem affected** Server: the PostHog reverse proxy for Cloud browser analytics (`server/src/routes/posthog-proxy.ts`). **Problem or motivation** The proxy limits are too low for busy Cloud stacks. One user with many tabs can use 16 requests in flight. Behind the Railway edge without `TRUST_PROXY`, `req.ip` is a private `100.64.x.x` hop, so all visitors share one 16-request limit. The proxy then answers 429 and posthog-js drops or delays events and replay uploads. Ops cannot change the limits and cannot see when the proxy refuses requests. **Proposed solution** Raise the defaults to 128 per public client address and 1024 in total. Read both limits from two environment variables with a strict parse and a ceiling. When `req.ip` is not a public address, apply only the total limit. Give each request a 60 second deadline. Log a warning at most once a minute when the proxy refuses requests at a limit. **Alternatives considered** Find the visitor for the per-client limit from `X-Forwarded-For`. A client can write that header, and an earlier security review asked the proxy to keep `req.ip` as the key, so this change does not do that. Scale the per-client limit for a shared hop. All visitors share the hop, so a scaled limit is only a second total limit. The proxy applies the total limit alone instead. **Additional context** Fleet operators can set the variables with an `env_config` campaign. `doc/cloud-browser-analytics.md` tells how. Refs #22012 (the proxy) and #22052 (visitor address behind a private hop). ## What Changed - Raised `DEFAULT_POSTHOG_PROXY_LIMITS` to 128 requests in flight per client address and 1024 in total. The 64 MiB body limit did not change. - Added `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` and `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT`. `resolvePostHogProxyLimits` reads them where `resolvePostHogProxyMount` builds the proxy. A value must be a whole number from 1 to 65536. An invalid value logs a warning and uses the default. The server does not stop. - Added `posthogProxyClientKey`. It returns `req.ip` only when it is a public address. For a private, carrier-grade NAT, loopback, link-local or other reserved address, or no address, the proxy applies only the total limit. The key never comes from a client header. - Added `createPostHogProxyLimitReporter`. When the proxy answers 429, it logs `PostHog proxy refused requests at its in-flight limit` at most once a minute. The fields are the refusal counts per limit since the last warning, the requests in flight, the limits and the time since the last warning. The warning has no addresses. It uses no timer: it checks the time when a refusal occurs. - Added a request deadline of 60 seconds from the time the proxy accepts a request. At the deadline, the proxy stops the request to PostHog and closes the browser response, also when PostHog already finished and the browser does not read the response. The slot is then free again. Before, the 30 second upstream timeout was an idle timeout, so a slow body that sent a byte at a time could hold a slot until the server request timeout (300 seconds). - The mount object now also returns the limits in use. - Updated `doc/cloud-browser-analytics.md`: the limits, why these defaults, the variables, the shared-hop behavior, the warning fields and how to tune a fleet with an `env_config` campaign. Why these defaults: - 128 per address covers about 10 people behind one office network, each with 4 tabs and 3 requests in flight per tab, with space left over. - 1024 in total keeps memory bounded. The proxy streams bodies, so a request holds two sockets and the stream buffers between them, not the full body. In a local test on this branch, 1024 held requests added about 96 MiB of resident memory with 4 KiB bodies and about 173 MiB with 256 KiB bodies (256 held requests with 4 KiB bodies: about 40 MiB). These numbers include the test client and the test upstream in the same process, so the proxy share is smaller. Operators can lower the limit for small stacks. ## Verification - `cd server && npx vitest run src/__tests__/cloud-posthog-browser.test.ts`: 87 tests pass (5 runs in a row). - New tests: - Defaults, values from the environment, an empty value, and invalid values (`0`, `-5`, `1.5`, `1e3`, `12abc`, `abc`, `0x10`, `+8`, `65537`, `1024000`, a value larger than a safe integer). `65536` is accepted. Each invalid value keeps the default and warns once per variable. - The mount passes the configured limits to the proxy. - Per-client limit for a public address (with `trust proxy` set): a second request from the same address gets 429, a request from another address gets 200. - Shared private hop (no trusted proxy, peer is loopback): the per-client limit does not apply, and the total limit still answers 429. - A forged `X-Forwarded-For` does not select the bucket when no proxy is trusted: three held requests with the same forged public address and a per-client limit of 1 all get 200. - Request deadline: an upstream that does not answer gives 502 at the deadline and frees the slot, a body that trickles in is cut off at the deadline, and a 16 MiB response that the browser does not read is closed at the deadline. - The tests wait until the held upstream receives each request, not for a fixed time. - Total limit across public addresses, and the reported reason (`total` or `per_client`) for each 429. - The warning rate limit: one warning for the first refusal, none in the next 60 seconds, then one warning with the counts of the refusals in between, and no addresses in any warning. - Red/green: with the shared-hop rule removed (`posthogProxyClientKey` returns `req.ip`), 9 of the new tests fail. With the deadline removed, the 2 deadline tests fail. With both, all pass. - `cd server && npx tsc --noEmit -p tsconfig.json`: passes. - `pnpm verify`: exit 2, because `doc/cloud-browser-analytics.md` is a global CI input. Required CI supplies the broad coverage. - Memory numbers come from a local script that held 1024 requests through the proxy against a local upstream (Node 26, macOS). ## Risks - Higher limits let more requests use memory and sockets at once. The worst case is bounded by the total limit, and operators can lower it with `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT`. - Behind a shared private hop, one client can use all of the total limit, and other visitors then get 429. The request deadline frees each slot after at most 60 seconds, but a client that sends again can keep the limit full. Before this change, the same visitors shared a limit of 16, which one client could also fill. A separate limit for the hop would not help the visitors behind it, because one client that fills it blocks all of them. Setting `TRUST_PROXY` brings back the per-client limit. - A PostHog request that takes longer than 60 seconds now fails with 502. posthog-js bodies are small and PostHog answers in well under a second, so this does not affect normal traffic. - If an operator sets `TRUST_PROXY` to trust every hop, `req.ip` comes from the client's `X-Forwarded-For`. This was already true before this change. The total limit still applies. - The proxy is mounted only on Cloud with browser analytics on. Self-hosted installs do not change. > For core feature work, check [`ROADMAP.md`](ROADMAP.md) first and discuss it in `#dev` before opening the PR. Feature PRs that overlap with planned core work may need to be redirected — check the roadmap first. See `CONTRIBUTING.md`. ## Model Used - Claude Opus 5.5 (claude-opus-5-5), Anthropic, used through Claude Code with tool use (shell, file edits, test runs). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/cloud-browser-analytics.md | 84 ++++- .../__tests__/cloud-posthog-browser.test.ts | 312 +++++++++++++++++- server/src/routes/posthog-proxy.ts | 120 ++++++- 3 files changed, 489 insertions(+), 27 deletions(-) diff --git a/doc/cloud-browser-analytics.md b/doc/cloud-browser-analytics.md index 70f09f0b63b..7d34cf3e94b 100644 --- a/doc/cloud-browser-analytics.md +++ b/doc/cloud-browser-analytics.md @@ -31,6 +31,8 @@ extensions posthog-js loads at run time (recorder, web vitals). | `PAPERCLIP_POSTHOG_ENVIRONMENT` | — | `staging` or `production`. Added to every event as `environment`. | | `PAPERCLIP_POSTHOG_BROWSER_ENABLED` | off | `1` turns browser analytics on for this stack. Leave it unset to keep it off. | | `PAPERCLIP_POSTHOG_REPLAY_ENABLED` | on (when browser analytics is on) | `0` turns session replay off for this stack and keeps the other browser analytics. | +| `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` | `1024` | Proxy requests in flight for the whole server. See [Proxy limits](#proxy-limits). | +| `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT` | `128` | Proxy requests in flight from one public client address. See [Proxy limits](#proxy-limits). | | `PAPERCLIP_POSTHOG_COUNTRY_HEADER` | — | Name of a request header that the platform edge sets to the visitor's two-letter country code (for example `CF-IPCountry` behind a proxied Cloudflare zone). Leave it unset unless the edge sets the header and drops a value from the client. See [Consent](#consent). | ### Per-stack opt-out @@ -126,19 +128,91 @@ The ingest host is `PAPERCLIP_POSTHOG_HOST`. The assets host is derived from it answers 404 without contacting PostHog. The proxy streams bodies, builds a new header set from an allowlist plus `X-Forwarded-For`, and never forwards the inbound Host, Cookie or Authorization headers. It drops `Set-Cookie` from -responses. It refuses bodies over 64 MiB (413) and limits requests in -flight to 16 per client address and 256 in total (429). It is mounted only when +responses. It refuses bodies over 64 MiB (413) and limits the requests in +flight (429, see [Proxy limits](#proxy-limits)). It is mounted only when browser analytics is on. It does not require a session: posthog-js sends some events while a page unloads, and the proxy can only reach the two PostHog hosts with the public project token, like PostHog's own ingest endpoint. -### Visitor address behind the platform proxy +### Proxy limits + +Each open tab runs posthog-js. One tab sends event batches, session replay +uploads and remote settings requests, so it can have a few requests in flight +at once, and more when PostHog is slow (the proxy waits up to 30 seconds for +an answer). The proxy has two limits. When a request is over a limit, the proxy +answers 429 with `Retry-After: 1`, and posthog-js sends it again later. + +| Limit | Default | Variable | +| --- | --- | --- | +| Requests in flight for the whole server | 1024 | `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` | +| Requests in flight from one public client address | 128 | `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT` | + +Why these values: + +- 128 per address covers about 10 people behind one office network, each with + 4 tabs and 3 requests in flight per tab, with space left over. +- 1024 in total keeps memory use bounded. The proxy streams bodies, so a + request holds two sockets and the stream buffers between them, not the full + body. In a local test, 1024 held requests added about 96 MiB of resident + memory with 4 KiB bodies and about 173 MiB with 256 KiB bodies. These + numbers include the test client and the test upstream, which ran in the same + process. Most PostHog bodies are a few KiB. + +A value must be a whole number from 1 to 65536 (for example `2048`). The +server reads the values at start. If a value is not valid, the server logs a +warning, uses the default, and starts as usual. + +Each request has a deadline of 60 seconds from the time the proxy accepts it, +whether or not data still moves. At the deadline, the proxy stops the request +to PostHog and closes the browser request, so a slow body or a slow answer +cannot hold a slot for longer. This deadline is not configurable. The proxy +also stops a request to PostHog that has no data for 30 seconds. + +#### Shared platform hop The per-client limit uses `req.ip`, which comes from the server's trusted proxy settings (`TRUST_PROXY`). It never comes from a header the client controls. If `TRUST_PROXY` is unset behind a platform proxy, `req.ip` is the proxy's private -hop, so every visitor behind that hop shares one 16-request limit. Set -`TRUST_PROXY` to the platform's proxy range so that `req.ip` is the visitor. +hop (for example `100.64.x.x` on Railway). All visitors come from that one +address, so a per-client limit would hold back all of them together. + +When `req.ip` is not a public address (private, carrier-grade NAT, loopback, +link-local or another reserved range, or no address), the proxy applies only +the total limit. A separate limit for the shared hop would not protect the +visitors behind it: one client that fills it blocks all of them, the same as +with the total limit. The total limit and the request deadline bound the +memory and sockets that clients can use. It does not read `X-Forwarded-For` to find the visitor for a +limit, because a client can write that header. Set `TRUST_PROXY` to the +platform's proxy range so that `req.ip` is the visitor, and the per-client +limit applies again. + +#### When the limits are hit + +When the proxy refuses a request at a limit, it logs the warning +`PostHog proxy refused requests at its in-flight limit` at most once a minute. +The warning has these fields, and no addresses or other visitor data: + +- `refusedAtTotalLimit` and `refusedAtPerClientLimit`: the requests refused + at each limit since the last warning (the first warning counts only one). +- `inFlight`: the requests in flight when the warning was written. +- `maxInFlight` and `maxInFlightPerClient`: the limits in use. +- `sinceMs`: the time since the last warning, or `null` for the first one. + +#### Tuning a fleet + +Set the variables with a fleet `env_config` campaign, the same way as the +other `PAPERCLIP_POSTHOG_*` variables. Start with a canary stack. The server +reads the values at start, so a campaign that drains and restarts the stacks +applies them at once, and a campaign with `"applyOnNextDeploy": true` applies +them on the next deploy. To go back to the defaults, remove the variables. + +Raise `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` when the warning shows refusals +at the total limit and the stack has memory to spare. Raise +`PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT` when it shows refusals at +the per-client limit, for example for a customer with many people behind one +address. + +### Visitor location The `X-Forwarded-For` value that the proxy sends to PostHog carries the visitor's location. When the socket peer is private, the proxy uses the diff --git a/server/src/__tests__/cloud-posthog-browser.test.ts b/server/src/__tests__/cloud-posthog-browser.test.ts index 75a7ad903ad..d484d68ef73 100644 --- a/server/src/__tests__/cloud-posthog-browser.test.ts +++ b/server/src/__tests__/cloud-posthog-browser.test.ts @@ -2,7 +2,7 @@ import http from "node:http"; import type { AddressInfo } from "node:net"; import express from "express"; import request from "supertest"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { posthogAssetsOrigin, posthogUiHost, @@ -10,10 +10,15 @@ import { } from "../services/cloud-posthog-browser.js"; import { resolveCloudPostHogConfig } from "../services/cloud-posthog-config.js"; import { + createPostHogProxyLimitReporter, + DEFAULT_POSTHOG_PROXY_LIMITS, + posthogProxyClientKey, posthogProxyRoutes, posthogUpstreamUrl, posthogVisitorIp, + resolvePostHogProxyLimits, resolvePostHogProxyMount, + type PostHogProxyLimits, } from "../routes/posthog-proxy.js"; const CLOUD = { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-token", PAPERCLIP_CLOUD_STACK_ID: "stack-1" }; @@ -212,7 +217,7 @@ describe("PostHog proxy forwarding", () => { it("refuses a declared body over the limit without contacting PostHog", async () => { const app = express(); - app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256 })); + app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256, requestDeadlineMs: 60_000 })); const res = await request(app).post("/_ph/e/").set("Content-Type", "text/plain").send("x".repeat(11)); expect(res.status).toBe(413); expect(seen).toHaveLength(0); @@ -220,7 +225,7 @@ describe("PostHog proxy forwarding", () => { it("cuts off a streamed body that grows past the limit", async () => { const app = express(); - app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256 })); + app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256, requestDeadlineMs: 60_000 })); const server = app.listen(0, "127.0.0.1"); await new Promise((resolve) => server.once("listening", () => resolve())); const port = (server.address() as AddressInfo).port; @@ -242,29 +247,188 @@ describe("PostHog proxy forwarding", () => { } }); - it("limits requests in flight from one client", async () => { + async function withHeldUpstream( + limits: PostHogProxyLimits, + run: (send: (ip?: string) => Promise, release: () => void, arrived: (count: number) => Promise) => Promise, + options: { trustProxy?: boolean; reportLimit?: Parameters[2] } = {}, + ) { let release: () => void = () => {}; const held = new Promise((resolve) => { release = resolve; }); - const slow = http.createServer(async (_req, res) => { await held; res.end("ok"); }); + let arrivals = 0; + const waiters: Array<{ count: number; resolve: () => void }> = []; + const slow = http.createServer(async (_req, res) => { + arrivals += 1; + for (const waiter of waiters.filter((entry) => entry.count <= arrivals)) { + waiters.splice(waiters.indexOf(waiter), 1); + waiter.resolve(); + } + await held; + res.end("ok"); + }); + const arrived = (count: number) => + count <= arrivals ? Promise.resolve() : new Promise((resolve) => { waiters.push({ count, resolve }); }); await new Promise((resolve) => slow.listen(0, "127.0.0.1", resolve)); const app = express(); - app.use("/_ph", posthogProxyRoutes(`http://127.0.0.1:${(slow.address() as AddressInfo).port}`, { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 256 })); + if (options.trustProxy) app.set("trust proxy", "loopback"); + app.use("/_ph", posthogProxyRoutes(`http://127.0.0.1:${(slow.address() as AddressInfo).port}`, limits, options.reportLimit)); const server = app.listen(0, "127.0.0.1"); await new Promise((resolve) => server.once("listening", () => resolve())); + const send = (ip?: string) => { + const req = request(server).get("/_ph/static/array.js"); + if (ip) req.set("X-Forwarded-For", ip); + return req.then((res) => res.status); + }; try { - const first = request(server).get("/_ph/static/array.js").then((res) => res.status); - await new Promise((resolve) => setTimeout(resolve, 50)); - const second = await request(server).get("/_ph/static/array.js"); - expect(second.status).toBe(429); - release(); - expect(await first).toBe(200); - // The slot frees up once the first request finishes. - expect((await request(server).get("/_ph/static/array.js")).status).toBe(200); + await run(send, release, arrived); } finally { release(); await new Promise((resolve) => server.close(() => resolve())); await new Promise((resolve) => slow.close(() => resolve())); } + } + + it("limits requests in flight from one public client address", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 256, requestDeadlineMs: 60_000 }; + await withHeldUpstream(limits, async (send, release, arrived) => { + const first = send("8.8.8.8"); + await arrived(1); + expect(await send("8.8.8.8")).toBe(429); + const other = send("9.9.9.9"); + await arrived(2); + release(); + expect(await first).toBe(200); + expect(await other).toBe(200); + expect(await send("8.8.8.8")).toBe(200); + }, { trustProxy: true }); + }); + + it("applies only the total limit when the client address is a shared private hop", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 3, requestDeadlineMs: 60_000 }; + await withHeldUpstream(limits, async (send, release, arrived) => { + const held = [send(), send(), send()]; + await arrived(3); + expect(await send()).toBe(429); + release(); + expect(await Promise.all(held)).toEqual([200, 200, 200]); + expect(await send()).toBe(200); + }); + }); + + it("does not key the per-client limit on a forged X-Forwarded-For when no proxy is trusted", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 8, requestDeadlineMs: 60_000 }; + await withHeldUpstream(limits, async (send, release, arrived) => { + const held = [send("8.8.8.8"), send("8.8.8.8"), send("8.8.8.8")]; + await arrived(3); + release(); + expect(await Promise.all(held)).toEqual([200, 200, 200]); + }); + }); + + it("frees a slot when PostHog holds a request past the request deadline", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 1, requestDeadlineMs: 200 }; + await withHeldUpstream(limits, async (send, _release, arrived) => { + const startedAt = Date.now(); + const first = send(); + await arrived(1); + expect(await send()).toBe(429); + expect(await first).toBe(502); + expect(Date.now() - startedAt).toBeLessThan(5_000); + const second = send(); + await arrived(2); + expect(await second).toBe(502); + }); + }); + + it("closes a response the browser does not read by the request deadline", async () => { + const big = Buffer.alloc(16 * 1024 * 1024, 120); + const finished: string[] = []; + const fast = http.createServer((_req, res) => { + res.writeHead(200, { "content-type": "application/javascript", "content-length": String(big.length) }); + res.end(big, () => finished.push("upstream")); + }); + await new Promise((resolve) => fast.listen(0, "127.0.0.1", resolve)); + const app = express(); + app.use("/_ph", posthogProxyRoutes(`http://127.0.0.1:${(fast.address() as AddressInfo).port}`, { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 1, requestDeadlineMs: 300 }, () => {})); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", () => resolve())); + const port = (server.address() as AddressInfo).port; + try { + let browserRequest: http.ClientRequest | undefined; + const headers = await new Promise((resolve) => { + browserRequest = http.get({ host: "127.0.0.1", port, path: "/_ph/static/recorder.js" }, (res) => { + res.on("error", () => {}); + res.pause(); + resolve(res.statusCode ?? 0); + }); + browserRequest.on("error", () => {}); + }); + expect(headers).toBe(200); + expect((await request(server).get("/_ph/static/array.js")).status).toBe(429); + await new Promise((resolve) => setTimeout(resolve, 1_000)); + expect(finished).toEqual(["upstream"]); + const next = await request(server).get("/_ph/static/array.js"); + browserRequest?.destroy(); + expect(next.status).toBe(200); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + fast.closeAllConnections(); + await new Promise((resolve) => fast.close(() => resolve())); + } + }); + + it("cuts off a body that trickles in past the request deadline", async () => { + const app = express(); + app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 1, requestDeadlineMs: 200 }, () => {})); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", () => resolve())); + const port = (server.address() as AddressInfo).port; + let drip: ReturnType | undefined; + try { + const startedAt = Date.now(); + const outcome = await new Promise((resolve) => { + const req = http.request({ host: "127.0.0.1", port, path: "/_ph/e/", method: "POST", headers: { "content-type": "text/plain", "transfer-encoding": "chunked" } }, (res) => { + res.on("error", () => {}); + res.resume(); + resolve(res.statusCode ?? 0); + }); + req.on("error", () => resolve("error")); + req.on("close", () => resolve("closed")); + drip = setInterval(() => req.write("x"), 20); + }); + expect([502, "error", "closed"]).toContain(outcome); + expect(Date.now() - startedAt).toBeLessThan(5_000); + expect((await request(server).get("/_ph/static/array.js")).status).not.toBe(429); + } finally { + clearInterval(drip); + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it("limits requests in flight in total across public clients and reports each refusal", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 2, requestDeadlineMs: 60_000 }; + const reported: Array<[string, number]> = []; + await withHeldUpstream(limits, async (send, release, arrived) => { + const held = [send("8.8.8.8"), send("9.9.9.9")]; + await arrived(2); + expect(await send("1.1.1.1")).toBe(429); + release(); + expect(await Promise.all(held)).toEqual([200, 200]); + }, { trustProxy: true, reportLimit: (reason, inFlight) => reported.push([reason, inFlight]) }); + expect(reported).toEqual([["total", 2]]); + }); + + it("reports a per-client refusal with its reason", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 8, requestDeadlineMs: 60_000 }; + const reported: Array<[string, number]> = []; + await withHeldUpstream(limits, async (send, release, arrived) => { + const first = send("8.8.8.8"); + await arrived(1); + expect(await send("8.8.8.8")).toBe(429); + release(); + expect(await first).toBe(200); + }, { trustProxy: true, reportLimit: (reason, inFlight) => reported.push([reason, inFlight]) }); + expect(reported).toEqual([["per_client", 1]]); }); it("never forwards the inbound Host header", async () => { @@ -344,3 +508,123 @@ describe("posthogVisitorIp", () => { expect(posthogVisitorIp("100.64.0.11", { "x-forwarded-for": "10.0.0.1, 100.64.0.3" })).toBe("100.64.0.11"); }); }); + +describe("PostHog proxy limits", () => { + const ON = { ...CLOUD, ...TOKEN, ...BROWSER }; + + it("fits many tabs per user and many users per stack by default", () => { + expect(DEFAULT_POSTHOG_PROXY_LIMITS).toEqual({ + maxBodyBytes: 64 * 1024 * 1024, + maxInFlightPerClient: 128, + maxInFlight: 1024, + requestDeadlineMs: 60_000, + }); + const warn = vi.fn(); + expect(resolvePostHogProxyLimits({}, warn)).toEqual(DEFAULT_POSTHOG_PROXY_LIMITS); + expect(warn).not.toHaveBeenCalled(); + }); + + it("reads both limits from the environment", () => { + const warn = vi.fn(); + const limits = resolvePostHogProxyLimits( + { + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: "65536", + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT: " 256 ", + }, + warn, + ); + expect(limits).toEqual({ ...DEFAULT_POSTHOG_PROXY_LIMITS, maxInFlight: 65536, maxInFlightPerClient: 256 }); + expect(warn).not.toHaveBeenCalled(); + }); + + it.each(["0", "-5", "1.5", "1e3", "12abc", "abc", "0x10", "+8", "65537", "1024000", "99999999999999999999"])( + "keeps the default and warns for the invalid value %s", + (value) => { + const warn = vi.fn(); + const limits = resolvePostHogProxyLimits( + { PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: value, PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT: value }, + warn, + ); + expect(limits).toEqual(DEFAULT_POSTHOG_PROXY_LIMITS); + expect(warn).toHaveBeenCalledTimes(2); + expect(warn.mock.calls.map(([fields]) => fields.variable)).toEqual([ + "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT", + "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT", + ]); + }, + ); + + it("treats an empty value as unset", () => { + const warn = vi.fn(); + expect(resolvePostHogProxyLimits({ PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: " " }, warn)).toEqual(DEFAULT_POSTHOG_PROXY_LIMITS); + expect(warn).not.toHaveBeenCalled(); + }); + + it("passes the configured limits to the mounted proxy", () => { + const mount = resolvePostHogProxyMount({ + ...ON, + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: "2048", + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT: "64", + }); + expect(mount?.limits).toEqual({ ...DEFAULT_POSTHOG_PROXY_LIMITS, maxInFlight: 2048, maxInFlightPerClient: 64 }); + expect(resolvePostHogProxyMount({ ...ON, PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: "lots" })?.limits).toEqual( + DEFAULT_POSTHOG_PROXY_LIMITS, + ); + }); + + it.each([ + ["8.8.8.8", "8.8.8.8"], + ["2001:4860:4860::8888", "2001:4860:4860::8888"], + ["100.64.0.11", null], + ["10.0.0.2", null], + ["127.0.0.1", null], + ["::1", null], + ["::ffff:127.0.0.1", null], + ["fd00::1", null], + [undefined, null], + ])("uses %s as a per-client bucket only when it is public", (ip, key) => { + expect(posthogProxyClientKey(ip)).toBe(key); + }); + + it("warns at most once a minute and counts the refusals in between", () => { + let now = 1_000; + const warn = vi.fn(); + const report = createPostHogProxyLimitReporter(DEFAULT_POSTHOG_PROXY_LIMITS, { now: () => now, warn }); + + report("total", 1024); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]![0]).toEqual({ + refusedAtTotalLimit: 1, + refusedAtPerClientLimit: 0, + inFlight: 1024, + maxInFlight: 1024, + maxInFlightPerClient: 128, + sinceMs: null, + }); + + for (let index = 0; index < 50; index += 1) { + now += 1_000; + report(index % 2 === 0 ? "per_client" : "total", 900); + } + expect(warn).toHaveBeenCalledTimes(1); + + now = 61_000; + report("per_client", 700); + expect(warn).toHaveBeenCalledTimes(2); + expect(warn.mock.calls[1]![0]).toMatchObject({ + refusedAtTotalLimit: 25, + refusedAtPerClientLimit: 26, + inFlight: 700, + sinceMs: 60_000, + }); + + now += 59_999; + report("total", 1024); + expect(warn).toHaveBeenCalledTimes(2); + now += 1; + report("total", 1024); + expect(warn).toHaveBeenCalledTimes(3); + expect(warn.mock.calls[2]![0]).toMatchObject({ refusedAtTotalLimit: 2, refusedAtPerClientLimit: 0 }); + expect(JSON.stringify(warn.mock.calls)).not.toMatch(/\d+\.\d+\.\d+\.\d+/); + }); +}); diff --git a/server/src/routes/posthog-proxy.ts b/server/src/routes/posthog-proxy.ts index b89c7006f77..bd4bb38979d 100644 --- a/server/src/routes/posthog-proxy.ts +++ b/server/src/routes/posthog-proxy.ts @@ -37,14 +37,100 @@ export type PostHogProxyLimits = { maxInFlightPerClient: number; /** Requests in flight for the whole proxy. */ maxInFlight: number; + requestDeadlineMs: number; }; export const DEFAULT_POSTHOG_PROXY_LIMITS: PostHogProxyLimits = { maxBodyBytes: 64 * 1024 * 1024, - maxInFlightPerClient: 16, - maxInFlight: 256, + maxInFlightPerClient: 128, + maxInFlight: 1024, + requestDeadlineMs: 60_000, }; +export const POSTHOG_PROXY_MAX_IN_FLIGHT_ENV = "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT"; +export const POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT_ENV = "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT"; + +const LIMIT_WARNING_INTERVAL_MS = 60_000; +const MAX_CONFIGURED_IN_FLIGHT = 65_536; + +type LimitWarning = (fields: Record, message: string) => void; + +function warnWithLogger(fields: Record, message: string) { + logger.warn(fields, message); +} + +function parseInFlightLimit( + env: CloudInstanceEnv, + name: string, + fallback: number, + warn: LimitWarning, +): number { + const raw = env[name]; + if (raw === undefined || raw.trim() === "") return fallback; + const value = raw.trim(); + const parsed = Number(value); + if (/^[1-9][0-9]*$/.test(value) && parsed <= MAX_CONFIGURED_IN_FLIGHT) return parsed; + warn( + { variable: name, defaultValue: fallback, maxValue: MAX_CONFIGURED_IN_FLIGHT }, + `${name} is not an integer from 1 to ${MAX_CONFIGURED_IN_FLIGHT}; the PostHog proxy uses the default`, + ); + return fallback; +} + +export function resolvePostHogProxyLimits( + env: CloudInstanceEnv = process.env, + warn: LimitWarning = warnWithLogger, +): PostHogProxyLimits { + return { + maxBodyBytes: DEFAULT_POSTHOG_PROXY_LIMITS.maxBodyBytes, + requestDeadlineMs: DEFAULT_POSTHOG_PROXY_LIMITS.requestDeadlineMs, + maxInFlight: parseInFlightLimit(env, POSTHOG_PROXY_MAX_IN_FLIGHT_ENV, DEFAULT_POSTHOG_PROXY_LIMITS.maxInFlight, warn), + maxInFlightPerClient: parseInFlightLimit( + env, + POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT_ENV, + DEFAULT_POSTHOG_PROXY_LIMITS.maxInFlightPerClient, + warn, + ), + }; +} + +export function posthogProxyClientKey(ip: string | undefined): string | null { + return ip && isPublicIpAddress(ip) ? ip : null; +} + +export type PostHogProxyLimitReason = "total" | "per_client"; + +export function createPostHogProxyLimitReporter( + limits: PostHogProxyLimits, + options: { now?: () => number; warn?: LimitWarning; intervalMs?: number } = {}, +) { + const now = options.now ?? Date.now; + const warn = options.warn ?? warnWithLogger; + const intervalMs = options.intervalMs ?? LIMIT_WARNING_INTERVAL_MS; + let lastWarnedAt: number | null = null; + const refused: Record = { total: 0, per_client: 0 }; + + return (reason: PostHogProxyLimitReason, inFlight: number) => { + refused[reason] += 1; + const at = now(); + if (lastWarnedAt !== null && at - lastWarnedAt < intervalMs) return; + warn( + { + refusedAtTotalLimit: refused.total, + refusedAtPerClientLimit: refused.per_client, + inFlight, + maxInFlight: limits.maxInFlight, + maxInFlightPerClient: limits.maxInFlightPerClient, + sinceMs: lastWarnedAt === null ? null : at - lastWarnedAt, + }, + "PostHog proxy refused requests at its in-flight limit", + ); + lastWarnedAt = at; + refused.total = 0; + refused.per_client = 0; + }; +} + const BODY_TOO_LARGE = "posthog_proxy_body_too_large"; type PostHogUpstream = "ingest" | "assets"; @@ -131,7 +217,11 @@ export function posthogForwardRequestHeaders( return forwarded; } -export function posthogProxyRoutes(ingestHost: string, limits: PostHogProxyLimits = DEFAULT_POSTHOG_PROXY_LIMITS) { +export function posthogProxyRoutes( + ingestHost: string, + limits: PostHogProxyLimits = DEFAULT_POSTHOG_PROXY_LIMITS, + reportLimit: (reason: PostHogProxyLimitReason, inFlight: number) => void = createPostHogProxyLimitReporter(limits), +) { const router = Router(); let inFlight = 0; const inFlightByClient = new Map(); @@ -158,17 +248,25 @@ export function posthogProxyRoutes(ingestHost: string, limits: PostHogProxyLimit } // Bound the work one browser, or all of them, can put on this process. - const client = req.ip ?? "unknown"; - const clientInFlight = inFlightByClient.get(client) ?? 0; - if (inFlight >= limits.maxInFlight || clientInFlight >= limits.maxInFlightPerClient) { + const client = posthogProxyClientKey(req.ip); + const clientInFlight = client === null ? 0 : inFlightByClient.get(client) ?? 0; + const refusedBy: PostHogProxyLimitReason | null = + inFlight >= limits.maxInFlight + ? "total" + : client !== null && clientInFlight >= limits.maxInFlightPerClient + ? "per_client" + : null; + if (refusedBy) { + reportLimit(refusedBy, inFlight); res.setHeader("Retry-After", "1"); res.status(429).json({ error: "too_many_requests" }); return; } inFlight += 1; - inFlightByClient.set(client, clientInFlight + 1); + if (client !== null) inFlightByClient.set(client, clientInFlight + 1); res.once("close", () => { inFlight -= 1; + if (client === null) return; const remaining = (inFlightByClient.get(client) ?? 1) - 1; if (remaining > 0) inFlightByClient.set(client, remaining); else inFlightByClient.delete(client); @@ -197,6 +295,11 @@ export function posthogProxyRoutes(ingestHost: string, limits: PostHogProxyLimit upstreamReq.on("timeout", () => { upstreamReq.destroy(new Error("posthog_proxy_timeout")); }); + const deadline = setTimeout(() => { + if (res.headersSent || upstreamReq.destroyed) res.destroy(); + upstreamReq.destroy(new Error("posthog_proxy_deadline")); + }, limits.requestDeadlineMs); + res.once("close", () => clearTimeout(deadline)); upstreamReq.on("error", (err) => { logger.debug({ err, path: target.pathname }, "PostHog proxy request failed"); if (!res.headersSent && bodyTooLarge) { @@ -246,5 +349,6 @@ export function resolvePostHogProxyMount(env: CloudInstanceEnv = process.env) { logger.warn("PAPERCLIP_POSTHOG_HOST is not a valid URL; the PostHog proxy is off"); return null; } - return { path: POSTHOG_PROXY_PATH, router: posthogProxyRoutes(config.host) }; + const limits = resolvePostHogProxyLimits(env); + return { path: POSTHOG_PROXY_PATH, router: posthogProxyRoutes(config.host, limits), limits }; }