diff --git a/doc/cloud-browser-analytics.md b/doc/cloud-browser-analytics.md index 70f09f0b63b..7d34cf3e94b 100644 --- a/doc/cloud-browser-analytics.md +++ b/doc/cloud-browser-analytics.md @@ -31,6 +31,8 @@ extensions posthog-js loads at run time (recorder, web vitals). | `PAPERCLIP_POSTHOG_ENVIRONMENT` | — | `staging` or `production`. Added to every event as `environment`. | | `PAPERCLIP_POSTHOG_BROWSER_ENABLED` | off | `1` turns browser analytics on for this stack. Leave it unset to keep it off. | | `PAPERCLIP_POSTHOG_REPLAY_ENABLED` | on (when browser analytics is on) | `0` turns session replay off for this stack and keeps the other browser analytics. | +| `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` | `1024` | Proxy requests in flight for the whole server. See [Proxy limits](#proxy-limits). | +| `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT` | `128` | Proxy requests in flight from one public client address. See [Proxy limits](#proxy-limits). | | `PAPERCLIP_POSTHOG_COUNTRY_HEADER` | — | Name of a request header that the platform edge sets to the visitor's two-letter country code (for example `CF-IPCountry` behind a proxied Cloudflare zone). Leave it unset unless the edge sets the header and drops a value from the client. See [Consent](#consent). | ### Per-stack opt-out @@ -126,19 +128,91 @@ The ingest host is `PAPERCLIP_POSTHOG_HOST`. The assets host is derived from it answers 404 without contacting PostHog. The proxy streams bodies, builds a new header set from an allowlist plus `X-Forwarded-For`, and never forwards the inbound Host, Cookie or Authorization headers. It drops `Set-Cookie` from -responses. It refuses bodies over 64 MiB (413) and limits requests in -flight to 16 per client address and 256 in total (429). It is mounted only when +responses. It refuses bodies over 64 MiB (413) and limits the requests in +flight (429, see [Proxy limits](#proxy-limits)). It is mounted only when browser analytics is on. It does not require a session: posthog-js sends some events while a page unloads, and the proxy can only reach the two PostHog hosts with the public project token, like PostHog's own ingest endpoint. -### Visitor address behind the platform proxy +### Proxy limits + +Each open tab runs posthog-js. One tab sends event batches, session replay +uploads and remote settings requests, so it can have a few requests in flight +at once, and more when PostHog is slow (the proxy waits up to 30 seconds for +an answer). The proxy has two limits. When a request is over a limit, the proxy +answers 429 with `Retry-After: 1`, and posthog-js sends it again later. + +| Limit | Default | Variable | +| --- | --- | --- | +| Requests in flight for the whole server | 1024 | `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` | +| Requests in flight from one public client address | 128 | `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT` | + +Why these values: + +- 128 per address covers about 10 people behind one office network, each with + 4 tabs and 3 requests in flight per tab, with space left over. +- 1024 in total keeps memory use bounded. The proxy streams bodies, so a + request holds two sockets and the stream buffers between them, not the full + body. In a local test, 1024 held requests added about 96 MiB of resident + memory with 4 KiB bodies and about 173 MiB with 256 KiB bodies. These + numbers include the test client and the test upstream, which ran in the same + process. Most PostHog bodies are a few KiB. + +A value must be a whole number from 1 to 65536 (for example `2048`). The +server reads the values at start. If a value is not valid, the server logs a +warning, uses the default, and starts as usual. + +Each request has a deadline of 60 seconds from the time the proxy accepts it, +whether or not data still moves. At the deadline, the proxy stops the request +to PostHog and closes the browser request, so a slow body or a slow answer +cannot hold a slot for longer. This deadline is not configurable. The proxy +also stops a request to PostHog that has no data for 30 seconds. + +#### Shared platform hop The per-client limit uses `req.ip`, which comes from the server's trusted proxy settings (`TRUST_PROXY`). It never comes from a header the client controls. If `TRUST_PROXY` is unset behind a platform proxy, `req.ip` is the proxy's private -hop, so every visitor behind that hop shares one 16-request limit. Set -`TRUST_PROXY` to the platform's proxy range so that `req.ip` is the visitor. +hop (for example `100.64.x.x` on Railway). All visitors come from that one +address, so a per-client limit would hold back all of them together. + +When `req.ip` is not a public address (private, carrier-grade NAT, loopback, +link-local or another reserved range, or no address), the proxy applies only +the total limit. A separate limit for the shared hop would not protect the +visitors behind it: one client that fills it blocks all of them, the same as +with the total limit. The total limit and the request deadline bound the +memory and sockets that clients can use. It does not read `X-Forwarded-For` to find the visitor for a +limit, because a client can write that header. Set `TRUST_PROXY` to the +platform's proxy range so that `req.ip` is the visitor, and the per-client +limit applies again. + +#### When the limits are hit + +When the proxy refuses a request at a limit, it logs the warning +`PostHog proxy refused requests at its in-flight limit` at most once a minute. +The warning has these fields, and no addresses or other visitor data: + +- `refusedAtTotalLimit` and `refusedAtPerClientLimit`: the requests refused + at each limit since the last warning (the first warning counts only one). +- `inFlight`: the requests in flight when the warning was written. +- `maxInFlight` and `maxInFlightPerClient`: the limits in use. +- `sinceMs`: the time since the last warning, or `null` for the first one. + +#### Tuning a fleet + +Set the variables with a fleet `env_config` campaign, the same way as the +other `PAPERCLIP_POSTHOG_*` variables. Start with a canary stack. The server +reads the values at start, so a campaign that drains and restarts the stacks +applies them at once, and a campaign with `"applyOnNextDeploy": true` applies +them on the next deploy. To go back to the defaults, remove the variables. + +Raise `PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT` when the warning shows refusals +at the total limit and the stack has memory to spare. Raise +`PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT` when it shows refusals at +the per-client limit, for example for a customer with many people behind one +address. + +### Visitor location The `X-Forwarded-For` value that the proxy sends to PostHog carries the visitor's location. When the socket peer is private, the proxy uses the diff --git a/server/src/__tests__/cloud-posthog-browser.test.ts b/server/src/__tests__/cloud-posthog-browser.test.ts index 75a7ad903ad..d484d68ef73 100644 --- a/server/src/__tests__/cloud-posthog-browser.test.ts +++ b/server/src/__tests__/cloud-posthog-browser.test.ts @@ -2,7 +2,7 @@ import http from "node:http"; import type { AddressInfo } from "node:net"; import express from "express"; import request from "supertest"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { posthogAssetsOrigin, posthogUiHost, @@ -10,10 +10,15 @@ import { } from "../services/cloud-posthog-browser.js"; import { resolveCloudPostHogConfig } from "../services/cloud-posthog-config.js"; import { + createPostHogProxyLimitReporter, + DEFAULT_POSTHOG_PROXY_LIMITS, + posthogProxyClientKey, posthogProxyRoutes, posthogUpstreamUrl, posthogVisitorIp, + resolvePostHogProxyLimits, resolvePostHogProxyMount, + type PostHogProxyLimits, } from "../routes/posthog-proxy.js"; const CLOUD = { PAPERCLIP_CLOUD_TENANT_SERVER_TOKEN: "tenant-token", PAPERCLIP_CLOUD_STACK_ID: "stack-1" }; @@ -212,7 +217,7 @@ describe("PostHog proxy forwarding", () => { it("refuses a declared body over the limit without contacting PostHog", async () => { const app = express(); - app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256 })); + app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256, requestDeadlineMs: 60_000 })); const res = await request(app).post("/_ph/e/").set("Content-Type", "text/plain").send("x".repeat(11)); expect(res.status).toBe(413); expect(seen).toHaveLength(0); @@ -220,7 +225,7 @@ describe("PostHog proxy forwarding", () => { it("cuts off a streamed body that grows past the limit", async () => { const app = express(); - app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256 })); + app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 10, maxInFlightPerClient: 16, maxInFlight: 256, requestDeadlineMs: 60_000 })); const server = app.listen(0, "127.0.0.1"); await new Promise((resolve) => server.once("listening", () => resolve())); const port = (server.address() as AddressInfo).port; @@ -242,29 +247,188 @@ describe("PostHog proxy forwarding", () => { } }); - it("limits requests in flight from one client", async () => { + async function withHeldUpstream( + limits: PostHogProxyLimits, + run: (send: (ip?: string) => Promise, release: () => void, arrived: (count: number) => Promise) => Promise, + options: { trustProxy?: boolean; reportLimit?: Parameters[2] } = {}, + ) { let release: () => void = () => {}; const held = new Promise((resolve) => { release = resolve; }); - const slow = http.createServer(async (_req, res) => { await held; res.end("ok"); }); + let arrivals = 0; + const waiters: Array<{ count: number; resolve: () => void }> = []; + const slow = http.createServer(async (_req, res) => { + arrivals += 1; + for (const waiter of waiters.filter((entry) => entry.count <= arrivals)) { + waiters.splice(waiters.indexOf(waiter), 1); + waiter.resolve(); + } + await held; + res.end("ok"); + }); + const arrived = (count: number) => + count <= arrivals ? Promise.resolve() : new Promise((resolve) => { waiters.push({ count, resolve }); }); await new Promise((resolve) => slow.listen(0, "127.0.0.1", resolve)); const app = express(); - app.use("/_ph", posthogProxyRoutes(`http://127.0.0.1:${(slow.address() as AddressInfo).port}`, { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 256 })); + if (options.trustProxy) app.set("trust proxy", "loopback"); + app.use("/_ph", posthogProxyRoutes(`http://127.0.0.1:${(slow.address() as AddressInfo).port}`, limits, options.reportLimit)); const server = app.listen(0, "127.0.0.1"); await new Promise((resolve) => server.once("listening", () => resolve())); + const send = (ip?: string) => { + const req = request(server).get("/_ph/static/array.js"); + if (ip) req.set("X-Forwarded-For", ip); + return req.then((res) => res.status); + }; try { - const first = request(server).get("/_ph/static/array.js").then((res) => res.status); - await new Promise((resolve) => setTimeout(resolve, 50)); - const second = await request(server).get("/_ph/static/array.js"); - expect(second.status).toBe(429); - release(); - expect(await first).toBe(200); - // The slot frees up once the first request finishes. - expect((await request(server).get("/_ph/static/array.js")).status).toBe(200); + await run(send, release, arrived); } finally { release(); await new Promise((resolve) => server.close(() => resolve())); await new Promise((resolve) => slow.close(() => resolve())); } + } + + it("limits requests in flight from one public client address", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 256, requestDeadlineMs: 60_000 }; + await withHeldUpstream(limits, async (send, release, arrived) => { + const first = send("8.8.8.8"); + await arrived(1); + expect(await send("8.8.8.8")).toBe(429); + const other = send("9.9.9.9"); + await arrived(2); + release(); + expect(await first).toBe(200); + expect(await other).toBe(200); + expect(await send("8.8.8.8")).toBe(200); + }, { trustProxy: true }); + }); + + it("applies only the total limit when the client address is a shared private hop", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 3, requestDeadlineMs: 60_000 }; + await withHeldUpstream(limits, async (send, release, arrived) => { + const held = [send(), send(), send()]; + await arrived(3); + expect(await send()).toBe(429); + release(); + expect(await Promise.all(held)).toEqual([200, 200, 200]); + expect(await send()).toBe(200); + }); + }); + + it("does not key the per-client limit on a forged X-Forwarded-For when no proxy is trusted", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 8, requestDeadlineMs: 60_000 }; + await withHeldUpstream(limits, async (send, release, arrived) => { + const held = [send("8.8.8.8"), send("8.8.8.8"), send("8.8.8.8")]; + await arrived(3); + release(); + expect(await Promise.all(held)).toEqual([200, 200, 200]); + }); + }); + + it("frees a slot when PostHog holds a request past the request deadline", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 1, requestDeadlineMs: 200 }; + await withHeldUpstream(limits, async (send, _release, arrived) => { + const startedAt = Date.now(); + const first = send(); + await arrived(1); + expect(await send()).toBe(429); + expect(await first).toBe(502); + expect(Date.now() - startedAt).toBeLessThan(5_000); + const second = send(); + await arrived(2); + expect(await second).toBe(502); + }); + }); + + it("closes a response the browser does not read by the request deadline", async () => { + const big = Buffer.alloc(16 * 1024 * 1024, 120); + const finished: string[] = []; + const fast = http.createServer((_req, res) => { + res.writeHead(200, { "content-type": "application/javascript", "content-length": String(big.length) }); + res.end(big, () => finished.push("upstream")); + }); + await new Promise((resolve) => fast.listen(0, "127.0.0.1", resolve)); + const app = express(); + app.use("/_ph", posthogProxyRoutes(`http://127.0.0.1:${(fast.address() as AddressInfo).port}`, { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 1, requestDeadlineMs: 300 }, () => {})); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", () => resolve())); + const port = (server.address() as AddressInfo).port; + try { + let browserRequest: http.ClientRequest | undefined; + const headers = await new Promise((resolve) => { + browserRequest = http.get({ host: "127.0.0.1", port, path: "/_ph/static/recorder.js" }, (res) => { + res.on("error", () => {}); + res.pause(); + resolve(res.statusCode ?? 0); + }); + browserRequest.on("error", () => {}); + }); + expect(headers).toBe(200); + expect((await request(server).get("/_ph/static/array.js")).status).toBe(429); + await new Promise((resolve) => setTimeout(resolve, 1_000)); + expect(finished).toEqual(["upstream"]); + const next = await request(server).get("/_ph/static/array.js"); + browserRequest?.destroy(); + expect(next.status).toBe(200); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + fast.closeAllConnections(); + await new Promise((resolve) => fast.close(() => resolve())); + } + }); + + it("cuts off a body that trickles in past the request deadline", async () => { + const app = express(); + app.use("/_ph", posthogProxyRoutes(upstreamUrl, { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 1, requestDeadlineMs: 200 }, () => {})); + const server = app.listen(0, "127.0.0.1"); + await new Promise((resolve) => server.once("listening", () => resolve())); + const port = (server.address() as AddressInfo).port; + let drip: ReturnType | undefined; + try { + const startedAt = Date.now(); + const outcome = await new Promise((resolve) => { + const req = http.request({ host: "127.0.0.1", port, path: "/_ph/e/", method: "POST", headers: { "content-type": "text/plain", "transfer-encoding": "chunked" } }, (res) => { + res.on("error", () => {}); + res.resume(); + resolve(res.statusCode ?? 0); + }); + req.on("error", () => resolve("error")); + req.on("close", () => resolve("closed")); + drip = setInterval(() => req.write("x"), 20); + }); + expect([502, "error", "closed"]).toContain(outcome); + expect(Date.now() - startedAt).toBeLessThan(5_000); + expect((await request(server).get("/_ph/static/array.js")).status).not.toBe(429); + } finally { + clearInterval(drip); + await new Promise((resolve) => server.close(() => resolve())); + } + }); + + it("limits requests in flight in total across public clients and reports each refusal", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 8, maxInFlight: 2, requestDeadlineMs: 60_000 }; + const reported: Array<[string, number]> = []; + await withHeldUpstream(limits, async (send, release, arrived) => { + const held = [send("8.8.8.8"), send("9.9.9.9")]; + await arrived(2); + expect(await send("1.1.1.1")).toBe(429); + release(); + expect(await Promise.all(held)).toEqual([200, 200]); + }, { trustProxy: true, reportLimit: (reason, inFlight) => reported.push([reason, inFlight]) }); + expect(reported).toEqual([["total", 2]]); + }); + + it("reports a per-client refusal with its reason", async () => { + const limits = { maxBodyBytes: 1024, maxInFlightPerClient: 1, maxInFlight: 8, requestDeadlineMs: 60_000 }; + const reported: Array<[string, number]> = []; + await withHeldUpstream(limits, async (send, release, arrived) => { + const first = send("8.8.8.8"); + await arrived(1); + expect(await send("8.8.8.8")).toBe(429); + release(); + expect(await first).toBe(200); + }, { trustProxy: true, reportLimit: (reason, inFlight) => reported.push([reason, inFlight]) }); + expect(reported).toEqual([["per_client", 1]]); }); it("never forwards the inbound Host header", async () => { @@ -344,3 +508,123 @@ describe("posthogVisitorIp", () => { expect(posthogVisitorIp("100.64.0.11", { "x-forwarded-for": "10.0.0.1, 100.64.0.3" })).toBe("100.64.0.11"); }); }); + +describe("PostHog proxy limits", () => { + const ON = { ...CLOUD, ...TOKEN, ...BROWSER }; + + it("fits many tabs per user and many users per stack by default", () => { + expect(DEFAULT_POSTHOG_PROXY_LIMITS).toEqual({ + maxBodyBytes: 64 * 1024 * 1024, + maxInFlightPerClient: 128, + maxInFlight: 1024, + requestDeadlineMs: 60_000, + }); + const warn = vi.fn(); + expect(resolvePostHogProxyLimits({}, warn)).toEqual(DEFAULT_POSTHOG_PROXY_LIMITS); + expect(warn).not.toHaveBeenCalled(); + }); + + it("reads both limits from the environment", () => { + const warn = vi.fn(); + const limits = resolvePostHogProxyLimits( + { + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: "65536", + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT: " 256 ", + }, + warn, + ); + expect(limits).toEqual({ ...DEFAULT_POSTHOG_PROXY_LIMITS, maxInFlight: 65536, maxInFlightPerClient: 256 }); + expect(warn).not.toHaveBeenCalled(); + }); + + it.each(["0", "-5", "1.5", "1e3", "12abc", "abc", "0x10", "+8", "65537", "1024000", "99999999999999999999"])( + "keeps the default and warns for the invalid value %s", + (value) => { + const warn = vi.fn(); + const limits = resolvePostHogProxyLimits( + { PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: value, PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT: value }, + warn, + ); + expect(limits).toEqual(DEFAULT_POSTHOG_PROXY_LIMITS); + expect(warn).toHaveBeenCalledTimes(2); + expect(warn.mock.calls.map(([fields]) => fields.variable)).toEqual([ + "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT", + "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT", + ]); + }, + ); + + it("treats an empty value as unset", () => { + const warn = vi.fn(); + expect(resolvePostHogProxyLimits({ PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: " " }, warn)).toEqual(DEFAULT_POSTHOG_PROXY_LIMITS); + expect(warn).not.toHaveBeenCalled(); + }); + + it("passes the configured limits to the mounted proxy", () => { + const mount = resolvePostHogProxyMount({ + ...ON, + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: "2048", + PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT: "64", + }); + expect(mount?.limits).toEqual({ ...DEFAULT_POSTHOG_PROXY_LIMITS, maxInFlight: 2048, maxInFlightPerClient: 64 }); + expect(resolvePostHogProxyMount({ ...ON, PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT: "lots" })?.limits).toEqual( + DEFAULT_POSTHOG_PROXY_LIMITS, + ); + }); + + it.each([ + ["8.8.8.8", "8.8.8.8"], + ["2001:4860:4860::8888", "2001:4860:4860::8888"], + ["100.64.0.11", null], + ["10.0.0.2", null], + ["127.0.0.1", null], + ["::1", null], + ["::ffff:127.0.0.1", null], + ["fd00::1", null], + [undefined, null], + ])("uses %s as a per-client bucket only when it is public", (ip, key) => { + expect(posthogProxyClientKey(ip)).toBe(key); + }); + + it("warns at most once a minute and counts the refusals in between", () => { + let now = 1_000; + const warn = vi.fn(); + const report = createPostHogProxyLimitReporter(DEFAULT_POSTHOG_PROXY_LIMITS, { now: () => now, warn }); + + report("total", 1024); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]![0]).toEqual({ + refusedAtTotalLimit: 1, + refusedAtPerClientLimit: 0, + inFlight: 1024, + maxInFlight: 1024, + maxInFlightPerClient: 128, + sinceMs: null, + }); + + for (let index = 0; index < 50; index += 1) { + now += 1_000; + report(index % 2 === 0 ? "per_client" : "total", 900); + } + expect(warn).toHaveBeenCalledTimes(1); + + now = 61_000; + report("per_client", 700); + expect(warn).toHaveBeenCalledTimes(2); + expect(warn.mock.calls[1]![0]).toMatchObject({ + refusedAtTotalLimit: 25, + refusedAtPerClientLimit: 26, + inFlight: 700, + sinceMs: 60_000, + }); + + now += 59_999; + report("total", 1024); + expect(warn).toHaveBeenCalledTimes(2); + now += 1; + report("total", 1024); + expect(warn).toHaveBeenCalledTimes(3); + expect(warn.mock.calls[2]![0]).toMatchObject({ refusedAtTotalLimit: 2, refusedAtPerClientLimit: 0 }); + expect(JSON.stringify(warn.mock.calls)).not.toMatch(/\d+\.\d+\.\d+\.\d+/); + }); +}); diff --git a/server/src/routes/posthog-proxy.ts b/server/src/routes/posthog-proxy.ts index b89c7006f77..bd4bb38979d 100644 --- a/server/src/routes/posthog-proxy.ts +++ b/server/src/routes/posthog-proxy.ts @@ -37,14 +37,100 @@ export type PostHogProxyLimits = { maxInFlightPerClient: number; /** Requests in flight for the whole proxy. */ maxInFlight: number; + requestDeadlineMs: number; }; export const DEFAULT_POSTHOG_PROXY_LIMITS: PostHogProxyLimits = { maxBodyBytes: 64 * 1024 * 1024, - maxInFlightPerClient: 16, - maxInFlight: 256, + maxInFlightPerClient: 128, + maxInFlight: 1024, + requestDeadlineMs: 60_000, }; +export const POSTHOG_PROXY_MAX_IN_FLIGHT_ENV = "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT"; +export const POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT_ENV = "PAPERCLIP_POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT"; + +const LIMIT_WARNING_INTERVAL_MS = 60_000; +const MAX_CONFIGURED_IN_FLIGHT = 65_536; + +type LimitWarning = (fields: Record, message: string) => void; + +function warnWithLogger(fields: Record, message: string) { + logger.warn(fields, message); +} + +function parseInFlightLimit( + env: CloudInstanceEnv, + name: string, + fallback: number, + warn: LimitWarning, +): number { + const raw = env[name]; + if (raw === undefined || raw.trim() === "") return fallback; + const value = raw.trim(); + const parsed = Number(value); + if (/^[1-9][0-9]*$/.test(value) && parsed <= MAX_CONFIGURED_IN_FLIGHT) return parsed; + warn( + { variable: name, defaultValue: fallback, maxValue: MAX_CONFIGURED_IN_FLIGHT }, + `${name} is not an integer from 1 to ${MAX_CONFIGURED_IN_FLIGHT}; the PostHog proxy uses the default`, + ); + return fallback; +} + +export function resolvePostHogProxyLimits( + env: CloudInstanceEnv = process.env, + warn: LimitWarning = warnWithLogger, +): PostHogProxyLimits { + return { + maxBodyBytes: DEFAULT_POSTHOG_PROXY_LIMITS.maxBodyBytes, + requestDeadlineMs: DEFAULT_POSTHOG_PROXY_LIMITS.requestDeadlineMs, + maxInFlight: parseInFlightLimit(env, POSTHOG_PROXY_MAX_IN_FLIGHT_ENV, DEFAULT_POSTHOG_PROXY_LIMITS.maxInFlight, warn), + maxInFlightPerClient: parseInFlightLimit( + env, + POSTHOG_PROXY_MAX_IN_FLIGHT_PER_CLIENT_ENV, + DEFAULT_POSTHOG_PROXY_LIMITS.maxInFlightPerClient, + warn, + ), + }; +} + +export function posthogProxyClientKey(ip: string | undefined): string | null { + return ip && isPublicIpAddress(ip) ? ip : null; +} + +export type PostHogProxyLimitReason = "total" | "per_client"; + +export function createPostHogProxyLimitReporter( + limits: PostHogProxyLimits, + options: { now?: () => number; warn?: LimitWarning; intervalMs?: number } = {}, +) { + const now = options.now ?? Date.now; + const warn = options.warn ?? warnWithLogger; + const intervalMs = options.intervalMs ?? LIMIT_WARNING_INTERVAL_MS; + let lastWarnedAt: number | null = null; + const refused: Record = { total: 0, per_client: 0 }; + + return (reason: PostHogProxyLimitReason, inFlight: number) => { + refused[reason] += 1; + const at = now(); + if (lastWarnedAt !== null && at - lastWarnedAt < intervalMs) return; + warn( + { + refusedAtTotalLimit: refused.total, + refusedAtPerClientLimit: refused.per_client, + inFlight, + maxInFlight: limits.maxInFlight, + maxInFlightPerClient: limits.maxInFlightPerClient, + sinceMs: lastWarnedAt === null ? null : at - lastWarnedAt, + }, + "PostHog proxy refused requests at its in-flight limit", + ); + lastWarnedAt = at; + refused.total = 0; + refused.per_client = 0; + }; +} + const BODY_TOO_LARGE = "posthog_proxy_body_too_large"; type PostHogUpstream = "ingest" | "assets"; @@ -131,7 +217,11 @@ export function posthogForwardRequestHeaders( return forwarded; } -export function posthogProxyRoutes(ingestHost: string, limits: PostHogProxyLimits = DEFAULT_POSTHOG_PROXY_LIMITS) { +export function posthogProxyRoutes( + ingestHost: string, + limits: PostHogProxyLimits = DEFAULT_POSTHOG_PROXY_LIMITS, + reportLimit: (reason: PostHogProxyLimitReason, inFlight: number) => void = createPostHogProxyLimitReporter(limits), +) { const router = Router(); let inFlight = 0; const inFlightByClient = new Map(); @@ -158,17 +248,25 @@ export function posthogProxyRoutes(ingestHost: string, limits: PostHogProxyLimit } // Bound the work one browser, or all of them, can put on this process. - const client = req.ip ?? "unknown"; - const clientInFlight = inFlightByClient.get(client) ?? 0; - if (inFlight >= limits.maxInFlight || clientInFlight >= limits.maxInFlightPerClient) { + const client = posthogProxyClientKey(req.ip); + const clientInFlight = client === null ? 0 : inFlightByClient.get(client) ?? 0; + const refusedBy: PostHogProxyLimitReason | null = + inFlight >= limits.maxInFlight + ? "total" + : client !== null && clientInFlight >= limits.maxInFlightPerClient + ? "per_client" + : null; + if (refusedBy) { + reportLimit(refusedBy, inFlight); res.setHeader("Retry-After", "1"); res.status(429).json({ error: "too_many_requests" }); return; } inFlight += 1; - inFlightByClient.set(client, clientInFlight + 1); + if (client !== null) inFlightByClient.set(client, clientInFlight + 1); res.once("close", () => { inFlight -= 1; + if (client === null) return; const remaining = (inFlightByClient.get(client) ?? 1) - 1; if (remaining > 0) inFlightByClient.set(client, remaining); else inFlightByClient.delete(client); @@ -197,6 +295,11 @@ export function posthogProxyRoutes(ingestHost: string, limits: PostHogProxyLimit upstreamReq.on("timeout", () => { upstreamReq.destroy(new Error("posthog_proxy_timeout")); }); + const deadline = setTimeout(() => { + if (res.headersSent || upstreamReq.destroyed) res.destroy(); + upstreamReq.destroy(new Error("posthog_proxy_deadline")); + }, limits.requestDeadlineMs); + res.once("close", () => clearTimeout(deadline)); upstreamReq.on("error", (err) => { logger.debug({ err, path: target.pathname }, "PostHog proxy request failed"); if (!res.headersSent && bodyTooLarge) { @@ -246,5 +349,6 @@ export function resolvePostHogProxyMount(env: CloudInstanceEnv = process.env) { logger.warn("PAPERCLIP_POSTHOG_HOST is not a valid URL; the PostHog proxy is off"); return null; } - return { path: POSTHOG_PROXY_PATH, router: posthogProxyRoutes(config.host) }; + const limits = resolvePostHogProxyLimits(env); + return { path: POSTHOG_PROXY_PATH, router: posthogProxyRoutes(config.host, limits), limits }; }