diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 197b10cb495..fc54cebf220 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -1131,6 +1131,21 @@ in the middle, and when the operating system chooses the chunks. Measure it with `pnpm --filter @paperclipai/db exec node --expose-gc --import tsx scripts/benchmark-postgres-receive-buffer.ts` (`CONNECTIONS` and `ROW_BYTES` are optional). +The patch also keeps the driver's connection queues proportional to the +connections they hold. Without this fix, the queue of idle connections keeps an +empty slot for every query it has handed out until it is completely empty. A +busy pool always has some idle connections, so that queue never empties. Each +new query then scans one slot for every query the process has already run. In a +simulation at 1,000 statements per second, dispatching a query used about 15% +of the main thread after 17 minutes, and the cost kept growing. The patched queue removes +the empty slots once there are more than 1,024 of them and more empty slots +than queued connections. It also searches only the queued connections. +`packages/db/src/postgres-queue.test.ts` checks the queue order and the work per +query, and `packages/db/src/postgres-queue-pool.test.ts` checks a real pool. +Measure it with +`pnpm --filter @paperclipai/db exec tsx scripts/benchmark-postgres-queue.ts` +(`SIMULATED_QUERIES` and `QUERIES` are optional). + Trusted-header actor synchronization retries transient connection failures, including `CONNECT_TIMEOUT`, at most twice. This retry applies only to the idempotent actor synchronization operations, not arbitrary transactions. A diff --git a/packages/db/scripts/benchmark-postgres-queue.ts b/packages/db/scripts/benchmark-postgres-queue.ts new file mode 100644 index 00000000000..ca5b8f707f4 --- /dev/null +++ b/packages/db/scripts/benchmark-postgres-queue.ts @@ -0,0 +1,69 @@ +import { execFile } from "node:child_process"; +import { createRequire } from "node:module"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { startEmbeddedPostgresTestDatabase } from "../src/test-embedded-postgres.js"; + +type Item = { queue: Queue | null }; +type Queue = { readonly length: number; push(item: Item): Item; shift(): Item | undefined; remove(item: Item): Item | null }; + +const simulatedQueries = Number(process.env.SIMULATED_QUERIES ?? 1_000_000); +const poolQueries = Number(process.env.QUERIES ?? 200_000); +const driverRoot = path.resolve(path.dirname(createRequire(import.meta.url).resolve("postgres")), "../.."); +const Queue: () => Queue = (await import(pathToFileURL(path.join(driverRoot, "src/queue.js")).href)).default; + +function simulate() { + const open = Queue(); + const busy = Queue(); + const move = (c: Item, queue: Queue) => { + c.queue?.remove(c); + queue.push(c); + c.queue = queue; + }; + for (let id = 0; id < 117; id++) move({ queue: null }, open); + const running: Item[] = new Array(8); + const windows: number[] = []; + let started = performance.now(); + for (let query = 1; query <= simulatedQueries; query++) { + const c = open.shift()!; + move(c, busy); + const slot = query % running.length; + if (running[slot]) move(running[slot], open); + running[slot] = c; + if (query % (simulatedQueries / 5) === 0) { + windows.push(Number((((performance.now() - started) * 1000) / (simulatedQueries / 5)).toFixed(3))); + started = performance.now(); + } + } + return { queries: simulatedQueries, connections: 117, inFlight: 8, microsPerQueryByWindow: windows }; +} + +const temporary = process.env.DATABASE_URL ? null : await startEmbeddedPostgresTestDatabase("postgres-queue-benchmark-"); +try { + const simulation = simulate(); + const { stdout } = await promisify(execFile)(process.execPath, [ + fileURLToPath(new URL("../src/__fixtures__/postgres-queue-pool.mjs", import.meta.url)), + ], { + env: { + ...process.env, + TEST_DATABASE_URL: process.env.DATABASE_URL ?? temporary!.connectionString, + QUERIES: String(poolQueries), + WINDOWS: "10", + MEASURE_SCANS: "0", + }, + timeout: 600_000, + }); + const pool = JSON.parse(stdout) as { windows: Array<{ cpuMicrosPerQuery: number; wallMicrosPerQuery: number }> }; + console.log(JSON.stringify({ + simulation, + pool: { + ...pool, + windows: undefined, + cpuMicrosPerQueryByWindow: pool.windows.map((window) => window.cpuMicrosPerQuery), + wallMicrosPerQueryByWindow: pool.windows.map((window) => window.wallMicrosPerQuery), + }, + })); +} finally { + await temporary?.cleanup(); +} diff --git a/packages/db/src/__fixtures__/postgres-queue-pool.mjs b/packages/db/src/__fixtures__/postgres-queue-pool.mjs new file mode 100644 index 00000000000..7058009ca75 --- /dev/null +++ b/packages/db/src/__fixtures__/postgres-queue-pool.mjs @@ -0,0 +1,60 @@ +import { createRequire } from "node:module"; + +const queries = Number(process.env.QUERIES ?? 20_000); +const connections = Number(process.env.CONNECTIONS ?? 24); +const concurrency = Number(process.env.CONCURRENCY ?? 8); +const windows = Number(process.env.WINDOWS ?? 4); +const measured = process.env.MEASURE_SCANS !== "0"; + +const scans = { max: 0 }; +const indexOf = Array.prototype.indexOf; +if (measured) { + Array.prototype.indexOf = function (item, from) { + if (item && typeof item === "object" && "idleTimer" in item && "execute" in item) { + scans.max = Math.max(scans.max, this.length - Math.max(0, from ?? 0)); + } + return indexOf.call(this, item, from); + }; +} + +const postgres = process.env.DRIVER_FORMAT === "cjs" + ? createRequire(import.meta.url)("postgres") + : (await import("postgres")).default; +const sql = postgres(process.env.TEST_DATABASE_URL, { + max: connections, + idle_timeout: 0, + max_lifetime: 0, + connection: { application_name: "driver-queue-pool" }, + onnotice() {}, +}); + +try { + await Promise.all(Array.from({ length: connections }, () => sql`select pg_sleep(0.05)`)); + const [{ count }] = await sql`select count(*)::int as count from pg_stat_activity where application_name = 'driver-queue-pool'`; + if (count !== connections) throw new Error(`expected ${connections} open connections, found ${count}`); + + const perWindow = Math.floor(queries / windows); + const results = []; + for (let window = 0; window < windows; window++) { + scans.max = 0; + let remaining = perWindow; + const cpu = process.cpuUsage(); + const started = performance.now(); + await Promise.all(Array.from({ length: concurrency }, async () => { + while (remaining-- > 0) { + const [row] = await sql`select 1 as value`; + if (row.value !== 1) throw new Error("unexpected result"); + } + })); + const used = process.cpuUsage(cpu); + results.push({ + cpuMicrosPerQuery: Number(((used.user + used.system) / perWindow).toFixed(2)), + wallMicrosPerQuery: Number((((performance.now() - started) * 1000) / perWindow).toFixed(2)), + maxScan: scans.max, + }); + } + console.log(JSON.stringify({ queries: perWindow * windows, connections, concurrency, windows: results })); +} finally { + Array.prototype.indexOf = indexOf; + await sql.end({ timeout: 1 }); +} diff --git a/packages/db/src/postgres-queue-pool.test.ts b/packages/db/src/postgres-queue-pool.test.ts new file mode 100644 index 00000000000..787db398a42 --- /dev/null +++ b/packages/db/src/postgres-queue-pool.test.ts @@ -0,0 +1,43 @@ +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, + type EmbeddedPostgresTestDatabase, +} from "./test-embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const run = promisify(execFile); +const fixture = fileURLToPath(new URL("./__fixtures__/postgres-queue-pool.mjs", import.meta.url)); + +describe.skipIf(!support.supported)("postgres connection queue in a busy pool", () => { + let database: EmbeddedPostgresTestDatabase; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-driver-queue-pool-"); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { await database?.cleanup(); }); + + for (const format of ["esm", "cjs"]) { + it(`${format}: dispatching a query scans only live connections while idle ones stay open`, async () => { + const connections = 24; + const { stdout, stderr } = await run(process.execPath, [fixture], { + env: { + ...process.env, + TEST_DATABASE_URL: database.connectionString, + DRIVER_FORMAT: format, + QUERIES: "20000", + CONNECTIONS: String(connections), + CONCURRENCY: "8", + }, + timeout: 60_000, + }); + expect(stderr).toBe(""); + const result = JSON.parse(stdout) as { queries: number; windows: Array<{ maxScan: number }> }; + expect(result.queries).toBe(20_000); + for (const window of result.windows) expect(window.maxScan).toBeLessThanOrEqual(connections); + }, 70_000); + } +}); diff --git a/packages/db/src/postgres-queue.test.ts b/packages/db/src/postgres-queue.test.ts new file mode 100644 index 00000000000..4ec4717cc43 --- /dev/null +++ b/packages/db/src/postgres-queue.test.ts @@ -0,0 +1,145 @@ +import { createRequire } from "node:module"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { describe, expect, it } from "vitest"; + +type Queue = { + readonly length: number; + push(item: T): T; + shift(): T | undefined; + remove(item: T): T | null; +}; +type QueueFactory = (initial?: T[]) => Queue; + +const require = createRequire(import.meta.url); +const driverRoot = path.resolve(path.dirname(require.resolve("postgres")), "../.."); +const builds: Array<[string, () => Promise]> = [ + ["esm", async () => (await import(pathToFileURL(path.join(driverRoot, "src/queue.js")).href)).default], + ["cjs", async () => require(path.join(driverRoot, "cjs/src/queue.js"))], +]; + +class Item { + queue: Queue | null = null; + constructor(readonly id: number) {} +} + +function random(seed: number) { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +function measureBackingWork(work: (stats: { scanned: number; backing: number }) => T) { + const indexOf = Array.prototype.indexOf; + const push = Array.prototype.push; + const stats = { scanned: 0, backing: 0 }; + Array.prototype.indexOf = function (this: unknown[], item: unknown, from?: number) { + if (item instanceof Item) stats.scanned = Math.max(stats.scanned, this.length - Math.max(0, from ?? 0)); + return indexOf.call(this, item, from); + }; + Array.prototype.push = function (this: unknown[], ...items: unknown[]) { + const length = push.apply(this, items); + if (items[0] instanceof Item) stats.backing = Math.max(stats.backing, length); + return length; + }; + try { + return work(stats); + } finally { + Array.prototype.indexOf = indexOf; + Array.prototype.push = push; + } +} + +describe.each(builds)("postgres.js connection queue (%s)", (_format, load) => { + it("keeps FIFO order, length and remove results", async () => { + const Queue = await load(); + const [a, b, c, d] = [1, 2, 3, 4].map((id) => new Item(id)); + const initial = [a, b]; + const queue = Queue(initial); + initial.push(c); + expect(queue.length).toBe(2); + expect(queue.push(c)).toBe(c); + expect(queue.push(d)).toBe(d); + expect(queue.remove(c)).toBe(c); + expect(queue.remove(c)).toBeNull(); + expect(queue.remove(new Item(9))).toBeNull(); + expect(queue.length).toBe(3); + expect(queue.shift()).toBe(a); + expect(queue.remove(a)).toBeNull(); + expect(queue.shift()).toBe(b); + expect(queue.shift()).toBe(d); + expect(queue.length).toBe(0); + expect(queue.shift()).toBeUndefined(); + expect(queue.length).toBe(0); + expect(queue.remove(a)).toBeNull(); + queue.push(a); + expect(queue.length).toBe(1); + expect(queue.shift()).toBe(a); + }); + + it("matches a plain array queue across random interleavings", async () => { + const Queue = await load(); + const next = random(21_595); + const queue = Queue(); + const model: Item[] = []; + const removed: Item[] = []; + let created = 0; + for (let step = 0; step < 300_000; step++) { + const target = [4, 150, 2_500][Math.floor(step / 20_000) % 3]; + const roll = next(); + if (roll < (model.length < target ? 0.55 : 0.4)) { + const item = removed.length && next() < 0.3 ? removed.pop()! : new Item(created++); + expect(queue.push(item)).toBe(item); + model.push(item); + } else if (roll < 0.9) { + expect(queue.shift()).toBe(model.shift()); + } else if (roll < 0.97 && model.length) { + const item = model[Math.floor(next() * model.length)]; + expect(queue.remove(item)).toBe(item); + model.splice(model.indexOf(item), 1); + removed.push(item); + } else { + expect(queue.remove(removed[removed.length - 1] ?? new Item(-1))).toBeNull(); + } + expect(queue.length).toBe(model.length); + } + while (model.length) expect(queue.shift()).toBe(model.shift()); + expect(queue.length).toBe(0); + }); + + it("keeps per-query work bounded by live connections when the open queue never drains", async () => { + const Queue = await load(); + const connections = 117; + const inFlight = 8; + const queries = 1_000_000; + const limit = { scanned: connections + inFlight, backing: 2 * connections + 1_100 }; + const open = Queue(); + const busy = Queue(); + const move = (c: Item, queue: Queue) => { + c.queue?.remove(c); + queue.push(c); + c.queue = queue; + }; + const stats = measureBackingWork((stats) => { + for (let id = 0; id < connections; id++) move(new Item(id), open); + const running: Item[] = new Array(inFlight); + for (let query = 0; query < queries; query++) { + const c = open.shift()!; + move(c, busy); + const slot = query % inFlight; + if (running[slot]) move(running[slot], open); + running[slot] = c; + if (stats.backing > 10 * limit.backing) break; + } + return { ...stats }; + }); + expect(open.length + busy.length).toBe(connections); + expect(stats.scanned).toBeLessThanOrEqual(limit.scanned); + expect(stats.backing).toBeLessThanOrEqual(limit.backing); + }); +}); diff --git a/patches/postgres@3.4.9.patch b/patches/postgres@3.4.9.patch index 20854f067b8..b9cd7a82c8f 100644 --- a/patches/postgres@3.4.9.patch +++ b/patches/postgres@3.4.9.patch @@ -102,6 +102,34 @@ index f09c61c7e998f411560b95c9a8e1ed6a97972c11..04dded5c7e2cdc34ac01b29a87cc9324 c.queue === full ? queries.push(q) : c.execute(q) || move(c, full) +diff --git a/cjs/src/queue.js b/cjs/src/queue.js +index 8438f5da92cf7637d98a33491d75c75b1c817c25..1c9a330fa173ecae84219055368cf630feadbda2 100644 +--- a/cjs/src/queue.js ++++ b/cjs/src/queue.js +@@ -9,10 +9,10 @@ function Queue(initial = []) { + return xs.length - index + }, + remove: (x) => { +- const index = xs.indexOf(x) +- return index === -1 ++ const at = xs.indexOf(x, index) ++ return at === -1 + ? null +- : (xs.splice(index, 1), x) ++ : (xs.splice(at, 1), x) + }, + push: (x) => (xs.push(x), x), + shift: () => { +@@ -21,6 +21,9 @@ function Queue(initial = []) { + if (index === xs.length) { + index = 0 + xs = [] ++ } else if (index > 1024 && index * 2 > xs.length) { ++ xs = xs.slice(index) ++ index = 0 + } else { + xs[index - 1] = undefined + } diff --git a/src/connection.js b/src/connection.js index 1b1cccde43b4570d5d071d6ffaab2669b3c2065a..0a4c4cc185f14afa30c7c6a294bafe689e685683 100644 --- a/src/connection.js @@ -206,3 +234,31 @@ index c7fba3dace20173f4cb13f3f138bb6d213b43683..f0888986add8ba02c8853f792ae42dd1 c.queue === full ? queries.push(q) : c.execute(q) || move(c, full) +diff --git a/src/queue.js b/src/queue.js +index c4ef97164f057db99dce1b9af3401dbb1af37e31..46e7ade55c0a3250137a21117c03ce9166b40bc2 100644 +--- a/src/queue.js ++++ b/src/queue.js +@@ -9,10 +9,10 @@ function Queue(initial = []) { + return xs.length - index + }, + remove: (x) => { +- const index = xs.indexOf(x) +- return index === -1 ++ const at = xs.indexOf(x, index) ++ return at === -1 + ? null +- : (xs.splice(index, 1), x) ++ : (xs.splice(at, 1), x) + }, + push: (x) => (xs.push(x), x), + shift: () => { +@@ -21,6 +21,9 @@ function Queue(initial = []) { + if (index === xs.length) { + index = 0 + xs = [] ++ } else if (index > 1024 && index * 2 > xs.length) { ++ xs = xs.slice(index) ++ index = 0 + } else { + xs[index - 1] = undefined + }