diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 138f1128461..f4fbdea8348 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -1095,6 +1095,18 @@ with slow background work. It does not make the SQL itself faster. The default pipeline limit remains unchanged. Test transaction behavior before using this setting with a driver build that does not include the patch. +The patch also releases a connection's receive buffer when the connection has +read every message in it. Without this fix, an idle connection keeps the buffer +of its last read until its next read: the socket chunk, or the whole buffer of +its largest recent message. At 340 open connections that each read a 1 MB row, +this held about 320 MB of `external` memory. The memory is outside the +JavaScript heap. `packages/db/src/postgres-receive-buffer.test.ts` checks that +the buffer is released. It also checks that results are reassembled correctly +when socket chunks end exactly at message boundaries, when they split messages +in the middle, and when the operating system chooses the chunks. Measure it with +`pnpm --filter @paperclipai/db exec node --expose-gc --import tsx scripts/benchmark-postgres-receive-buffer.ts` +(`CONNECTIONS` and `ROW_BYTES` are optional). + Trusted-header actor synchronization retries transient connection failures, including `CONNECT_TIMEOUT`, at most twice. This retry applies only to the idempotent actor synchronization operations, not arbitrary transactions. A diff --git a/packages/db/scripts/benchmark-postgres-receive-buffer.ts b/packages/db/scripts/benchmark-postgres-receive-buffer.ts new file mode 100644 index 00000000000..6ad249bb59c --- /dev/null +++ b/packages/db/scripts/benchmark-postgres-receive-buffer.ts @@ -0,0 +1,38 @@ +import postgres from "postgres"; +import { startEmbeddedPostgresTestDatabase } from "../src/test-embedded-postgres.js"; + +const gc = (globalThis as { gc?: () => void }).gc; +if (!gc) throw new Error("Run with node --expose-gc"); +const connections = Number(process.env.CONNECTIONS ?? 80); +const rowBytes = Number(process.env.ROW_BYTES ?? 1_000_000); +const temporary = process.env.DATABASE_URL ? null : await startEmbeddedPostgresTestDatabase("postgres-receive-buffer-"); +const sql = postgres(process.env.DATABASE_URL ?? temporary!.connectionString, { + max: connections, idle_timeout: 60, prepare: false, onnotice: () => {}, + connection: { application_name: "receive-buffer-benchmark" }, +}); + +function memory() { + gc!(); gc!(); + const { heapUsed, external, rss } = process.memoryUsage(); + return { heapUsed, external, rss }; +} + +try { + await sql.unsafe("select 1"); + const before = memory(); + await Promise.all(Array.from({ length: connections }, () => sql.unsafe("select pg_sleep(0.5)"))); + const open = memory(); + await Promise.all(Array.from({ length: connections }, () => sql.unsafe("select repeat('x', $1::int) as value", [rowBytes]).then(() => undefined))); + const idleAfterLargeRow = memory(); + const perConnectionKB = (from: ReturnType, to: ReturnType, key: keyof ReturnType) => + Math.round((to[key] - from[key]) / connections / 1024); + console.log(JSON.stringify({ + connections, rowBytes, + openIdle: { heapKB: perConnectionKB(before, open, "heapUsed"), externalKB: perConnectionKB(before, open, "external"), rssKB: perConnectionKB(before, open, "rss") }, + idleAfterLargeRow: { heapKB: perConnectionKB(before, idleAfterLargeRow, "heapUsed"), externalKB: perConnectionKB(before, idleAfterLargeRow, "external"), rssKB: perConnectionKB(before, idleAfterLargeRow, "rss") }, + pinnedExternalMB: Math.round((idleAfterLargeRow.external - open.external) / 1048576), + })); +} finally { + await sql.end({ timeout: 1 }); + await temporary?.cleanup(); +} diff --git a/packages/db/src/__fixtures__/postgres-receive-buffer.mjs b/packages/db/src/__fixtures__/postgres-receive-buffer.mjs new file mode 100644 index 00000000000..05c9d52d2f2 --- /dev/null +++ b/packages/db/src/__fixtures__/postgres-receive-buffer.mjs @@ -0,0 +1,108 @@ +import assert from "node:assert/strict"; +import { createRequire } from "node:module"; +import net from "node:net"; +import { setImmediate } from "node:timers/promises"; + +const mode = process.env.CHUNKING ?? "socket"; +const databasePort = Number(new URL(process.env.TEST_DATABASE_URL).port || 5432); +const chunking = { exactEnds: 0, splitInside: 0 }; +const parsers = new WeakMap(); +const emit = net.Socket.prototype.emit; +net.Socket.prototype.emit = function (event, ...args) { + if (event !== "data" || mode === "socket" || this.remotePort !== databasePort) return emit.call(this, event, ...args); + const chunk = args[0]; + const state = parsers.get(this) ?? { header: Buffer.alloc(0), remaining: 0 }; + parsers.set(this, state); + const cuts = []; + let offset = 0; + while (offset < chunk.length) { + if (state.remaining === 0) { + const take = Math.min(5 - state.header.length, chunk.length - offset); + state.header = Buffer.concat([state.header, chunk.subarray(offset, offset + take)]); + offset += take; + if (state.header.length < 5) break; + state.remaining = state.header.readUInt32BE(1) - 4; + state.header = Buffer.alloc(0); + if (mode === "straddle" && state.remaining > 1) { + const middle = offset + Math.min(Math.floor(state.remaining / 2), chunk.length - offset); + if (middle > offset && middle < chunk.length) { cuts.push(middle); chunking.splitInside += 1; } + } + } + const take = Math.min(state.remaining, chunk.length - offset); + offset += take; + state.remaining -= take; + if (state.remaining === 0 && state.header.length === 0) { + cuts.push(offset); + chunking.exactEnds += 1; + } + } + let start = 0; + for (const cut of [...new Set(cuts)].sort((a, b) => a - b).concat(chunk.length)) { + if (cut > start) emit.call(this, "data", chunk.subarray(start, cut)); + start = Math.max(start, cut); + } + return true; +}; + +const concat = Buffer.concat; +const received = []; +Buffer.concat = function (...args) { + const result = concat.apply(this, args); + if (result.length >= 512 * 1024) received.push(new WeakRef(result.buffer)); + return result; +}; + +const postgres = process.env.DRIVER_FORMAT === "cjs" + ? createRequire(import.meta.url)("postgres") + : (await import("postgres")).default; +const sql = postgres(process.env.TEST_DATABASE_URL, { + max: 1, + connection: { application_name: "driver-receive-buffer" }, + onnotice() {}, +}); + +async function collected() { + for (let attempt = 0; attempt < 5; attempt++) { + await setImmediate(); + globalThis.gc(); + } + return received.filter(ref => ref.deref() !== undefined).length; +} + +try { + const [large] = await sql.unsafe("select repeat('x', 3000000) as value"); + assert.equal(large.value.length, 3_000_000); + assert.equal(large.value, "x".repeat(3_000_000)); + + for (let size = 65_400; size <= 65_560; size++) { + const [row] = await sql.unsafe("select repeat('z', $1::int) as value", [size]); + assert.equal(row.value.length, size); + } + + const rows = await sql.unsafe("select g, repeat('y', 50) as value from generate_series(1, 20000) g"); + assert.equal(rows.length, 20_000); + assert.equal(rows[19_999].g, 20_000); + assert.ok(rows.every((row, index) => row.g === index + 1 && row.value === "y".repeat(50))); + + const [pair] = await sql.unsafe("select repeat('a', 1500000) as a, repeat('b', 1500000) as b"); + assert.equal(pair.a, "a".repeat(1_500_000)); + assert.equal(pair.b, "b".repeat(1_500_000)); + + assert.ok(received.length > 0, "a large row must be reassembled from several socket chunks"); + assert.equal(await collected(), 0, "an idle connection must not keep the receive buffer of its last result"); + + for (let round = 0; round < 20; round++) { + const size = 1_000_000 + round * 100_003; + const [row] = await sql.unsafe("select repeat('r', $1::int) as value", [size]); + assert.equal(row.value.length, size); + assert.equal(await collected(), 0, `an idle connection kept the receive buffer of a ${size} byte row`); + } + + const [after] = await sql.unsafe("select 1 as value"); + assert.equal(after.value, 1); + if (mode === "message") assert.ok(chunking.exactEnds > 20_000, "chunks must end exactly at message boundaries"); + if (mode === "straddle") assert.ok(chunking.splitInside > 20_000, "chunks must split messages in the middle"); + console.log("released"); +} finally { + await sql.end({ timeout: 1 }); +} diff --git a/packages/db/src/postgres-receive-buffer.test.ts b/packages/db/src/postgres-receive-buffer.test.ts new file mode 100644 index 00000000000..c704de8ee81 --- /dev/null +++ b/packages/db/src/postgres-receive-buffer.test.ts @@ -0,0 +1,35 @@ +import { execFile } from "node:child_process"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { + EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, + getEmbeddedPostgresTestSupport, + startEmbeddedPostgresTestDatabase, + type EmbeddedPostgresTestDatabase, +} from "./test-embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const run = promisify(execFile); +const fixture = fileURLToPath(new URL("./__fixtures__/postgres-receive-buffer.mjs", import.meta.url)); + +describe.skipIf(!support.supported)("postgres receive buffer", () => { + let database: EmbeddedPostgresTestDatabase; + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("paperclip-driver-receive-buffer-"); + }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); + afterAll(async () => { await database?.cleanup(); }); + + for (const format of ["esm", "cjs"]) { + for (const chunking of ["socket", "message", "straddle"]) { + it(`${format}: reassembles ${chunking}-chunked results and releases the buffer once the connection is idle`, async () => { + const { stdout, stderr } = await run(process.execPath, ["--expose-gc", fixture], { + env: { ...process.env, TEST_DATABASE_URL: database.connectionString, DRIVER_FORMAT: format, CHUNKING: chunking }, + timeout: 60_000, + }); + expect(stdout.trim()).toBe("released"); + expect(stderr).toBe(""); + }, 70_000); + } + } +}); diff --git a/patches/postgres@3.4.9.patch b/patches/postgres@3.4.9.patch index c45fb197a7c..20854f067b8 100644 --- a/patches/postgres@3.4.9.patch +++ b/patches/postgres@3.4.9.patch @@ -1,5 +1,5 @@ diff --git a/cjs/src/connection.js b/cjs/src/connection.js -index 07f6716702ac888215c86ad6e0071aaf55ae519c..85700681ccc7bf98f1fe987978db47e0097227f3 100644 +index 07f6716702ac888215c86ad6e0071aaf55ae519c..4ee75faa8ae47555576e3b4e0ec6b7c4b140ec2f 100644 --- a/cjs/src/connection.js +++ b/cjs/src/connection.js @@ -173,8 +173,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose @@ -12,7 +12,16 @@ index 07f6716702ac888215c86ad6e0071aaf55ae519c..85700681ccc7bf98f1fe987978db47e0 } catch (error) { sent.length === 0 && write(Sync) errored(error) -@@ -438,6 +438,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose +@@ -330,6 +330,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose + remaining = 0 + incomings = null + } ++ ++ incoming.length === 0 && (incoming = Buffer.alloc(0)) + } + + async function connect() { +@@ -438,6 +440,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose remaining = 0 incomings = null clearImmediate(nextWriteTimer) @@ -20,7 +29,7 @@ index 07f6716702ac888215c86ad6e0071aaf55ae519c..85700681ccc7bf98f1fe987978db47e0 socket.removeListener('data', data) socket.removeListener('connect', connected) idleTimer.cancel() -@@ -451,6 +452,9 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose +@@ -451,6 +454,9 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return reconnect() !hadError && (query || sent.length) && error(Errors.connection('CONNECTION_CLOSED', options, socket)) @@ -94,7 +103,7 @@ index f09c61c7e998f411560b95c9a8e1ed6a97972c11..04dded5c7e2cdc34ac01b29a87cc9324 ? queries.push(q) : c.execute(q) || move(c, full) diff --git a/src/connection.js b/src/connection.js -index 1b1cccde43b4570d5d071d6ffaab2669b3c2065a..81541f9cea75925256d7b02b2079c60802b2f92f 100644 +index 1b1cccde43b4570d5d071d6ffaab2669b3c2065a..0a4c4cc185f14afa30c7c6a294bafe689e685683 100644 --- a/src/connection.js +++ b/src/connection.js @@ -173,8 +173,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose @@ -107,7 +116,16 @@ index 1b1cccde43b4570d5d071d6ffaab2669b3c2065a..81541f9cea75925256d7b02b2079c608 } catch (error) { sent.length === 0 && write(Sync) errored(error) -@@ -438,6 +438,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose +@@ -330,6 +330,8 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose + remaining = 0 + incomings = null + } ++ ++ incoming.length === 0 && (incoming = Buffer.alloc(0)) + } + + async function connect() { +@@ -438,6 +440,7 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose remaining = 0 incomings = null clearImmediate(nextWriteTimer) @@ -115,7 +133,7 @@ index 1b1cccde43b4570d5d071d6ffaab2669b3c2065a..81541f9cea75925256d7b02b2079c608 socket.removeListener('data', data) socket.removeListener('connect', connected) idleTimer.cancel() -@@ -451,6 +452,9 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose +@@ -451,6 +454,9 @@ function Connection(options, queues = {}, { onopen = noop, onend = noop, onclose return reconnect() !hadError && (query || sent.length) && error(Errors.connection('CONNECTION_CLOSED', options, socket))