From b1e70ab6dd80a3ec45144f9c32b4f48ad874bbfd Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Sat, 14 Nov 2026 21:57:04 -0600 Subject: [PATCH] fix(db): recover prepared statements that a pooler or schema change invalidates (#21718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work. > - The server runs every database statement from one Node main thread. With `prepare: true`, postgres.js sends a statement in one round trip instead of two, and the CPU cost of each statement is about half (benchmark from #21707). > - `DATABASE_PREPARED_STATEMENTS=true` turns on preparation for ORM queries (#17673). Hosted instances connect through a transaction-mode pooler (PgBouncer 1.22 or later, Neon `-pooler` hosts), which supports protocol-level prepared statements. > - I tested the flag locally through PgBouncer 1.26.1 in transaction mode. Evictions, server connection churn, `RECONNECT`, transactions, pipelining, `SET LOCAL`, advisory transaction locks and savepoints all work. One failure is not acceptable: after a migration changes a column type, every prepared statement that reads that column fails, for every client and for every new process. > - PgBouncer shares one server-side prepared statement by SQL text. PostgreSQL never repairs a named statement whose result type changed, so it fails with `cached plan must not change result type` until PgBouncer drops it. The postgres.js retry prepares the same text again, and PgBouncer gives it the same broken statement. postgres.js also caches the failed statement without columns, so the connection throws `TypeError: Cannot read properties of undefined (reading 'length')` for that text even after the schema is correct again. > - This pull request makes the client recover. When a prepared statement fails with a prepared-statement error, the client stops preparing that text, retries once without preparation where that is safe, and counts it. It also caps the number of prepared texts. > - The benefit is that the flag can be turned on behind a pooler without a deploy migration that breaks reads. Behavior with the flag unset or `false` does not change. ## Linked Issues or Issue Description Refs #17673 (the opt-in), #10795 (the flag), #21707 (the CPU benchmark). **What happened?** With `DATABASE_PREPARED_STATEMENTS=true` behind PgBouncer 1.26.1 in transaction mode (`max_prepared_statements=5`, local PostgreSQL 17), a change of a column type made every later execution of an affected statement fail. A new client process failed in the same way. Measured with the local probe, 20 executions after the change and 10 from a new process: | Change | Before: failures | After: failures | | --- | --- | --- | | `int` to `bigint`, ORM read outside a transaction | 20/20, new process 10/10 (0A000) | 0/20, 0/10 | | `varchar` length change, ORM read | 20/20, 10/10 | 0/20, 0/10 | | Add column, `select *` | 20/20, 10/10 | 0/20, 0/10 | | `int` to `bigint`, first statement of a transaction | 20/20, 10/10 | 0/20, 0/10 | | `int` to `bigint`, second statement of a transaction | 20/20, 10/10 | 1/20, 1/10 | | Table recreated with its original shape, same client | 10/10 (TypeError in `DataRow`) | 0/10 | | Add column, explicit-column ORM read | 0/20 | 0/20 | **Expected behavior** A statement whose pooled prepared statement is stale or lost runs anyway, and the failure does not reach the caller. **Steps to reproduce** 1. Run PostgreSQL behind PgBouncer 1.22 or later with `pool_mode = transaction` and `max_prepared_statements` above 0. 2. Start Paperclip with `DATABASE_PREPARED_STATEMENTS=true` and run a read of a table a few times. 3. Change the type of a column that the read returns, for example `alter table ... alter column n type bigint`. 4. Run the read again from the same process or from a new process: it fails with `cached plan must not change result type`. ## What Changed - `packages/db/src/prepared-queries.ts`: - The prepare decision for each `unsafe` query is made when the driver builds the query, not when the caller creates it, so a query that waits on a busy connection sees a text that was demoted after it was queued. - A failure with 26000 (`prepared statement does not exist`), 42P05 (duplicate statement name), or 0A000 `cached plan must not change result type` demotes the text: the client does not prepare it again in that process. - A pooled statement that fails in that way is retried once with `prepare: false`. Statements in a reserved connection, cursors, `forEach`, `describe` and queries with `onexecute` are not retried. - When the failed statement is the first statement that the transaction ran (only `SET TRANSACTION` before it), the client restarts the transaction on the same connection: `ROLLBACK`, `BEGIN` with the same options, the same `SET TRANSACTION` statements, then the statement unprepared. The callback runs once and receives the result. Tagged statements and statements in savepoints count toward the execution order. The client does not restart when another statement of the transaction is queued, and it never restarts inside a savepoint. - At most 1,000 distinct texts are prepared for each client and its private pools. Texts longer than 8 KB are not prepared. Explicit per-query `prepare` options, simple queries and the flag-off paths are unchanged. - `packages/db/src/client.ts`, `packages/db/src/index.ts`: a client and its dedicated, liveness, checkpoint, file-storage and listener pools share one state. `preparedStatementStats(db)` returns the prepared, unprepared, demoted and retried counts. - `packages/db/src/prepared-queries-postgres.test.ts`: nine tests against embedded PostgreSQL: restart after a stale plan and after a lost statement (the callback runs once and its later work commits), `repeatable read read only` kept after a restart for driver and ORM transactions, no restart after a later statement, after a tagged first statement, or for an out-of-order await, a savepoint whose first statement fails while the outer transaction continues once, the pooled retry with `.values()`, and the cap. - `doc/DATABASE.md`, `docs/deploy/database.md`: the cap and the recovery. ## Verification - Red/green, `pnpm --filter @paperclipai/db exec vitest run src/prepared-queries-postgres.test.ts`. With the old `prepared-queries.ts` (stub stats exports only), the five new tests fail: two transactions fail with the stale-plan and the missing-statement errors, the ORM read fails with 42P05, the invalidated text is prepared again, and 5 texts are prepared where the cap is 2. With this change, 23/23 pass. - `pnpm --filter @paperclipai/db exec vitest run` on `prepared-queries-postgres`, `client-options`, `client`, `dedicated-connection-pool`, `dedicated-connection-admission`, `response-capture-postgres`, `lazy-transaction-queries`, `client-disconnect-replay`: 91/91 pass on the final revision. `tsc --noEmit -p packages/db` passes. - 31 server test files with `DATABASE_PREPARED_STATEMENTS=true` (services, routes, inbox, recovery, heartbeat events, agent auth, prepared-statement helpers): 829/830 pass. The one failure (`execution-workspaces-service`, a git-scan timing test) passes alone with the flag on and off. The machine load average was 9 to 20. - Local PgBouncer probe (not in CI, because CI has no PgBouncer): PgBouncer 1.26.1, PostgreSQL 17, postgres.js 3.4.9 (patched), Node 26.4. The results in the table above, plus 0 failures for 4,800 statements over 60 distinct texts with `max_prepared_statements=5`, 0 failures with `pg_terminate_backend` and `RECONNECT` every 25 statements, and 0 failures for transactions with `SET LOCAL`, pipelines of 20 statements, `pg_advisory_xact_lock` and savepoint rollback. - Through PgBouncer 1.26.1 (transaction mode, `max_prepared_statements=5`, 4 server connections) and PostgreSQL 17, at head 84732c8f67, compared with the master `prepared-queries.ts`. Each row ran 10 times on its own client, warmed before the migration. "Type" is `int` to `bigint` under an ORM read; "star" is an added column under `select *`. Failures before → after: | Row | Type change | Added column, `select *` | | --- | --- | --- | | (a) outside a transaction, same process | 10 → 0 | 10 → 0 | | (b) first statement of an ORM transaction (callback runs once) | 10 → 0 | 10 → 0 | | (b) first statement, `begin isolation level repeatable read read only` | 0 → 0 (the statement was evicted and prepared again) | 10 → 0 | | (b) first statement, ORM `repeatable read`, `read only` (`SET TRANSACTION` repeated) | 10 → 0 | 10 → 0 | | (c) later statement of a transaction, same process | 10 → 1 | 10 → 1 | | (d) new process after the migration: (a) / (b) / (b) driver / (b) ORM / (c) | 10/10/0/0/0 → 0/0/0/0/1 | 10/10/10/10/10 → 0/0/0/0/1 | | (e) same client after the table is restored: (a) / (b) / (b) driver / (b) ORM / (c) | 10/10/0/10/10 → 0 | 10 TypeError each → 0 | Every restarted transaction checked `transaction_isolation = repeatable read` and `transaction_read_only = on` after the restart. The PostgreSQL log shows each restart as `ROLLBACK`, `BEGIN` with the same options, then the statement as an unnamed statement. With 12 clients restarting at once over 2 server connections and 6 clients of background load, 12/12 transactions succeeded. In the 15 restarts I paired by `application_name`, the second `BEGIN` ran on the same server connection as the `ROLLBACK`, because both are sent together. A different server connection is also safe: only `SET TRANSACTION` ran before the restart point, and the client repeats it after the new `BEGIN`. - Silent rollback check through PgBouncer with `max_prepared_statements=2`: 3 clients ran 600 transactions, each with an insert and 6 statements over 40 distinct texts, and more distinct statements between transactions. PgBouncer parsed 6,409 statements on the server for 1,089 client parses, so it evicted and prepared again the shared `commit` (`PGBOUNCER_15`) many times. All 600 `COMMIT`s succeeded, 600/600 rows were present, and no 26000 error was logged. The silent rollback needs a `DEALLOCATE` on the session, which PgBouncer does not allow (it closes the client) and Paperclip does not send. ## Risks - A transaction restart happens below the callback. Code in the callback before its first statement runs once. The restarted transaction has a new transaction id. - A postgres.js behavior that this PR does not change: with driver-level preparation (the flag `true`, or unset for tagged SQL), the tagged `commit` of `sql.begin` is a prepared statement. If something deallocates it on the session, the driver retries `COMMIT` in the aborted transaction, which PostgreSQL turns into a rollback without an error. Through PgBouncer this did not happen in 600 transactions with the `commit` statement evicted repeatedly (see Verification). It needs a manual `DEALLOCATE` on a direct session. - A later statement in a transaction still fails once for each affected text in each process after a migration that changes a column type. The text then runs unprepared. - A demoted text stays unprepared until the process restarts. The cap admits texts in first-use order. - A client that sends `DEALLOCATE ALL` through PgBouncer is disconnected by PgBouncer. Paperclip does not send it. - With the flag unset or `false`, the code path is the same as before. ## Model Used Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, via Claude Code with tool use (shell, local PostgreSQL and PgBouncer in a throwaway container, test execution). ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- doc/DATABASE.md | 20 ++ docs/deploy/database.md | 4 + packages/db/src/client.ts | 15 +- packages/db/src/index.ts | 1 + .../db/src/prepared-queries-postgres.test.ts | 184 +++++++++++++++++- packages/db/src/prepared-queries.ts | 165 ++++++++++++++-- 6 files changed, 375 insertions(+), 14 deletions(-) diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 47f849b72b7..15ecde9114a 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -184,6 +184,26 @@ still apply, including `prepare: false` and simple-protocol execution. With the setting unset, ORM queries keep their existing unprepared behavior and tagged SQL uses the driver default. With `false`, preparation stays disabled. +With `true`, a database client and its private pools prepare at most 1,000 +distinct SQL texts, and never a text longer than 8 KB; other texts run +unprepared. A transaction-mode pooler +(PgBouncer 1.22 or later, Neon `-pooler` hosts) shares prepared statements by +SQL text across clients, so after a migration changes a column type, the +pooler's statement for an affected text keeps failing with `cached plan must +not change result type` for every client and every process. When a statement +fails with that error, `prepared statement does not exist` (26000), or a +duplicate statement name (42P05), the client stops preparing that text for the +rest of the process and retries once without preparation. A pooled statement +is retried on its own. When the failed statement is the first one that the +transaction ran (after `SET TRANSACTION` only), the client rolls back, begins +again with the same options, repeats the `SET TRANSACTION` statements, and runs +the statement unprepared on the same connection. The transaction callback does +not run again and receives the result. A later statement in a transaction, or a +statement in a savepoint, is not retried, so that transaction or savepoint fails +once for each affected text in each process. `preparedStatementStats(db)` +reports the prepared texts and the unprepared, demoted, retried, and restarted +counts. + ## Switching between modes The database mode is controlled by `DATABASE_URL`: diff --git a/docs/deploy/database.md b/docs/deploy/database.md index a61a1b400ce..7d9ea38fe3e 100644 --- a/docs/deploy/database.md +++ b/docs/deploy/database.md @@ -68,6 +68,10 @@ queries inside transactions and reserved or dedicated connections. Check the provider's configuration before you enable it. Preparation caches statement metadata, not query results. Unset preserves the existing ORM behavior and the driver default for tagged SQL. Explicit per-query options still apply. +Each database client prepares at most 1,000 distinct statements and skips +statements longer than 8 KB. If a prepared statement fails because the pooler's copy is +stale after a schema change or was lost, Paperclip stops preparing that +statement and retries it once unprepared. Related optional client tuning: `DATABASE_POOL_MAX`, `DATABASE_IDLE_TIMEOUT_SECONDS`, `DATABASE_CONNECT_TIMEOUT_SECONDS`, `DATABASE_MAX_LIFETIME_SECONDS`, `DATABASE_APPLICATION_NAME`. Driver defaults apply when unset, except that idle pooled connections close after 60 seconds (`DATABASE_IDLE_TIMEOUT_SECONDS=0` keeps them open) and the pool reports `application_name=paperclip`. See [Connection pool settings](#connection-pool-settings). diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index d0bc17061f4..61d2e52b9d4 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url"; import postgres from "postgres"; import { installDatabaseWorkSignals } from "./work-signals.js"; import { installDatabaseResponseCapture } from "./response-capture.js"; -import { installPreparedQueries } from "./prepared-queries.js"; +import { createPreparedQueryState, installPreparedQueries, preparedQueryStats, type PreparedQueryState, type PreparedQueryStats } from "./prepared-queries.js"; import { drizzleWithLazyTransactionQueries } from "./lazy-transaction-queries.js"; import * as schema from "./schema/index.js"; @@ -377,6 +377,13 @@ export function databaseConnectionBudget(options: DatabaseClientOptions = {}) { total: main + privatePools + DATABASE_LISTENER_CONNECTIONS + dedicated }; } +const preparedQueryStates = new WeakMap(); + +export function preparedStatementStats(db: Db): PreparedQueryStats | null { + const state = preparedQueryStates.get(db); + return state ? preparedQueryStats(state) : null; +} + export function dedicatedDbConnectionStats(db: Db): DedicatedDbConnectionStats | null { const set = dedicatedConnectionSets.get(db); return set ? { ...set.stats } : null; @@ -573,7 +580,10 @@ export function createDb(url: string, options?: DatabaseClientOptions) { function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object, dedicatedSet?: DedicatedConnectionSet) { const sql = postgres(url, postgresJsOptions(resolved)); - if (resolved.prepare === true) installPreparedQueries(sql); + const preparedState = resolved.prepare === true + ? (owner && preparedQueryStates.get(owner)) || createPreparedQueryState() + : undefined; + if (preparedState) installPreparedQueries(sql, preparedState); const key = hostPortKeyOrNull(url); if (key) registerClient(key, sql); // A disconnect can lose the response after a statement has committed. @@ -583,6 +593,7 @@ function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: ob const db = installDatabaseWorkSignals(drizzleWithLazyTransactionQueries(sql, schema), owner); installDatabaseResponseCapture(db, schema); markOpenTransactions(db); + if (preparedState) preparedQueryStates.set(db, preparedState); const ownerSet = owner ? dedicatedConnectionSets.get(owner) : undefined; if (dedicatedSet) { trackClientEnd(db, () => { dedicatedSet.ended.add(db); }); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index b8f1421ed29..c5b68087976 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -8,6 +8,7 @@ export { dedicatedDbConnectionStats, type DedicatedDbConnectionStats, type DedicatedDbConnectionOptions, + preparedStatementStats, databaseConnectionBudget, PRIVATE_POOL_MAX_CONNECTIONS, createOwnedDatabaseListener, diff --git a/packages/db/src/prepared-queries-postgres.test.ts b/packages/db/src/prepared-queries-postgres.test.ts index edef6dc060e..c021db95429 100644 --- a/packages/db/src/prepared-queries-postgres.test.ts +++ b/packages/db/src/prepared-queries-postgres.test.ts @@ -1,11 +1,14 @@ import { eq, sql } from "drizzle-orm"; +import postgres from "postgres"; import { integer, jsonb, pgTable, text } from "drizzle-orm/pg-core"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { closeLivenessDb, createDb, databaseClientOptionsFromEnv, livenessDb, withDedicatedDbConnection, type DatabaseClientOptions, type Db } from "./client.js"; +import { closeLivenessDb, createDb, databaseClientOptionsFromEnv, livenessDb, preparedStatementStats, withDedicatedDbConnection, type DatabaseClientOptions, type Db } from "./client.js"; +import { createPreparedQueryState, installPreparedQueries, preparedQueryStats } from "./prepared-queries.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; import { runEventPayload } from "./run-event-payload.js"; const records = pgTable("prepared_query_fixture", { id: text("id").primaryKey(), companyId: text("company_id").notNull(), value: integer("value").notNull(), payload: jsonb("payload") }); +const stale = pgTable("prepared_stale_fixture", { id: text("id").primaryKey(), value: integer("value").notNull() }); const encoded = pgTable("prepared_encoded_fixture", { id: text("id").primaryKey(), payload: runEventPayload("payload") }); const support = await getEmbeddedPostgresTestSupport(); @@ -19,6 +22,8 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre await setup.execute(sql`create table prepared_query_fixture(id text primary key, company_id text not null, value integer not null, payload jsonb)`); await setup.execute(sql`create table prepared_encoded_fixture(id text primary key, payload jsonb)`); await setup.execute(sql`create table prepared_schema_change_fixture(id text primary key)`); + await setup.execute(sql`create table prepared_stale_fixture(id text primary key, value integer not null)`); + await setup.execute(sql`insert into prepared_stale_fixture(id, value) values ('stale', 5)`); await setup.execute(sql`insert into prepared_schema_change_fixture(id) values ('retained')`); }, 90_000); beforeEach(async () => { @@ -172,4 +177,181 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre await setup.execute(sql`alter table prepared_schema_change_fixture drop column if exists extra`); } }); + + async function changeStaleValueType(type: "bigint" | "integer") { + await setup.execute(sql.raw(`alter table prepared_stale_fixture alter column value type ${type}`)); + } + + it("restarts a transaction whose first statement meets an invalidated plan without running the callback again", async () => { + const db = fixture({ prepare: true }); + let calls = 0; + const read = () => db.transaction(tx => { calls++; return tx.select().from(stale).where(eq(stale.id, "stale")); }); + expect(await read()).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + try { + expect(await read()).toEqual([{ id: "stale", value: 5 }]); + expect(calls).toBe(2); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 1, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("restarts a transaction whose first statement lost its prepared statement and keeps its later work", async () => { + const db = fixture({ prepare: true }); + const run = (value: number) => db.transaction(async tx => { + const [row] = await tx.select({ value: records.value }).from(records).where(eq(records.id, "a")); + await tx.execute(sql`update prepared_query_fixture set value = ${value} where id = ${"b"} ${sql.raw(`/* run ${value} */`)}`); + return row; + }); + expect(await run(3)).toEqual({ value: 1 }); + const [{ name }] = await statements(db, 'select "value" from "prepared_query_fixture"%'); + await db.$client.unsafe(`deallocate "${String(name)}"`); + expect(await run(4)).toEqual({ value: 1 }); + expect(await setup.select({ value: records.value }).from(records).where(eq(records.id, "b"))).toEqual([{ value: 4 }]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 1, demoted: 1 }); + }); + + it("keeps the isolation level and access mode of a restarted transaction", async () => { + const db = fixture({ prepare: true }); + const settings = "select current_setting('transaction_isolation') as isolation, current_setting('transaction_read_only') as read_only, $1::integer as marker"; + const snapshot = { isolation: "repeatable read", read_only: "on", marker: 1 }; + const readDriver = () => db.$client.begin("isolation level repeatable read read only", async tx => { + const rows = await tx.unsafe("select id, value from prepared_stale_fixture where id = $1", ["stale"]); + return [rows, await tx.unsafe(settings, [1])]; + }); + const readOrm = () => db.transaction(async tx => [ + await tx.select().from(stale).where(eq(stale.id, "stale")), + await tx.execute(sql`select current_setting('transaction_isolation') as isolation, current_setting('transaction_read_only') as read_only, ${1}::integer as marker`), + ], { isolationLevel: "repeatable read", accessMode: "read only" }); + expect(await readDriver()).toEqual([[{ id: "stale", value: 5 }], [snapshot]]); + expect(await readOrm()).toEqual([[{ id: "stale", value: 5 }], [snapshot]]); + await changeStaleValueType("bigint"); + try { + expect(await readDriver()).toEqual([[{ id: "stale", value: "5" }], [snapshot]]); + expect(await readOrm()).toEqual([[{ id: "stale", value: 5 }], [snapshot]]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 2, demoted: 2 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("does not restart a transaction after its first statement and runs the invalidated text unprepared", async () => { + const db = fixture({ prepare: true }); + let calls = 0; + const update = () => db.transaction(async tx => { + calls++; + await tx.update(records).set({ value: 7 }).where(eq(records.id, "b")); + return tx.select().from(stale).where(eq(stale.id, "stale")); + }); + expect(await update()).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + try { + await expect(update()).rejects.toMatchObject({ cause: { code: "0A000" } }); + expect(calls).toBe(2); + expect(await update()).toEqual([{ id: "stale", value: 5 }]); + expect(await statements(db, '%"prepared_stale_fixture"%')).toHaveLength(1); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("counts a tagged statement as the first statement of a transaction", async () => { + const db = fixture({ prepare: true }); + const read = "select id, value from prepared_stale_fixture where id = $1"; + expect(await db.$client.unsafe(read, ["stale"])).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + try { + await expect(db.$client.begin(async tx => { + await tx`update prepared_query_fixture set value = value + 1 where id = 'a'`; + return tx.unsafe(read, ["stale"]); + })).rejects.toMatchObject({ code: "0A000" }); + expect(await setup.select({ value: records.value }).from(records).where(eq(records.id, "a"))).toEqual([{ value: 1 }]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("rolls back only the savepoint whose first statement fails and continues the outer transaction once", async () => { + const db = fixture({ prepare: true }); + const nestedRead = (tx: Parameters[0]>[0]) => + tx.transaction(nested => nested.select().from(stale).where(eq(stale.id, "stale"))); + await db.transaction(async tx => { await nestedRead(tx); }); + await changeStaleValueType("bigint"); + let outerCalls = 0; + try { + const result = await db.transaction(async tx => { + outerCalls++; + await tx.execute(sql`update prepared_query_fixture set value = value + 10 where id = ${"a"}`); + const nested = await nestedRead(tx).catch((error: { cause?: { code?: string } }) => error.cause?.code); + await tx.execute(sql`update prepared_query_fixture set value = value + 100 where id = ${"b"}`); + return [nested, await tx.select().from(stale).where(eq(stale.id, "stale"))]; + }); + expect(result).toEqual(["0A000", [{ id: "stale", value: 5 }]]); + expect(outerCalls).toBe(1); + expect(await setup.select({ id: records.id, value: records.value }).from(records).orderBy(records.id)) + .toEqual([{ id: "a", value: 11 }, { id: "b", value: 102 }]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + async function occupyNextStatementName(db: Db) { + const names = (await statements(db, "%")).map(row => String(row.name)); + const prefix = names.find(name => /\d$/.test(name) && !name.includes(" "))!.replace(/\d+$/, ""); + const counters = names.filter(name => name.startsWith(prefix)).map(name => Number(name.slice(prefix.length))); + await db.$client.unsafe(`prepare "${prefix}${Math.max(...counters) + 1}" as select 1`, [], { prepare: false }); + } + + it("retries a pooled statement unprepared when its prepared statement cannot be created", async () => { + const db = fixture({ prepare: true }); + await statements(db, "%"); + await occupyNextStatementName(db); + expect(await db.select({ value: records.value }).from(records).where(eq(records.id, "b"))).toEqual([{ value: 2 }]); + await occupyNextStatementName(db); + expect(await db.$client.unsafe("select $1::integer as raw_retry_marker", [3]).values()).toEqual([[3]]); + expect(await statements(db)).toHaveLength(0); + expect(await statements(db, "%raw_retry_marker%")).toHaveLength(0); + expect(preparedStatementStats(db)).toMatchObject({ retried: 2, demoted: 2 }); + }); + + it("does not restart a transaction when a query created first executes after another statement", async () => { + const db = fixture({ prepare: true }); + const read = "select id, value from prepared_stale_fixture where id = $1"; + expect(await db.$client.unsafe(read, ["stale"])).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + let calls = 0; + try { + await expect(db.$client.begin(async tx => { + calls++; + const saved = tx.unsafe(read, ["stale"]); + await tx.unsafe("select $1::integer as ordering_marker", [1]); + return await saved; + })).rejects.toMatchObject({ code: "0A000" }); + expect(calls).toBe(1); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("prepares at most the configured number of distinct texts and skips long texts", async () => { + const raw = postgres(temporary.connectionString, { max: 1, prepare: true, onnotice: () => {} }); + const state = createPreparedQueryState({ limit: 2, maxLength: 80 }); + installPreparedQueries(raw, state); + try { + for (const marker of ["cap_a", "cap_b", "cap_c", "cap_d"]) { + expect(await raw.unsafe(`select $1::integer as ${marker}`, [1])).toEqual([{ [marker]: 1 }]); + } + const long = `select $1::integer as cap_long /* ${"x".repeat(80)} */`; + expect(await raw.unsafe(long, [2])).toEqual([{ cap_long: 2 }]); + expect((await raw.unsafe("select name from pg_prepared_statements where statement like 'select $1::integer as cap_%'", [], { prepare: false })).length).toBe(2); + expect(preparedQueryStats(state)).toMatchObject({ texts: 2, unprepared: 3 }); + } finally { + await raw.end(); + } + }); }); diff --git a/packages/db/src/prepared-queries.ts b/packages/db/src/prepared-queries.ts index 7622722027a..0efa5ac1d97 100644 --- a/packages/db/src/prepared-queries.ts +++ b/packages/db/src/prepared-queries.ts @@ -1,9 +1,108 @@ type Method = (this: unknown, ...args: unknown[]) => unknown; -export function installPreparedQueries(client: unknown): void { +export const PREPARED_QUERY_TEXT_LIMIT = 1_000; +export const PREPARED_QUERY_TEXT_MAX_LENGTH = 8_192; + +export type PreparedQueryStats = { + texts: number; + unprepared: number; + demoted: number; + retried: number; + restartedTransactions: number; +}; + +export type PreparedQueryState = { + limit: number; + maxLength: number; + texts: Set; + demoted: Set; + stats: Omit; +}; + +type Counts = { created: number; executed: number; setup: string[] }; +type Scope = { retry: "statement" | "restart" | "none"; begin?: string; counts: Counts }; + +type PendingQuery = { + options: Record; + reject(error: unknown): void; + resolve(value: unknown): void; + isRaw?: unknown; + cursorFn?: unknown; + forEachFn?: unknown; + onlyDescribe?: unknown; + then(onFulfilled: (value: unknown) => void, onRejected: (error: unknown) => void): unknown; +}; + +export function createPreparedQueryState(options: { limit?: number; maxLength?: number } = {}): PreparedQueryState { + return { + limit: options.limit ?? PREPARED_QUERY_TEXT_LIMIT, + maxLength: options.maxLength ?? PREPARED_QUERY_TEXT_MAX_LENGTH, + texts: new Set(), + demoted: new Set(), + stats: { unprepared: 0, demoted: 0, retried: 0, restartedTransactions: 0 }, + }; +} + +export function preparedQueryStats(state: PreparedQueryState): PreparedQueryStats { + return { texts: state.texts.size, ...state.stats }; +} + +export function isPreparedStatementFailure(error: unknown): boolean { + if (typeof error !== "object" || error === null) return false; + const { code, routine, message } = error as { code?: unknown; routine?: unknown; message?: unknown }; + if (code === "26000" || code === "42P05") return true; + return code === "0A000" && (routine === "RevalidateCachedQuery" + || (typeof message === "string" && message.includes("cached plan must not change result type"))); +} + +function shouldPrepare(state: PreparedQueryState, text: string): boolean { + if (state.texts.has(text)) return true; + if (state.demoted.has(text) || text.length > state.maxLength || state.texts.size >= state.limit) { + state.stats.unprepared++; + return false; + } + state.texts.add(text); + return true; +} + +function demote(state: PreparedQueryState, text: string): void { + if (state.demoted.has(text)) return; + state.texts.delete(text); + state.demoted.add(text); + state.stats.demoted++; +} + +const TRANSACTION_SETUP = /^\s*set\s+transaction\b/i; + +function scope(retry: Scope["retry"], begin?: string, counts: Counts = { created: 0, executed: 0, setup: [] }): Scope { + return { retry, begin, counts }; +} + +function track(query: PendingQuery, current: Counts, text: string | undefined, decide: () => unknown) { + current.created++; + let explicit: unknown = "prepare" in query.options ? query.options.prepare : undefined; + const execution = { started: false, first: false }; + Object.defineProperty(query.options, "prepare", { + enumerable: true, + configurable: true, + get: () => { + if (!execution.started) { + execution.started = true; + execution.first = current.executed === current.setup.length; + current.executed++; + if (execution.first && text !== undefined && query.options.simple && TRANSACTION_SETUP.test(text)) current.setup.push(text); + } + return explicit ?? decide(); + }, + set: (value: unknown) => { explicit = value; }, + }); + return { execution, explicit: () => explicit }; +} + +export function installPreparedQueries(client: unknown, state: PreparedQueryState = createPreparedQueryState()): void { const visited = new WeakSet(); - function visit(value: unknown): void { + function visit(value: unknown, current: Scope): void { if ((typeof value !== "object" || value === null) && typeof value !== "function") return; if (visited.has(value)) return; visited.add(value); @@ -12,22 +111,52 @@ export function installPreparedQueries(client: unknown): void { if (typeof unsafe === "function") { target.unsafe = function (this: unknown, ...args: unknown[]) { const index = args.length === 2 && !Array.isArray(args[1]) ? 1 : 2; - const options = args[index]; + const options = args[index] as Record | undefined; if (options !== undefined && (options === null || typeof options !== "object")) { return Reflect.apply(unsafe, this, args); } if (index === 2 && args.length < 2) args[1] = []; + const text = args[0]; args[index] = { prepare: true, ...options }; - return Reflect.apply(unsafe, this, args); - }; + if (typeof text !== "string") return Reflect.apply(unsafe, this, args); + const query = Reflect.apply(unsafe, this, args) as PendingQuery; + const caller = (options !== undefined && "prepare" in options) || query.options.simple; + if (!caller) delete query.options.prepare; + const tracked = track(query, current.counts, text, () => shouldPrepare(state, text)); + const reject = query.reject; + query.reject = (error: unknown) => { + if (tracked.explicit() !== undefined || !isPreparedStatementFailure(error)) return reject(error); + demote(state, text); + if (query.cursorFn || query.forEachFn || query.onlyDescribe || options?.onexecute) return reject(error); + const params = index === 2 ? args[1] : []; + const unprepared = { ...options, prepare: false }; + if (current.retry === "statement") { + state.stats.retried++; + const retry = Reflect.apply(unsafe, this, [text, params, unprepared]) as PendingQuery; + retry.isRaw = query.isRaw; + retry.then(query.resolve, reject); + return; + } + if (current.retry !== "restart" || !tracked.execution.first || current.counts.created !== current.counts.executed) return reject(error); + state.stats.restartedTransactions++; + const run = (sql: string, ...rest: unknown[]) => Reflect.apply(unsafe, this, [sql, ...rest]) as PendingQuery; + const steps = [run("rollback"), run(`begin ${current.begin ?? ""}`.trim()), ...current.counts.setup.map(setup => run(setup))]; + const retry = run(text, params, unprepared); + retry.isRaw = query.isRaw; + Promise.all([...steps, retry]).then(results => query.resolve(results[results.length - 1]), reject); + }; + return query; + } satisfies Method; } - for (const method of ["begin", "savepoint"]) { + for (const method of ["begin", "savepoint"] as const) { const original = target[method]; if (typeof original !== "function") continue; target[method] = function (this: unknown, ...args: unknown[]) { - const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...inner: unknown[]) { - visit(inner[0]); - return Reflect.apply(arg, this, inner); + const begin = method === "begin" && typeof args[0] === "string" ? args[0].replace(/[^a-z ]/ig, "") : undefined; + const inner = method === "begin" ? scope("restart", begin) : scope("none", undefined, current.counts); + const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...params: unknown[]) { + visit(params[0], inner); + return Reflect.apply(arg, this, [counted(params[0], inner), ...params.slice(1)]); }); return Reflect.apply(original, this, wrapped); } satisfies Method; @@ -36,12 +165,26 @@ export function installPreparedQueries(client: unknown): void { if (typeof reserve === "function") { target.reserve = function (this: unknown, ...args: unknown[]) { return Promise.resolve(Reflect.apply(reserve, this, args)).then(reserved => { - visit(reserved); + visit(reserved, scope("none")); return reserved; }); } satisfies Method; } } - visit(client); + function counted(sql: unknown, current: Scope): unknown { + if (typeof sql !== "function") return sql; + return new Proxy(sql, { + apply(target, self, args) { + const result = Reflect.apply(target, self, args) as unknown; + const strings = args[0] as { raw?: unknown } | undefined; + if (Array.isArray(strings?.raw) && result instanceof Promise && "options" in result) { + track(result as unknown as PendingQuery, current.counts, undefined, () => true); + } + return result; + }, + }); + } + + visit(client, scope("statement")); }