diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 47f849b72b7..21445e975b2 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -184,6 +184,21 @@ still apply, including `prepare: false` and simple-protocol execution. With the setting unset, ORM queries keep their existing unprepared behavior and tagged SQL uses the driver default. With `false`, preparation stays disabled. +With `true`, a database client and its private pools prepare at most 1,000 +distinct SQL texts, and never a text longer than 8 KB; other texts run +unprepared. A transaction-mode pooler +(PgBouncer 1.22 or later, Neon `-pooler` hosts) shares prepared statements by +SQL text across clients, so after a migration changes a column type, the +pooler's statement for an affected text keeps failing with `cached plan must +not change result type` for every client and every process. When a statement +fails with that error, `prepared statement does not exist` (26000), or a +duplicate statement name (42P05), the client stops preparing that text for the +rest of the process and retries once without preparation. The retry applies to +pooled statements and to a transaction whose first statement failed. A later +statement in a transaction cannot be retried, so that transaction fails once +for each affected text in each process. `preparedStatementStats(db)` reports +the prepared texts and the unprepared, demoted, and retried counts. + ## Switching between modes The database mode is controlled by `DATABASE_URL`: diff --git a/docs/deploy/database.md b/docs/deploy/database.md index a61a1b400ce..7d9ea38fe3e 100644 --- a/docs/deploy/database.md +++ b/docs/deploy/database.md @@ -68,6 +68,10 @@ queries inside transactions and reserved or dedicated connections. Check the provider's configuration before you enable it. Preparation caches statement metadata, not query results. Unset preserves the existing ORM behavior and the driver default for tagged SQL. Explicit per-query options still apply. +Each database client prepares at most 1,000 distinct statements and skips +statements longer than 8 KB. If a prepared statement fails because the pooler's copy is +stale after a schema change or was lost, Paperclip stops preparing that +statement and retries it once unprepared. Related optional client tuning: `DATABASE_POOL_MAX`, `DATABASE_IDLE_TIMEOUT_SECONDS`, `DATABASE_CONNECT_TIMEOUT_SECONDS`, `DATABASE_MAX_LIFETIME_SECONDS`, `DATABASE_APPLICATION_NAME`. Driver defaults apply when unset, except that idle pooled connections close after 60 seconds (`DATABASE_IDLE_TIMEOUT_SECONDS=0` keeps them open) and the pool reports `application_name=paperclip`. See [Connection pool settings](#connection-pool-settings). diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index d0bc17061f4..61d2e52b9d4 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url"; import postgres from "postgres"; import { installDatabaseWorkSignals } from "./work-signals.js"; import { installDatabaseResponseCapture } from "./response-capture.js"; -import { installPreparedQueries } from "./prepared-queries.js"; +import { createPreparedQueryState, installPreparedQueries, preparedQueryStats, type PreparedQueryState, type PreparedQueryStats } from "./prepared-queries.js"; import { drizzleWithLazyTransactionQueries } from "./lazy-transaction-queries.js"; import * as schema from "./schema/index.js"; @@ -377,6 +377,13 @@ export function databaseConnectionBudget(options: DatabaseClientOptions = {}) { total: main + privatePools + DATABASE_LISTENER_CONNECTIONS + dedicated }; } +const preparedQueryStates = new WeakMap(); + +export function preparedStatementStats(db: Db): PreparedQueryStats | null { + const state = preparedQueryStates.get(db); + return state ? preparedQueryStats(state) : null; +} + export function dedicatedDbConnectionStats(db: Db): DedicatedDbConnectionStats | null { const set = dedicatedConnectionSets.get(db); return set ? { ...set.stats } : null; @@ -573,7 +580,10 @@ export function createDb(url: string, options?: DatabaseClientOptions) { function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object, dedicatedSet?: DedicatedConnectionSet) { const sql = postgres(url, postgresJsOptions(resolved)); - if (resolved.prepare === true) installPreparedQueries(sql); + const preparedState = resolved.prepare === true + ? (owner && preparedQueryStates.get(owner)) || createPreparedQueryState() + : undefined; + if (preparedState) installPreparedQueries(sql, preparedState); const key = hostPortKeyOrNull(url); if (key) registerClient(key, sql); // A disconnect can lose the response after a statement has committed. @@ -583,6 +593,7 @@ function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: ob const db = installDatabaseWorkSignals(drizzleWithLazyTransactionQueries(sql, schema), owner); installDatabaseResponseCapture(db, schema); markOpenTransactions(db); + if (preparedState) preparedQueryStates.set(db, preparedState); const ownerSet = owner ? dedicatedConnectionSets.get(owner) : undefined; if (dedicatedSet) { trackClientEnd(db, () => { dedicatedSet.ended.add(db); }); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 9b84a1ce668..251f020aa99 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -8,6 +8,7 @@ export { dedicatedDbConnectionStats, type DedicatedDbConnectionStats, type DedicatedDbConnectionOptions, + preparedStatementStats, databaseConnectionBudget, PRIVATE_POOL_MAX_CONNECTIONS, createOwnedDatabaseListener, diff --git a/packages/db/src/prepared-queries-postgres.test.ts b/packages/db/src/prepared-queries-postgres.test.ts index edef6dc060e..e5af87fe7f7 100644 --- a/packages/db/src/prepared-queries-postgres.test.ts +++ b/packages/db/src/prepared-queries-postgres.test.ts @@ -1,11 +1,14 @@ import { eq, sql } from "drizzle-orm"; +import postgres from "postgres"; import { integer, jsonb, pgTable, text } from "drizzle-orm/pg-core"; import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest"; -import { closeLivenessDb, createDb, databaseClientOptionsFromEnv, livenessDb, withDedicatedDbConnection, type DatabaseClientOptions, type Db } from "./client.js"; +import { closeLivenessDb, createDb, databaseClientOptionsFromEnv, livenessDb, preparedStatementStats, withDedicatedDbConnection, type DatabaseClientOptions, type Db } from "./client.js"; +import { createPreparedQueryState, installPreparedQueries, preparedQueryStats } from "./prepared-queries.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; import { runEventPayload } from "./run-event-payload.js"; const records = pgTable("prepared_query_fixture", { id: text("id").primaryKey(), companyId: text("company_id").notNull(), value: integer("value").notNull(), payload: jsonb("payload") }); +const stale = pgTable("prepared_stale_fixture", { id: text("id").primaryKey(), value: integer("value").notNull() }); const encoded = pgTable("prepared_encoded_fixture", { id: text("id").primaryKey(), payload: runEventPayload("payload") }); const support = await getEmbeddedPostgresTestSupport(); @@ -19,6 +22,8 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre await setup.execute(sql`create table prepared_query_fixture(id text primary key, company_id text not null, value integer not null, payload jsonb)`); await setup.execute(sql`create table prepared_encoded_fixture(id text primary key, payload jsonb)`); await setup.execute(sql`create table prepared_schema_change_fixture(id text primary key)`); + await setup.execute(sql`create table prepared_stale_fixture(id text primary key, value integer not null)`); + await setup.execute(sql`insert into prepared_stale_fixture(id, value) values ('stale', 5)`); await setup.execute(sql`insert into prepared_schema_change_fixture(id) values ('retained')`); }, 90_000); beforeEach(async () => { @@ -172,4 +177,81 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre await setup.execute(sql`alter table prepared_schema_change_fixture drop column if exists extra`); } }); + + async function changeStaleValueType(type: "bigint" | "integer") { + await setup.execute(sql.raw(`alter table prepared_stale_fixture alter column value type ${type}`)); + } + + it("retries a transaction whose first statement meets a plan invalidated by a schema change", async () => { + const db = fixture({ prepare: true }); + const read = () => db.transaction(tx => tx.select().from(stale).where(eq(stale.id, "stale"))); + expect(await read()).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + try { + expect(await read()).toEqual([{ id: "stale", value: 5 }]); + expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 1, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("retries a transaction whose first statement lost its prepared statement", async () => { + const db = fixture({ prepare: true }); + const read = () => db.transaction(tx => tx.select({ value: records.value }).from(records).where(eq(records.id, "a"))); + expect(await read()).toEqual([{ value: 1 }]); + await db.$client.unsafe("deallocate all"); + expect(await read()).toEqual([{ value: 1 }]); + expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 1, demoted: 1 }); + }); + + it("does not replay a transaction after its first statement and runs the invalidated text unprepared", async () => { + const db = fixture({ prepare: true }); + let calls = 0; + const update = () => db.transaction(async tx => { + calls++; + await tx.update(records).set({ value: 7 }).where(eq(records.id, "b")); + return tx.select().from(stale).where(eq(stale.id, "stale")); + }); + expect(await update()).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + try { + await expect(update()).rejects.toMatchObject({ cause: { code: "0A000" } }); + expect(calls).toBe(2); + expect(await update()).toEqual([{ id: "stale", value: 5 }]); + expect(await statements(db, '%"prepared_stale_fixture"%')).toHaveLength(1); + expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("retries a pooled statement unprepared when its prepared statement cannot be created", async () => { + const db = fixture({ prepare: true }); + await statements(db, "%"); + const names = (await statements(db, "%")).map(row => String(row.name)); + const prefix = names[0].replace(/\d+$/, ""); + const next = `${prefix}${Math.max(...names.map(name => Number(name.slice(prefix.length)))) + 1}`; + await db.$client.unsafe(`prepare "${next}" as select 1`, [], { prepare: false }); + expect(await db.select({ value: records.value }).from(records).where(eq(records.id, "b"))).toEqual([{ value: 2 }]); + expect(await db.$client.unsafe("select $1::integer as raw_retry_marker", [3]).values()).toEqual([[3]]); + expect(await statements(db)).toHaveLength(0); + expect(preparedStatementStats(db)).toMatchObject({ retried: 1, demoted: 1 }); + }); + + it("prepares at most the configured number of distinct texts and skips long texts", async () => { + const raw = postgres(temporary.connectionString, { max: 1, prepare: true, onnotice: () => {} }); + const state = createPreparedQueryState({ limit: 2, maxLength: 80 }); + installPreparedQueries(raw, state); + try { + for (const marker of ["cap_a", "cap_b", "cap_c", "cap_d"]) { + expect(await raw.unsafe(`select $1::integer as ${marker}`, [1])).toEqual([{ [marker]: 1 }]); + } + const long = `select $1::integer as cap_long /* ${"x".repeat(80)} */`; + expect(await raw.unsafe(long, [2])).toEqual([{ cap_long: 2 }]); + expect((await raw.unsafe("select name from pg_prepared_statements where statement like 'select $1::integer as cap_%'", [], { prepare: false })).length).toBe(2); + expect(preparedQueryStats(state)).toMatchObject({ texts: 2, unprepared: 3 }); + } finally { + await raw.end(); + } + }); }); diff --git a/packages/db/src/prepared-queries.ts b/packages/db/src/prepared-queries.ts index 7622722027a..e2d13f5c3b2 100644 --- a/packages/db/src/prepared-queries.ts +++ b/packages/db/src/prepared-queries.ts @@ -1,9 +1,80 @@ type Method = (this: unknown, ...args: unknown[]) => unknown; -export function installPreparedQueries(client: unknown): void { +export const PREPARED_QUERY_TEXT_LIMIT = 1_000; +export const PREPARED_QUERY_TEXT_MAX_LENGTH = 8_192; + +export type PreparedQueryStats = { + texts: number; + unprepared: number; + demoted: number; + retried: number; + retriedTransactions: number; +}; + +export type PreparedQueryState = { + limit: number; + maxLength: number; + texts: Set; + demoted: Set; + stats: Omit; +}; + +type Scope = { retry: "statement" | "first" | "none"; statements: number; firstFailure?: unknown }; + +type PendingQuery = { + options: Record; + reject(error: unknown): void; + resolve(value: unknown): void; + isRaw?: unknown; + cursorFn?: unknown; + forEachFn?: unknown; + onlyDescribe?: unknown; + then(onFulfilled: (value: unknown) => void, onRejected: (error: unknown) => void): unknown; +}; + +export function createPreparedQueryState(options: { limit?: number; maxLength?: number } = {}): PreparedQueryState { + return { + limit: options.limit ?? PREPARED_QUERY_TEXT_LIMIT, + maxLength: options.maxLength ?? PREPARED_QUERY_TEXT_MAX_LENGTH, + texts: new Set(), + demoted: new Set(), + stats: { unprepared: 0, demoted: 0, retried: 0, retriedTransactions: 0 }, + }; +} + +export function preparedQueryStats(state: PreparedQueryState): PreparedQueryStats { + return { texts: state.texts.size, ...state.stats }; +} + +export function isPreparedStatementFailure(error: unknown): boolean { + if (typeof error !== "object" || error === null) return false; + const { code, routine, message } = error as { code?: unknown; routine?: unknown; message?: unknown }; + if (code === "26000" || code === "42P05") return true; + return code === "0A000" && (routine === "RevalidateCachedQuery" + || (typeof message === "string" && message.includes("cached plan must not change result type"))); +} + +function shouldPrepare(state: PreparedQueryState, text: string): boolean { + if (state.texts.has(text)) return true; + if (state.demoted.has(text) || text.length > state.maxLength || state.texts.size >= state.limit) { + state.stats.unprepared++; + return false; + } + state.texts.add(text); + return true; +} + +function demote(state: PreparedQueryState, text: string): void { + if (state.demoted.has(text)) return; + state.texts.delete(text); + state.demoted.add(text); + state.stats.demoted++; +} + +export function installPreparedQueries(client: unknown, state: PreparedQueryState = createPreparedQueryState()): void { const visited = new WeakSet(); - function visit(value: unknown): void { + function visit(value: unknown, scope: Scope): void { if ((typeof value !== "object" || value === null) && typeof value !== "function") return; if (visited.has(value)) return; visited.add(value); @@ -12,36 +83,77 @@ export function installPreparedQueries(client: unknown): void { if (typeof unsafe === "function") { target.unsafe = function (this: unknown, ...args: unknown[]) { const index = args.length === 2 && !Array.isArray(args[1]) ? 1 : 2; - const options = args[index]; + const options = args[index] as Record | undefined; if (options !== undefined && (options === null || typeof options !== "object")) { return Reflect.apply(unsafe, this, args); } if (index === 2 && args.length < 2) args[1] = []; - args[index] = { prepare: true, ...options }; - return Reflect.apply(unsafe, this, args); - }; + const position = ++scope.statements; + const text = args[0]; + if (typeof text !== "string" || (options !== undefined && "prepare" in options)) { + args[index] = { prepare: true, ...options }; + return Reflect.apply(unsafe, this, args); + } + args[index] = { ...options }; + const query = Reflect.apply(unsafe, this, args) as PendingQuery; + if (query.options.simple) { + query.options.prepare = true; + return query; + } + let explicit: unknown; + Object.defineProperty(query.options, "prepare", { + enumerable: true, + configurable: true, + get: () => explicit ?? shouldPrepare(state, text), + set: (value: unknown) => { explicit = value; }, + }); + const reject = query.reject; + query.reject = (error: unknown) => { + if (explicit !== undefined || !isPreparedStatementFailure(error)) return reject(error); + demote(state, text); + if (scope.retry === "first" && position === 1) scope.firstFailure = error; + if (scope.retry !== "statement" || query.cursorFn || query.forEachFn || query.onlyDescribe || options?.onexecute) { + return reject(error); + } + state.stats.retried++; + const retry = Reflect.apply(unsafe, this, [text, index === 2 ? args[1] : [], { ...options, prepare: false }]) as PendingQuery; + retry.isRaw = query.isRaw; + retry.then(query.resolve, reject); + }; + return query; + } satisfies Method; } for (const method of ["begin", "savepoint"]) { const original = target[method]; if (typeof original !== "function") continue; target[method] = function (this: unknown, ...args: unknown[]) { - const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...inner: unknown[]) { - visit(inner[0]); - return Reflect.apply(arg, this, inner); - }); - return Reflect.apply(original, this, wrapped); + const attempt = (retried: boolean): Promise => { + const inner: Scope = { retry: "first", statements: 0 }; + const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...params: unknown[]) { + visit(params[0], inner); + return Reflect.apply(arg, this, params); + }); + return Promise.resolve(Reflect.apply(original, this, wrapped)).catch((error: unknown) => { + const failure = inner.firstFailure; + if (retried || failure === undefined + || (error !== failure && (error as { cause?: unknown } | null)?.cause !== failure)) throw error; + state.stats.retriedTransactions++; + return attempt(true); + }); + }; + return attempt(false); } satisfies Method; } const reserve = target.reserve; if (typeof reserve === "function") { target.reserve = function (this: unknown, ...args: unknown[]) { return Promise.resolve(Reflect.apply(reserve, this, args)).then(reserved => { - visit(reserved); + visit(reserved, { retry: "none", statements: 0 }); return reserved; }); } satisfies Method; } } - visit(client); + visit(client, { retry: "statement", statements: 0 }); }