mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 20:12:20 +01:00
fix(db): recover prepared statements that a pooler or schema change invalidates
With DATABASE_PREPARED_STATEMENTS=true behind a transaction-mode pooler (PgBouncer 1.22+, Neon -pooler hosts), a migration that changes a column type makes the pooler's shared prepared statement fail for every client and process with "cached plan must not change result type", and postgres.js then caches a statement without columns, so later executions on that connection throw a TypeError even after the schema is fixed. When a prepared statement fails with 26000, 42P05, or that 0A000 error, stop preparing its text for the rest of the process (decided at execution, so queued queries see it), retry pooled statements once unprepared, and retry a transaction whose first statement failed. Cap prepared texts at 1,000 per client and skip texts over 8 KB. Expose the counts through preparedStatementStats. Behaviour with the flag unset or false is unchanged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8adf0a463a
commit
f9b5ddc93b
6 files changed
+241
-16
No files matched your search
@@ -184,6 +184,21 @@ still apply, including `prepare: false` and simple-protocol execution. With the
|
||||
setting unset, ORM queries keep their existing unprepared behavior and tagged
|
||||
SQL uses the driver default. With `false`, preparation stays disabled.
|
||||
|
||||
With `true`, a database client and its private pools prepare at most 1,000
|
||||
distinct SQL texts, and never a text longer than 8 KB; other texts run
|
||||
unprepared. A transaction-mode pooler
|
||||
(PgBouncer 1.22 or later, Neon `-pooler` hosts) shares prepared statements by
|
||||
SQL text across clients, so after a migration changes a column type, the
|
||||
pooler's statement for an affected text keeps failing with `cached plan must
|
||||
not change result type` for every client and every process. When a statement
|
||||
fails with that error, `prepared statement does not exist` (26000), or a
|
||||
duplicate statement name (42P05), the client stops preparing that text for the
|
||||
rest of the process and retries once without preparation. The retry applies to
|
||||
pooled statements and to a transaction whose first statement failed. A later
|
||||
statement in a transaction cannot be retried, so that transaction fails once
|
||||
for each affected text in each process. `preparedStatementStats(db)` reports
|
||||
the prepared texts and the unprepared, demoted, and retried counts.
|
||||
|
||||
## Switching between modes
|
||||
|
||||
The database mode is controlled by `DATABASE_URL`:
|
||||
|
||||
@@ -68,6 +68,10 @@ queries inside transactions and reserved or dedicated connections. Check the
|
||||
provider's configuration before you enable it. Preparation caches statement
|
||||
metadata, not query results. Unset preserves the existing ORM behavior and the
|
||||
driver default for tagged SQL. Explicit per-query options still apply.
|
||||
Each database client prepares at most 1,000 distinct statements and skips
|
||||
statements longer than 8 KB. If a prepared statement fails because the pooler's copy is
|
||||
stale after a schema change or was lost, Paperclip stops preparing that
|
||||
statement and retries it once unprepared.
|
||||
|
||||
Related optional client tuning: `DATABASE_POOL_MAX`, `DATABASE_IDLE_TIMEOUT_SECONDS`, `DATABASE_CONNECT_TIMEOUT_SECONDS`, `DATABASE_MAX_LIFETIME_SECONDS`, `DATABASE_APPLICATION_NAME`. Driver defaults apply when unset, except that idle pooled connections close after 60 seconds (`DATABASE_IDLE_TIMEOUT_SECONDS=0` keeps them open) and the pool reports `application_name=paperclip`. See [Connection pool settings](#connection-pool-settings).
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import { fileURLToPath } from "node:url";
|
||||
import postgres from "postgres";
|
||||
import { installDatabaseWorkSignals } from "./work-signals.js";
|
||||
import { installDatabaseResponseCapture } from "./response-capture.js";
|
||||
import { installPreparedQueries } from "./prepared-queries.js";
|
||||
import { createPreparedQueryState, installPreparedQueries, preparedQueryStats, type PreparedQueryState, type PreparedQueryStats } from "./prepared-queries.js";
|
||||
import { drizzleWithLazyTransactionQueries } from "./lazy-transaction-queries.js";
|
||||
import * as schema from "./schema/index.js";
|
||||
|
||||
@@ -377,6 +377,13 @@ export function databaseConnectionBudget(options: DatabaseClientOptions = {}) {
|
||||
total: main + privatePools + DATABASE_LISTENER_CONNECTIONS + dedicated };
|
||||
}
|
||||
|
||||
const preparedQueryStates = new WeakMap<object, PreparedQueryState>();
|
||||
|
||||
export function preparedStatementStats(db: Db): PreparedQueryStats | null {
|
||||
const state = preparedQueryStates.get(db);
|
||||
return state ? preparedQueryStats(state) : null;
|
||||
}
|
||||
|
||||
export function dedicatedDbConnectionStats(db: Db): DedicatedDbConnectionStats | null {
|
||||
const set = dedicatedConnectionSets.get(db);
|
||||
return set ? { ...set.stats } : null;
|
||||
@@ -573,7 +580,10 @@ export function createDb(url: string, options?: DatabaseClientOptions) {
|
||||
|
||||
function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object, dedicatedSet?: DedicatedConnectionSet) {
|
||||
const sql = postgres(url, postgresJsOptions(resolved));
|
||||
if (resolved.prepare === true) installPreparedQueries(sql);
|
||||
const preparedState = resolved.prepare === true
|
||||
? (owner && preparedQueryStates.get(owner)) || createPreparedQueryState()
|
||||
: undefined;
|
||||
if (preparedState) installPreparedQueries(sql, preparedState);
|
||||
const key = hostPortKeyOrNull(url);
|
||||
if (key) registerClient(key, sql);
|
||||
// A disconnect can lose the response after a statement has committed.
|
||||
@@ -583,6 +593,7 @@ function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: ob
|
||||
const db = installDatabaseWorkSignals(drizzleWithLazyTransactionQueries(sql, schema), owner);
|
||||
installDatabaseResponseCapture(db, schema);
|
||||
markOpenTransactions(db);
|
||||
if (preparedState) preparedQueryStates.set(db, preparedState);
|
||||
const ownerSet = owner ? dedicatedConnectionSets.get(owner) : undefined;
|
||||
if (dedicatedSet) {
|
||||
trackClientEnd(db, () => { dedicatedSet.ended.add(db); });
|
||||
|
||||
@@ -8,6 +8,7 @@ export {
|
||||
dedicatedDbConnectionStats,
|
||||
type DedicatedDbConnectionStats,
|
||||
type DedicatedDbConnectionOptions,
|
||||
preparedStatementStats,
|
||||
databaseConnectionBudget,
|
||||
PRIVATE_POOL_MAX_CONNECTIONS,
|
||||
createOwnedDatabaseListener,
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import postgres from "postgres";
|
||||
import { integer, jsonb, pgTable, text } from "drizzle-orm/pg-core";
|
||||
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it } from "vitest";
|
||||
import { closeLivenessDb, createDb, databaseClientOptionsFromEnv, livenessDb, withDedicatedDbConnection, type DatabaseClientOptions, type Db } from "./client.js";
|
||||
import { closeLivenessDb, createDb, databaseClientOptionsFromEnv, livenessDb, preparedStatementStats, withDedicatedDbConnection, type DatabaseClientOptions, type Db } from "./client.js";
|
||||
import { createPreparedQueryState, installPreparedQueries, preparedQueryStats } from "./prepared-queries.js";
|
||||
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js";
|
||||
import { runEventPayload } from "./run-event-payload.js";
|
||||
|
||||
const records = pgTable("prepared_query_fixture", { id: text("id").primaryKey(), companyId: text("company_id").notNull(), value: integer("value").notNull(), payload: jsonb("payload") });
|
||||
const stale = pgTable("prepared_stale_fixture", { id: text("id").primaryKey(), value: integer("value").notNull() });
|
||||
const encoded = pgTable("prepared_encoded_fixture", { id: text("id").primaryKey(), payload: runEventPayload("payload") });
|
||||
const support = await getEmbeddedPostgresTestSupport();
|
||||
|
||||
@@ -19,6 +22,8 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre
|
||||
await setup.execute(sql`create table prepared_query_fixture(id text primary key, company_id text not null, value integer not null, payload jsonb)`);
|
||||
await setup.execute(sql`create table prepared_encoded_fixture(id text primary key, payload jsonb)`);
|
||||
await setup.execute(sql`create table prepared_schema_change_fixture(id text primary key)`);
|
||||
await setup.execute(sql`create table prepared_stale_fixture(id text primary key, value integer not null)`);
|
||||
await setup.execute(sql`insert into prepared_stale_fixture(id, value) values ('stale', 5)`);
|
||||
await setup.execute(sql`insert into prepared_schema_change_fixture(id) values ('retained')`);
|
||||
}, 90_000);
|
||||
beforeEach(async () => {
|
||||
@@ -172,4 +177,81 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre
|
||||
await setup.execute(sql`alter table prepared_schema_change_fixture drop column if exists extra`);
|
||||
}
|
||||
});
|
||||
|
||||
async function changeStaleValueType(type: "bigint" | "integer") {
|
||||
await setup.execute(sql.raw(`alter table prepared_stale_fixture alter column value type ${type}`));
|
||||
}
|
||||
|
||||
it("retries a transaction whose first statement meets a plan invalidated by a schema change", async () => {
|
||||
const db = fixture({ prepare: true });
|
||||
const read = () => db.transaction(tx => tx.select().from(stale).where(eq(stale.id, "stale")));
|
||||
expect(await read()).toEqual([{ id: "stale", value: 5 }]);
|
||||
await changeStaleValueType("bigint");
|
||||
try {
|
||||
expect(await read()).toEqual([{ id: "stale", value: 5 }]);
|
||||
expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 1, demoted: 1 });
|
||||
} finally {
|
||||
await changeStaleValueType("integer");
|
||||
}
|
||||
});
|
||||
|
||||
it("retries a transaction whose first statement lost its prepared statement", async () => {
|
||||
const db = fixture({ prepare: true });
|
||||
const read = () => db.transaction(tx => tx.select({ value: records.value }).from(records).where(eq(records.id, "a")));
|
||||
expect(await read()).toEqual([{ value: 1 }]);
|
||||
await db.$client.unsafe("deallocate all");
|
||||
expect(await read()).toEqual([{ value: 1 }]);
|
||||
expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 1, demoted: 1 });
|
||||
});
|
||||
|
||||
it("does not replay a transaction after its first statement and runs the invalidated text unprepared", async () => {
|
||||
const db = fixture({ prepare: true });
|
||||
let calls = 0;
|
||||
const update = () => db.transaction(async tx => {
|
||||
calls++;
|
||||
await tx.update(records).set({ value: 7 }).where(eq(records.id, "b"));
|
||||
return tx.select().from(stale).where(eq(stale.id, "stale"));
|
||||
});
|
||||
expect(await update()).toEqual([{ id: "stale", value: 5 }]);
|
||||
await changeStaleValueType("bigint");
|
||||
try {
|
||||
await expect(update()).rejects.toMatchObject({ cause: { code: "0A000" } });
|
||||
expect(calls).toBe(2);
|
||||
expect(await update()).toEqual([{ id: "stale", value: 5 }]);
|
||||
expect(await statements(db, '%"prepared_stale_fixture"%')).toHaveLength(1);
|
||||
expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 0, demoted: 1 });
|
||||
} finally {
|
||||
await changeStaleValueType("integer");
|
||||
}
|
||||
});
|
||||
|
||||
it("retries a pooled statement unprepared when its prepared statement cannot be created", async () => {
|
||||
const db = fixture({ prepare: true });
|
||||
await statements(db, "%");
|
||||
const names = (await statements(db, "%")).map(row => String(row.name));
|
||||
const prefix = names[0].replace(/\d+$/, "");
|
||||
const next = `${prefix}${Math.max(...names.map(name => Number(name.slice(prefix.length)))) + 1}`;
|
||||
await db.$client.unsafe(`prepare "${next}" as select 1`, [], { prepare: false });
|
||||
expect(await db.select({ value: records.value }).from(records).where(eq(records.id, "b"))).toEqual([{ value: 2 }]);
|
||||
expect(await db.$client.unsafe("select $1::integer as raw_retry_marker", [3]).values()).toEqual([[3]]);
|
||||
expect(await statements(db)).toHaveLength(0);
|
||||
expect(preparedStatementStats(db)).toMatchObject({ retried: 1, demoted: 1 });
|
||||
});
|
||||
|
||||
it("prepares at most the configured number of distinct texts and skips long texts", async () => {
|
||||
const raw = postgres(temporary.connectionString, { max: 1, prepare: true, onnotice: () => {} });
|
||||
const state = createPreparedQueryState({ limit: 2, maxLength: 80 });
|
||||
installPreparedQueries(raw, state);
|
||||
try {
|
||||
for (const marker of ["cap_a", "cap_b", "cap_c", "cap_d"]) {
|
||||
expect(await raw.unsafe(`select $1::integer as ${marker}`, [1])).toEqual([{ [marker]: 1 }]);
|
||||
}
|
||||
const long = `select $1::integer as cap_long /* ${"x".repeat(80)} */`;
|
||||
expect(await raw.unsafe(long, [2])).toEqual([{ cap_long: 2 }]);
|
||||
expect((await raw.unsafe("select name from pg_prepared_statements where statement like 'select $1::integer as cap_%'", [], { prepare: false })).length).toBe(2);
|
||||
expect(preparedQueryStats(state)).toMatchObject({ texts: 2, unprepared: 3 });
|
||||
} finally {
|
||||
await raw.end();
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,9 +1,80 @@
|
||||
type Method = (this: unknown, ...args: unknown[]) => unknown;
|
||||
|
||||
export function installPreparedQueries(client: unknown): void {
|
||||
export const PREPARED_QUERY_TEXT_LIMIT = 1_000;
|
||||
export const PREPARED_QUERY_TEXT_MAX_LENGTH = 8_192;
|
||||
|
||||
export type PreparedQueryStats = {
|
||||
texts: number;
|
||||
unprepared: number;
|
||||
demoted: number;
|
||||
retried: number;
|
||||
retriedTransactions: number;
|
||||
};
|
||||
|
||||
export type PreparedQueryState = {
|
||||
limit: number;
|
||||
maxLength: number;
|
||||
texts: Set<string>;
|
||||
demoted: Set<string>;
|
||||
stats: Omit<PreparedQueryStats, "texts">;
|
||||
};
|
||||
|
||||
type Scope = { retry: "statement" | "first" | "none"; statements: number; firstFailure?: unknown };
|
||||
|
||||
type PendingQuery = {
|
||||
options: Record<string, unknown>;
|
||||
reject(error: unknown): void;
|
||||
resolve(value: unknown): void;
|
||||
isRaw?: unknown;
|
||||
cursorFn?: unknown;
|
||||
forEachFn?: unknown;
|
||||
onlyDescribe?: unknown;
|
||||
then(onFulfilled: (value: unknown) => void, onRejected: (error: unknown) => void): unknown;
|
||||
};
|
||||
|
||||
export function createPreparedQueryState(options: { limit?: number; maxLength?: number } = {}): PreparedQueryState {
|
||||
return {
|
||||
limit: options.limit ?? PREPARED_QUERY_TEXT_LIMIT,
|
||||
maxLength: options.maxLength ?? PREPARED_QUERY_TEXT_MAX_LENGTH,
|
||||
texts: new Set(),
|
||||
demoted: new Set(),
|
||||
stats: { unprepared: 0, demoted: 0, retried: 0, retriedTransactions: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
export function preparedQueryStats(state: PreparedQueryState): PreparedQueryStats {
|
||||
return { texts: state.texts.size, ...state.stats };
|
||||
}
|
||||
|
||||
export function isPreparedStatementFailure(error: unknown): boolean {
|
||||
if (typeof error !== "object" || error === null) return false;
|
||||
const { code, routine, message } = error as { code?: unknown; routine?: unknown; message?: unknown };
|
||||
if (code === "26000" || code === "42P05") return true;
|
||||
return code === "0A000" && (routine === "RevalidateCachedQuery"
|
||||
|| (typeof message === "string" && message.includes("cached plan must not change result type")));
|
||||
}
|
||||
|
||||
function shouldPrepare(state: PreparedQueryState, text: string): boolean {
|
||||
if (state.texts.has(text)) return true;
|
||||
if (state.demoted.has(text) || text.length > state.maxLength || state.texts.size >= state.limit) {
|
||||
state.stats.unprepared++;
|
||||
return false;
|
||||
}
|
||||
state.texts.add(text);
|
||||
return true;
|
||||
}
|
||||
|
||||
function demote(state: PreparedQueryState, text: string): void {
|
||||
if (state.demoted.has(text)) return;
|
||||
state.texts.delete(text);
|
||||
state.demoted.add(text);
|
||||
state.stats.demoted++;
|
||||
}
|
||||
|
||||
export function installPreparedQueries(client: unknown, state: PreparedQueryState = createPreparedQueryState()): void {
|
||||
const visited = new WeakSet<object>();
|
||||
|
||||
function visit(value: unknown): void {
|
||||
function visit(value: unknown, scope: Scope): void {
|
||||
if ((typeof value !== "object" || value === null) && typeof value !== "function") return;
|
||||
if (visited.has(value)) return;
|
||||
visited.add(value);
|
||||
@@ -12,36 +83,77 @@ export function installPreparedQueries(client: unknown): void {
|
||||
if (typeof unsafe === "function") {
|
||||
target.unsafe = function (this: unknown, ...args: unknown[]) {
|
||||
const index = args.length === 2 && !Array.isArray(args[1]) ? 1 : 2;
|
||||
const options = args[index];
|
||||
const options = args[index] as Record<string, unknown> | undefined;
|
||||
if (options !== undefined && (options === null || typeof options !== "object")) {
|
||||
return Reflect.apply(unsafe, this, args);
|
||||
}
|
||||
if (index === 2 && args.length < 2) args[1] = [];
|
||||
args[index] = { prepare: true, ...options };
|
||||
return Reflect.apply(unsafe, this, args);
|
||||
};
|
||||
const position = ++scope.statements;
|
||||
const text = args[0];
|
||||
if (typeof text !== "string" || (options !== undefined && "prepare" in options)) {
|
||||
args[index] = { prepare: true, ...options };
|
||||
return Reflect.apply(unsafe, this, args);
|
||||
}
|
||||
args[index] = { ...options };
|
||||
const query = Reflect.apply(unsafe, this, args) as PendingQuery;
|
||||
if (query.options.simple) {
|
||||
query.options.prepare = true;
|
||||
return query;
|
||||
}
|
||||
let explicit: unknown;
|
||||
Object.defineProperty(query.options, "prepare", {
|
||||
enumerable: true,
|
||||
configurable: true,
|
||||
get: () => explicit ?? shouldPrepare(state, text),
|
||||
set: (value: unknown) => { explicit = value; },
|
||||
});
|
||||
const reject = query.reject;
|
||||
query.reject = (error: unknown) => {
|
||||
if (explicit !== undefined || !isPreparedStatementFailure(error)) return reject(error);
|
||||
demote(state, text);
|
||||
if (scope.retry === "first" && position === 1) scope.firstFailure = error;
|
||||
if (scope.retry !== "statement" || query.cursorFn || query.forEachFn || query.onlyDescribe || options?.onexecute) {
|
||||
return reject(error);
|
||||
}
|
||||
state.stats.retried++;
|
||||
const retry = Reflect.apply(unsafe, this, [text, index === 2 ? args[1] : [], { ...options, prepare: false }]) as PendingQuery;
|
||||
retry.isRaw = query.isRaw;
|
||||
retry.then(query.resolve, reject);
|
||||
};
|
||||
return query;
|
||||
} satisfies Method;
|
||||
}
|
||||
for (const method of ["begin", "savepoint"]) {
|
||||
const original = target[method];
|
||||
if (typeof original !== "function") continue;
|
||||
target[method] = function (this: unknown, ...args: unknown[]) {
|
||||
const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...inner: unknown[]) {
|
||||
visit(inner[0]);
|
||||
return Reflect.apply(arg, this, inner);
|
||||
});
|
||||
return Reflect.apply(original, this, wrapped);
|
||||
const attempt = (retried: boolean): Promise<unknown> => {
|
||||
const inner: Scope = { retry: "first", statements: 0 };
|
||||
const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...params: unknown[]) {
|
||||
visit(params[0], inner);
|
||||
return Reflect.apply(arg, this, params);
|
||||
});
|
||||
return Promise.resolve(Reflect.apply(original, this, wrapped)).catch((error: unknown) => {
|
||||
const failure = inner.firstFailure;
|
||||
if (retried || failure === undefined
|
||||
|| (error !== failure && (error as { cause?: unknown } | null)?.cause !== failure)) throw error;
|
||||
state.stats.retriedTransactions++;
|
||||
return attempt(true);
|
||||
});
|
||||
};
|
||||
return attempt(false);
|
||||
} satisfies Method;
|
||||
}
|
||||
const reserve = target.reserve;
|
||||
if (typeof reserve === "function") {
|
||||
target.reserve = function (this: unknown, ...args: unknown[]) {
|
||||
return Promise.resolve(Reflect.apply(reserve, this, args)).then(reserved => {
|
||||
visit(reserved);
|
||||
visit(reserved, { retry: "none", statements: 0 });
|
||||
return reserved;
|
||||
});
|
||||
} satisfies Method;
|
||||
}
|
||||
}
|
||||
|
||||
visit(client);
|
||||
visit(client, { retry: "statement", statements: 0 });
|
||||
}
|
||||
Reference in new issue
Block a user