diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 21445e975b2..15ecde9114a 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -193,11 +193,16 @@ pooler's statement for an affected text keeps failing with `cached plan must not change result type` for every client and every process. When a statement fails with that error, `prepared statement does not exist` (26000), or a duplicate statement name (42P05), the client stops preparing that text for the -rest of the process and retries once without preparation. The retry applies to -pooled statements and to a transaction whose first statement failed. A later -statement in a transaction cannot be retried, so that transaction fails once -for each affected text in each process. `preparedStatementStats(db)` reports -the prepared texts and the unprepared, demoted, and retried counts. +rest of the process and retries once without preparation. A pooled statement +is retried on its own. When the failed statement is the first one that the +transaction ran (after `SET TRANSACTION` only), the client rolls back, begins +again with the same options, repeats the `SET TRANSACTION` statements, and runs +the statement unprepared on the same connection. The transaction callback does +not run again and receives the result. A later statement in a transaction, or a +statement in a savepoint, is not retried, so that transaction or savepoint fails +once for each affected text in each process. `preparedStatementStats(db)` +reports the prepared texts and the unprepared, demoted, retried, and restarted +counts. ## Switching between modes diff --git a/packages/db/src/prepared-queries-postgres.test.ts b/packages/db/src/prepared-queries-postgres.test.ts index 34e4c6a488c..c021db95429 100644 --- a/packages/db/src/prepared-queries-postgres.test.ts +++ b/packages/db/src/prepared-queries-postgres.test.ts @@ -182,29 +182,61 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre await setup.execute(sql.raw(`alter table prepared_stale_fixture alter column value type ${type}`)); } - it("retries a transaction whose first statement meets a plan invalidated by a schema change", async () => { + it("restarts a transaction whose first statement meets an invalidated plan without running the callback again", async () => { const db = fixture({ prepare: true }); - const read = () => db.transaction(tx => tx.select().from(stale).where(eq(stale.id, "stale"))); + let calls = 0; + const read = () => db.transaction(tx => { calls++; return tx.select().from(stale).where(eq(stale.id, "stale")); }); expect(await read()).toEqual([{ id: "stale", value: 5 }]); await changeStaleValueType("bigint"); try { expect(await read()).toEqual([{ id: "stale", value: 5 }]); - expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 1, demoted: 1 }); + expect(calls).toBe(2); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 1, demoted: 1 }); } finally { await changeStaleValueType("integer"); } }); - it("retries a transaction whose first statement lost its prepared statement", async () => { + it("restarts a transaction whose first statement lost its prepared statement and keeps its later work", async () => { const db = fixture({ prepare: true }); - const read = () => db.transaction(tx => tx.select({ value: records.value }).from(records).where(eq(records.id, "a"))); - expect(await read()).toEqual([{ value: 1 }]); - await db.$client.unsafe("deallocate all"); - expect(await read()).toEqual([{ value: 1 }]); - expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 1, demoted: 1 }); + const run = (value: number) => db.transaction(async tx => { + const [row] = await tx.select({ value: records.value }).from(records).where(eq(records.id, "a")); + await tx.execute(sql`update prepared_query_fixture set value = ${value} where id = ${"b"} ${sql.raw(`/* run ${value} */`)}`); + return row; + }); + expect(await run(3)).toEqual({ value: 1 }); + const [{ name }] = await statements(db, 'select "value" from "prepared_query_fixture"%'); + await db.$client.unsafe(`deallocate "${String(name)}"`); + expect(await run(4)).toEqual({ value: 1 }); + expect(await setup.select({ value: records.value }).from(records).where(eq(records.id, "b"))).toEqual([{ value: 4 }]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 1, demoted: 1 }); }); - it("does not replay a transaction after its first statement and runs the invalidated text unprepared", async () => { + it("keeps the isolation level and access mode of a restarted transaction", async () => { + const db = fixture({ prepare: true }); + const settings = "select current_setting('transaction_isolation') as isolation, current_setting('transaction_read_only') as read_only, $1::integer as marker"; + const snapshot = { isolation: "repeatable read", read_only: "on", marker: 1 }; + const readDriver = () => db.$client.begin("isolation level repeatable read read only", async tx => { + const rows = await tx.unsafe("select id, value from prepared_stale_fixture where id = $1", ["stale"]); + return [rows, await tx.unsafe(settings, [1])]; + }); + const readOrm = () => db.transaction(async tx => [ + await tx.select().from(stale).where(eq(stale.id, "stale")), + await tx.execute(sql`select current_setting('transaction_isolation') as isolation, current_setting('transaction_read_only') as read_only, ${1}::integer as marker`), + ], { isolationLevel: "repeatable read", accessMode: "read only" }); + expect(await readDriver()).toEqual([[{ id: "stale", value: 5 }], [snapshot]]); + expect(await readOrm()).toEqual([[{ id: "stale", value: 5 }], [snapshot]]); + await changeStaleValueType("bigint"); + try { + expect(await readDriver()).toEqual([[{ id: "stale", value: "5" }], [snapshot]]); + expect(await readOrm()).toEqual([[{ id: "stale", value: 5 }], [snapshot]]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 2, demoted: 2 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("does not restart a transaction after its first statement and runs the invalidated text unprepared", async () => { const db = fixture({ prepare: true }); let calls = 0; const update = () => db.transaction(async tx => { @@ -219,7 +251,49 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre expect(calls).toBe(2); expect(await update()).toEqual([{ id: "stale", value: 5 }]); expect(await statements(db, '%"prepared_stale_fixture"%')).toHaveLength(1); - expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 0, demoted: 1 }); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("counts a tagged statement as the first statement of a transaction", async () => { + const db = fixture({ prepare: true }); + const read = "select id, value from prepared_stale_fixture where id = $1"; + expect(await db.$client.unsafe(read, ["stale"])).toEqual([{ id: "stale", value: 5 }]); + await changeStaleValueType("bigint"); + try { + await expect(db.$client.begin(async tx => { + await tx`update prepared_query_fixture set value = value + 1 where id = 'a'`; + return tx.unsafe(read, ["stale"]); + })).rejects.toMatchObject({ code: "0A000" }); + expect(await setup.select({ value: records.value }).from(records).where(eq(records.id, "a"))).toEqual([{ value: 1 }]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); + } finally { + await changeStaleValueType("integer"); + } + }); + + it("rolls back only the savepoint whose first statement fails and continues the outer transaction once", async () => { + const db = fixture({ prepare: true }); + const nestedRead = (tx: Parameters[0]>[0]) => + tx.transaction(nested => nested.select().from(stale).where(eq(stale.id, "stale"))); + await db.transaction(async tx => { await nestedRead(tx); }); + await changeStaleValueType("bigint"); + let outerCalls = 0; + try { + const result = await db.transaction(async tx => { + outerCalls++; + await tx.execute(sql`update prepared_query_fixture set value = value + 10 where id = ${"a"}`); + const nested = await nestedRead(tx).catch((error: { cause?: { code?: string } }) => error.cause?.code); + await tx.execute(sql`update prepared_query_fixture set value = value + 100 where id = ${"b"}`); + return [nested, await tx.select().from(stale).where(eq(stale.id, "stale"))]; + }); + expect(result).toEqual(["0A000", [{ id: "stale", value: 5 }]]); + expect(outerCalls).toBe(1); + expect(await setup.select({ id: records.id, value: records.value }).from(records).orderBy(records.id)) + .toEqual([{ id: "a", value: 11 }, { id: "b", value: 102 }]); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); } finally { await changeStaleValueType("integer"); } @@ -244,7 +318,7 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre expect(preparedStatementStats(db)).toMatchObject({ retried: 2, demoted: 2 }); }); - it("does not replay a transaction when a query created first executes after another statement", async () => { + it("does not restart a transaction when a query created first executes after another statement", async () => { const db = fixture({ prepare: true }); const read = "select id, value from prepared_stale_fixture where id = $1"; expect(await db.$client.unsafe(read, ["stale"])).toEqual([{ id: "stale", value: 5 }]); @@ -258,7 +332,7 @@ describe.skipIf(!support.supported)("explicit ORM prepared statements on Postgre return await saved; })).rejects.toMatchObject({ code: "0A000" }); expect(calls).toBe(1); - expect(preparedStatementStats(db)).toMatchObject({ retriedTransactions: 0, demoted: 1 }); + expect(preparedStatementStats(db)).toMatchObject({ restartedTransactions: 0, demoted: 1 }); } finally { await changeStaleValueType("integer"); } diff --git a/packages/db/src/prepared-queries.ts b/packages/db/src/prepared-queries.ts index 677b60449d9..0efa5ac1d97 100644 --- a/packages/db/src/prepared-queries.ts +++ b/packages/db/src/prepared-queries.ts @@ -8,7 +8,7 @@ export type PreparedQueryStats = { unprepared: number; demoted: number; retried: number; - retriedTransactions: number; + restartedTransactions: number; }; export type PreparedQueryState = { @@ -19,7 +19,8 @@ export type PreparedQueryState = { stats: Omit; }; -type Scope = { retry: "statement" | "first" | "none"; statements: number; firstFailure?: unknown }; +type Counts = { created: number; executed: number; setup: string[] }; +type Scope = { retry: "statement" | "restart" | "none"; begin?: string; counts: Counts }; type PendingQuery = { options: Record; @@ -38,7 +39,7 @@ export function createPreparedQueryState(options: { limit?: number; maxLength?: maxLength: options.maxLength ?? PREPARED_QUERY_TEXT_MAX_LENGTH, texts: new Set(), demoted: new Set(), - stats: { unprepared: 0, demoted: 0, retried: 0, retriedTransactions: 0 }, + stats: { unprepared: 0, demoted: 0, retried: 0, restartedTransactions: 0 }, }; } @@ -71,10 +72,37 @@ function demote(state: PreparedQueryState, text: string): void { state.stats.demoted++; } +const TRANSACTION_SETUP = /^\s*set\s+transaction\b/i; + +function scope(retry: Scope["retry"], begin?: string, counts: Counts = { created: 0, executed: 0, setup: [] }): Scope { + return { retry, begin, counts }; +} + +function track(query: PendingQuery, current: Counts, text: string | undefined, decide: () => unknown) { + current.created++; + let explicit: unknown = "prepare" in query.options ? query.options.prepare : undefined; + const execution = { started: false, first: false }; + Object.defineProperty(query.options, "prepare", { + enumerable: true, + configurable: true, + get: () => { + if (!execution.started) { + execution.started = true; + execution.first = current.executed === current.setup.length; + current.executed++; + if (execution.first && text !== undefined && query.options.simple && TRANSACTION_SETUP.test(text)) current.setup.push(text); + } + return explicit ?? decide(); + }, + set: (value: unknown) => { explicit = value; }, + }); + return { execution, explicit: () => explicit }; +} + export function installPreparedQueries(client: unknown, state: PreparedQueryState = createPreparedQueryState()): void { const visited = new WeakSet(); - function visit(value: unknown, scope: Scope): void { + function visit(value: unknown, current: Scope): void { if ((typeof value !== "object" || value === null) && typeof value !== "function") return; if (visited.has(value)) return; visited.add(value); @@ -92,65 +120,71 @@ export function installPreparedQueries(client: unknown, state: PreparedQueryStat args[index] = { prepare: true, ...options }; if (typeof text !== "string") return Reflect.apply(unsafe, this, args); const query = Reflect.apply(unsafe, this, args) as PendingQuery; - let explicit: unknown = (options !== undefined && "prepare" in options) || query.options.simple - ? query.options.prepare : undefined; - let position: number | undefined; - Object.defineProperty(query.options, "prepare", { - enumerable: true, - configurable: true, - get: () => { - position ??= ++scope.statements; - return explicit ?? shouldPrepare(state, text); - }, - set: (value: unknown) => { explicit = value; }, - }); + const caller = (options !== undefined && "prepare" in options) || query.options.simple; + if (!caller) delete query.options.prepare; + const tracked = track(query, current.counts, text, () => shouldPrepare(state, text)); const reject = query.reject; query.reject = (error: unknown) => { - if (explicit !== undefined || !isPreparedStatementFailure(error)) return reject(error); + if (tracked.explicit() !== undefined || !isPreparedStatementFailure(error)) return reject(error); demote(state, text); - if (scope.retry === "first" && position === 1) scope.firstFailure = error; - if (scope.retry !== "statement" || query.cursorFn || query.forEachFn || query.onlyDescribe || options?.onexecute) { - return reject(error); + if (query.cursorFn || query.forEachFn || query.onlyDescribe || options?.onexecute) return reject(error); + const params = index === 2 ? args[1] : []; + const unprepared = { ...options, prepare: false }; + if (current.retry === "statement") { + state.stats.retried++; + const retry = Reflect.apply(unsafe, this, [text, params, unprepared]) as PendingQuery; + retry.isRaw = query.isRaw; + retry.then(query.resolve, reject); + return; } - state.stats.retried++; - const retry = Reflect.apply(unsafe, this, [text, index === 2 ? args[1] : [], { ...options, prepare: false }]) as PendingQuery; + if (current.retry !== "restart" || !tracked.execution.first || current.counts.created !== current.counts.executed) return reject(error); + state.stats.restartedTransactions++; + const run = (sql: string, ...rest: unknown[]) => Reflect.apply(unsafe, this, [sql, ...rest]) as PendingQuery; + const steps = [run("rollback"), run(`begin ${current.begin ?? ""}`.trim()), ...current.counts.setup.map(setup => run(setup))]; + const retry = run(text, params, unprepared); retry.isRaw = query.isRaw; - retry.then(query.resolve, reject); + Promise.all([...steps, retry]).then(results => query.resolve(results[results.length - 1]), reject); }; return query; } satisfies Method; } - for (const method of ["begin", "savepoint"]) { + for (const method of ["begin", "savepoint"] as const) { const original = target[method]; if (typeof original !== "function") continue; target[method] = function (this: unknown, ...args: unknown[]) { - const attempt = (retried: boolean): Promise => { - const inner: Scope = { retry: "first", statements: 0 }; - const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...params: unknown[]) { - visit(params[0], inner); - return Reflect.apply(arg, this, params); - }); - return Promise.resolve(Reflect.apply(original, this, wrapped)).catch((error: unknown) => { - const failure = inner.firstFailure; - if (retried || failure === undefined - || (error !== failure && (error as { cause?: unknown } | null)?.cause !== failure)) throw error; - state.stats.retriedTransactions++; - return attempt(true); - }); - }; - return attempt(false); + const begin = method === "begin" && typeof args[0] === "string" ? args[0].replace(/[^a-z ]/ig, "") : undefined; + const inner = method === "begin" ? scope("restart", begin) : scope("none", undefined, current.counts); + const wrapped = args.map(arg => typeof arg !== "function" ? arg : function (this: unknown, ...params: unknown[]) { + visit(params[0], inner); + return Reflect.apply(arg, this, [counted(params[0], inner), ...params.slice(1)]); + }); + return Reflect.apply(original, this, wrapped); } satisfies Method; } const reserve = target.reserve; if (typeof reserve === "function") { target.reserve = function (this: unknown, ...args: unknown[]) { return Promise.resolve(Reflect.apply(reserve, this, args)).then(reserved => { - visit(reserved, { retry: "none", statements: 0 }); + visit(reserved, scope("none")); return reserved; }); } satisfies Method; } } - visit(client, { retry: "statement", statements: 0 }); + function counted(sql: unknown, current: Scope): unknown { + if (typeof sql !== "function") return sql; + return new Proxy(sql, { + apply(target, self, args) { + const result = Reflect.apply(target, self, args) as unknown; + const strings = args[0] as { raw?: unknown } | undefined; + if (Array.isArray(strings?.raw) && result instanceof Promise && "options" in result) { + track(result as unknown as PendingQuery, current.counts, undefined, () => true); + } + return result; + }, + }); + } + + visit(client, scope("statement")); }