diff --git a/doc/DATABASE.md b/doc/DATABASE.md index 5b1ecb4c9b1..4941227615e 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -916,6 +916,20 @@ DATABASE_MAX_LIFETIME_SECONDS=1800 # recycle a pooled connection after this lo DATABASE_APPLICATION_NAME=paperclip # application_name in pg_stat_activity; default: paperclip ``` +`withDedicatedDbConnection` gives an action exclusive use of one session outside +the main pool (application name `paperclip-workspace-finalization-lock`). Each +`createDb` owner keeps at most `DEDICATED_CONNECTION_IDLE_MAX` (4) of these +single-connection clients idle for reuse; they follow the same idle timeout and +max lifetime as the main pool. This is not an admission limit: callers hold +advisory locks and nest dedicated actions, so an action that finds no idle +client opens one and closes it on release when four are already retained. A +connection whose action throws is closed, never reused. Actions must keep +session state transaction-scoped (`pg_advisory_xact_lock`, +`set_config(..., true)`). Like the main pool, a retained session sees an +`ALTER DATABASE` or `ALTER ROLE ... SET` change only after it reconnects; a test +that changes such a setting calls `closeDedicatedDbConnections` first. Shutdown +and ending the owner client close the idle set. + An explicit `DATABASE_PREPARED_STATEMENTS=true` enables named prepared statements for parameterized ORM queries, transactions, reserved connections, and dedicated clients. Postgres.js manages the statement cache. This does not cache query diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 7a362b0ae71..2860e95c383 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -271,9 +271,37 @@ export function postgresJsOptions(options: DatabaseClientOptions): Record Db>(); const ownedDatabaseListenerFactories = new WeakMap Db>(); +export const DEDICATED_CONNECTION_IDLE_MAX = 4; + +type DedicatedConnectionSet = { + create(): Db; + idle: Db[]; + ended: WeakSet; + closed: boolean; +}; +const dedicatedConnectionSets = new WeakMap(); + +function trackClientEnd(client: Db, onEnd: (timeoutSeconds: number | undefined) => Promise | void): void { + const sql = client.$client; + const end = sql.end; + sql.end = async (options?: { timeout?: number }) => { + await Promise.all([onEnd(options?.timeout), end(options)]); + }; +} + +async function endDedicatedConnections(clients: Db[], timeoutSeconds: number): Promise { + await Promise.all(clients.map(client => client.$client.end({ timeout: timeoutSeconds }).catch(() => undefined))); +} + +export async function closeDedicatedDbConnections(db: Db, timeoutSeconds: number): Promise { + const set = dedicatedConnectionSets.get(db); + if (!set) return; + set.closed = true; + await endDedicatedConnections(set.idle.splice(0), timeoutSeconds); +} + export interface OwnedDatabaseListener { listen: Db["$client"]["listen"]; close(): Promise; @@ -290,11 +318,26 @@ export function createOwnedDatabaseListener(db: Db): OwnedDatabaseListener { } export async function withDedicatedDbConnection(db: Db, action: (dedicated: Db) => Promise): Promise { - const factory = dedicatedDbFactories.get(db); - if (!factory) throw new Error("dedicated_connection_requires_create_db"); - const dedicated = factory(); - try { return await action(dedicated); } - finally { await dedicated.$client.end({ timeout: 1 }); } + const set = dedicatedConnectionSets.get(db); + if (!set) throw new Error("dedicated_connection_requires_create_db"); + let dedicated: Db | undefined; + while (!dedicated && set.idle.length) { + const candidate = set.idle.pop()!; + if (!set.ended.has(candidate)) dedicated = candidate; + } + dedicated ??= set.create(); + let completed = false; + try { + const result = await action(dedicated); + completed = true; + return result; + } finally { + if (completed && !set.closed && !set.ended.has(dedicated) && set.idle.length < DEDICATED_CONNECTION_IDLE_MAX) { + set.idle.push(dedicated); + } else if (!set.ended.has(dedicated)) { + await dedicated.$client.end({ timeout: 1 }); + } + } } // postgres.js pipelines a query onto a busy connection when none is idle, so @@ -420,7 +463,7 @@ export function createDb(url: string, options?: DatabaseClientOptions) { return createDbClient(url, resolved); } -function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object) { +function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: object, dedicatedSet?: DedicatedConnectionSet) { const sql = postgres(url, postgresJsOptions(resolved)); if (resolved.prepare === true) installPreparedQueries(sql); const key = hostPortKeyOrNull(url); @@ -431,9 +474,22 @@ function createDbClient(url: string, resolved: DatabaseClientOptions, owner?: ob // that know the complete operation is idempotent. const db = installDatabaseWorkSignals(drizzleWithLazyTransactionQueries(sql, schema), owner); installDatabaseResponseCapture(db, schema); - dedicatedDbFactories.set(db, () => createDbClient(url, { - ...resolved, maxConnections: 1, applicationName: "paperclip-workspace-finalization-lock", - }, db)); + const ownerSet = owner ? dedicatedConnectionSets.get(owner) : undefined; + if (dedicatedSet) { + trackClientEnd(db, () => { dedicatedSet.ended.add(db); }); + dedicatedConnectionSets.set(db, dedicatedSet); + } else if (ownerSet) { + dedicatedConnectionSets.set(db, ownerSet); + } else { + const set: DedicatedConnectionSet = { + create: () => createDbClient(url, { + ...resolved, maxConnections: 1, applicationName: "paperclip-workspace-finalization-lock", + }, db, set), + idle: [], ended: new WeakSet(), closed: false, + }; + dedicatedConnectionSets.set(db, set); + trackClientEnd(db, timeoutSeconds => closeDedicatedDbConnections(db, timeoutSeconds ?? 1)); + } ownedDatabaseListenerFactories.set(db, () => createDbClient(url, { ...resolved, maxConnections: 1, }, db)); diff --git a/packages/db/src/dedicated-connection-pool.test.ts b/packages/db/src/dedicated-connection-pool.test.ts new file mode 100644 index 00000000000..74933f3ec31 --- /dev/null +++ b/packages/db/src/dedicated-connection-pool.test.ts @@ -0,0 +1,194 @@ +import { sql } from "drizzle-orm"; +import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest"; +import { + DEDICATED_CONNECTION_IDLE_MAX, + closeDedicatedDbConnections, + createDb, + withDedicatedDbConnection, + type Db, +} from "./client.js"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; + +const support = await getEmbeddedPostgresTestSupport(); +const APPLICATION_NAME = "paperclip-workspace-finalization-lock"; + +type Session = { pid: number; backendStart: string }; + +async function session(db: Pick): Promise { + const [row] = await db.execute<{ pid: number; backend_start: string }>(sql`select pg_backend_pid() as pid, + (select backend_start::text from pg_stat_activity where pid = pg_backend_pid()) as backend_start`); + return { pid: Number(row.pid), backendStart: String(row.backend_start) }; +} + +const key = (value: Session) => `${value.pid}@${value.backendStart}`; + +async function eventually(read: () => Promise, accept: (value: T) => boolean): Promise { + const deadline = Date.now() + 5_000; + for (;;) { + const value = await read(); + if (accept(value) || Date.now() > deadline) return value; + await new Promise(resolve => setTimeout(resolve, 25)); + } +} + +describe.skipIf(!support.supported)("withDedicatedDbConnection connection reuse", () => { + let temporary: Awaited>; + let observer: Db; + const owners: Db[] = []; + + beforeAll(async () => { + temporary = await startEmbeddedPostgresTestDatabase("dedicated-pool-"); + observer = createDb(temporary.connectionString, { maxConnections: 1, applicationName: "dedicated-pool-observer" }); + }, 90_000); + afterEach(async () => { + for (const db of owners.splice(0)) { + await closeDedicatedDbConnections(db, 1); + await db.$client.end({ timeout: 1 }); + } + }); + afterAll(async () => { + await observer?.$client.end({ timeout: 1 }); + await temporary?.cleanup(); + }); + + function owner() { + const db = createDb(temporary.connectionString, { maxConnections: 2 }); + owners.push(db); + return db; + } + + async function openDedicatedSessions(): Promise { + const rows = await observer.execute<{ pid: number; backend_start: string }>(sql`select pid, backend_start::text as backend_start + from pg_stat_activity where application_name = ${APPLICATION_NAME} and datname = current_database()`); + return rows.map(row => ({ pid: Number(row.pid), backendStart: String(row.backend_start) })); + } + + it("reuses a bounded set of physical connections across 200 sequential calls", async () => { + const db = owner(); + const seen = new Set(); + for (let i = 0; i < 200; i++) { + seen.add(key(await withDedicatedDbConnection(db, dedicated => dedicated.transaction(tx => session(tx))))); + } + expect(seen.size).toBe(1); + expect(seen.size).toBeLessThanOrEqual(DEDICATED_CONNECTION_IDLE_MAX); + }); + + it("gives concurrent callers exclusive sessions and keeps at most the cap open afterwards", async () => { + const db = owner(); + const concurrency = DEDICATED_CONNECTION_IDLE_MAX + 3; + let release!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + let arrived = 0; + let allArrived!: () => void; + const everyoneHolding = new Promise(resolve => { allArrived = resolve; }); + const held = Array.from({ length: concurrency }, () => withDedicatedDbConnection(db, dedicated => + dedicated.transaction(async tx => { + const mine = await session(tx); + await tx.execute(sql`select pg_advisory_xact_lock(${mine.pid})`); + if (++arrived === concurrency) allArrived(); + await gate; + expect(await session(tx)).toEqual(mine); + return mine; + }))); + await everyoneHolding; + release(); + const sessions = await Promise.all(held); + expect(new Set(sessions.map(key)).size).toBe(concurrency); + expect((await eventually(openDedicatedSessions, open => open.length <= DEDICATED_CONNECTION_IDLE_MAX)).length) + .toBeLessThanOrEqual(DEDICATED_CONNECTION_IDLE_MAX); + const retained = new Set(sessions.map(key)); + for (let i = 0; i < 20; i++) { + expect(retained.has(key(await withDedicatedDbConnection(db, session)))).toBe(true); + } + }); + + it("discards a connection whose action failed mid-transaction", async () => { + const db = owner(); + let failed: Session | undefined; + await expect(withDedicatedDbConnection(db, dedicated => dedicated.transaction(async tx => { + failed = await session(tx); + await tx.execute(sql`select 1 / 0`); + }))).rejects.toThrow(); + expect(failed).toBeDefined(); + const next = await withDedicatedDbConnection(db, session); + expect(key(next)).not.toBe(key(failed!)); + const open = await eventually(openDedicatedSessions, rows => !rows.map(key).includes(key(failed!))); + expect(open.map(key)).not.toContain(key(failed!)); + }); + + it("discards a connection whose socket broke during the action", async () => { + const db = owner(); + let broken: Session | undefined; + await expect(withDedicatedDbConnection(db, async dedicated => { + broken = await session(dedicated); + await observer.execute(sql`select pg_terminate_backend(${broken.pid})`); + await session(dedicated); + })).rejects.toThrow(); + const next = await withDedicatedDbConnection(db, session); + expect(next.pid).not.toBe(broken!.pid); + }); + + it("recovers when an idle retained connection is terminated between calls", async () => { + const db = owner(); + const first = await withDedicatedDbConnection(db, session); + await observer.execute(sql`select pg_terminate_backend(${first.pid})`); + await eventually(openDedicatedSessions, rows => !rows.some(row => row.pid === first.pid)); + await new Promise(resolve => setTimeout(resolve, 50)); + const second = await withDedicatedDbConnection(db, session); + expect(second.pid).not.toBe(first.pid); + }); + + it("never reuses a connection the action ended itself", async () => { + const db = owner(); + const first = await withDedicatedDbConnection(db, async dedicated => { + const value = await session(dedicated); + await dedicated.$client.end({ timeout: 0 }); + return value; + }); + const second = await withDedicatedDbConnection(db, session); + expect(second.pid).not.toBe(first.pid); + }); + + it("keeps transaction-local settings out of the next action", async () => { + const db = owner(); + await withDedicatedDbConnection(db, dedicated => dedicated.transaction(tx => + tx.execute(sql`select set_config('statement_timeout', '1234', true), set_config('lock_timeout', '567', true)`))); + const [row] = await withDedicatedDbConnection(db, dedicated => dedicated.execute<{ statement_timeout: string; lock_timeout: string }>( + sql`select current_setting('statement_timeout') as statement_timeout, current_setting('lock_timeout') as lock_timeout`)); + expect(row).toEqual({ statement_timeout: "0", lock_timeout: "0" }); + }); + + it("closes every retained connection on shutdown and stays usable without retaining more", async () => { + const db = owner(); + await Promise.all(Array.from({ length: DEDICATED_CONNECTION_IDLE_MAX }, () => + withDedicatedDbConnection(db, dedicated => dedicated.transaction(async tx => { + await tx.execute(sql`select pg_sleep(0.05)`); + })))); + expect((await openDedicatedSessions()).length).toBeGreaterThan(0); + await closeDedicatedDbConnections(db, 1); + expect(await eventually(openDedicatedSessions, rows => rows.length === 0)).toEqual([]); + await withDedicatedDbConnection(db, session); + expect(await eventually(openDedicatedSessions, rows => rows.length === 0)).toEqual([]); + }); + + it("closes retained connections when the owner client ends", async () => { + const db = createDb(temporary.connectionString, { maxConnections: 1 }); + await withDedicatedDbConnection(db, session); + expect((await openDedicatedSessions()).length).toBe(1); + await db.$client.end({ timeout: 1 }); + expect(await eventually(openDedicatedSessions, rows => rows.length === 0)).toEqual([]); + }); + + it("shares one retained set with nested dedicated actions without blocking them", async () => { + const db = owner(); + const [outer, inner] = await withDedicatedDbConnection(db, async dedicated => { + const mine = await session(dedicated); + const nested = await withDedicatedDbConnection(dedicated, session); + return [mine, nested]; + }); + expect(inner.pid).not.toBe(outer.pid); + const reused = new Set(); + for (let i = 0; i < 10; i++) reused.add((await withDedicatedDbConnection(db, session)).pid); + expect([...reused].every(pid => pid === outer.pid || pid === inner.pid)).toBe(true); + }); +}); diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 7e41782d377..fa0e2d7bdde 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -1,6 +1,8 @@ export { createDb, withDedicatedDbConnection, + closeDedicatedDbConnections, + DEDICATED_CONNECTION_IDLE_MAX, createOwnedDatabaseListener, type OwnedDatabaseListener, livenessDb, diff --git a/server/src/__tests__/company-skills-observability.test.ts b/server/src/__tests__/company-skills-observability.test.ts index 98db6802775..31cc9a965a9 100644 --- a/server/src/__tests__/company-skills-observability.test.ts +++ b/server/src/__tests__/company-skills-observability.test.ts @@ -7,7 +7,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { context, SpanStatusCode, trace } from "@opentelemetry/api"; import { AsyncLocalStorageContextManager } from "@opentelemetry/context-async-hooks"; import { InMemorySpanExporter, SimpleSpanProcessor, TracerProvider } from "@opentelemetry/sdk-trace"; -import { companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb, +import { closeDedicatedDbConnections, companies, companySkills, companySkillVersions, companySkillRepositorySnapshots, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS } from "@paperclipai/db"; import { eq } from "drizzle-orm"; import { createRunPreparationTiming, withRunExecutionTiming } from "../services/execution-preparation-timing.js"; @@ -133,6 +133,7 @@ describeDatabase("runtime skill observability", () => { const name = new URL(database.connectionString).pathname.slice(1); assert.match(name, /^[a-zA-Z0-9_-]+$/); await db.$client.unsafe(`ALTER DATABASE "${name}" SET lock_timeout TO '100ms'`); + await closeDedicatedDbConnections(db, 1); const entered = deferred(), release = deferred(); const holder = withSkillVersionCacheLock(db, f.companyId, async () => { entered.resolve(); await release.promise; }); try { diff --git a/server/src/__tests__/status-card-evaluation-work.test.ts b/server/src/__tests__/status-card-evaluation-work.test.ts index 870eba73ea1..09d90c4a4f2 100644 --- a/server/src/__tests__/status-card-evaluation-work.test.ts +++ b/server/src/__tests__/status-card-evaluation-work.test.ts @@ -546,8 +546,8 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu try { const created = await card("reactive", { ...policy(), debounceSeconds: 1 }); const countLocks = async () => Number((await db.execute<{ count: string }>(sql` - select count(*) from pg_stat_activity where datname = current_database() - and application_name = 'paperclip-workspace-finalization-lock'`))[0]!.count); + select count(*) from pg_locks l join pg_database d on d.oid = l.database + where l.locktype = 'advisory' and d.datname = current_database()`))[0]!.count); const baseline = await countLocks(); await issueService(db).create(companyId, { title: "Pool-one blocker", status: "blocked", priority: "medium" }); const service = statusCardEvaluationService(db, { wakeup }); @@ -573,7 +573,7 @@ const support = external ? { supported: true } : await getEmbeddedPostgresTestSu expect(await service.next()).toBeNull(); expect(idleWorkSnapshot().active).toBe(0); await vi.waitFor(async () => expect(await countLocks()).toBe(baseline)); - // Failure exits also release the physical connection and advisory lock. + // Failure exits also release the advisory lock. await db.update(statusCards).set({ queries: [{ scope: "invalid" } as never], nextEvalAt: new Date() }).where(eq(statusCards.id, created.id)); await service.run(new AbortController().signal); expect((await statusCardService(db).getById(created.id))!.nextEvalReason).toBe("retry"); diff --git a/server/src/index.ts b/server/src/index.ts index 8da56303243..af3393d0076 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -131,6 +131,7 @@ import { closeLivenessDb, closeCheckpointDb, closeFileStorageDb, + closeDedicatedDbConnections, } from "@paperclipai/db"; import detectPort from "detect-port"; import { createApp } from "./app.js"; @@ -752,7 +753,7 @@ async function startServerWithDatabaseTeardown( const closeDatabaseClients = async () => { setSentryRunLinkResolver(null); const clients = pluginMigrationDb === db ? [db] : [db, pluginMigrationDb]; - await Promise.all([...clients.map((client) => endDatabaseClient(client, 5)), closeLivenessDb(db, 5), closeCheckpointDb(db, 5), closeFileStorageDb(db, 5)]); + await Promise.all([...clients.map((client) => endDatabaseClient(client, 5)), closeLivenessDb(db, 5), closeCheckpointDb(db, 5), closeFileStorageDb(db, 5), closeDedicatedDbConnections(db, 5)]); }; startupDatabase.close = closeDatabaseClients; startupTimeline.mark("database"); diff --git a/server/src/services/native-runtime/native-workspace-finalizer.test.ts b/server/src/services/native-runtime/native-workspace-finalizer.test.ts index 5b5e18b3049..ae98d0f80a3 100644 --- a/server/src/services/native-runtime/native-workspace-finalizer.test.ts +++ b/server/src/services/native-runtime/native-workspace-finalizer.test.ts @@ -22,6 +22,7 @@ import { agentWakeupRequests, agents, companies, + closeDedicatedDbConnections, completionContracts, createDb, environmentLeases, @@ -1050,6 +1051,7 @@ describe("native workspace finalization recovery", () => { const rows = await db.execute(sql`select current_user as name`); const role = String(rows[0].name); await db.execute(sql`alter role ${sql.identifier(role)} set idle_in_transaction_session_timeout = '1s'`); + await closeDedicatedDbConnections(db, 1); try { const result = await withNativeWorkspaceFinalizationOwnership({ db, companyId, runId: seeded.runId }, async ownership => { await new Promise(resolve => setTimeout(resolve, 3200));