diff --git a/server/src/services/heartbeat/reattach-phase.test.ts b/server/src/services/heartbeat/reattach-phase.test.ts index 4a6fb49d195..352502cfd8d 100644 --- a/server/src/services/heartbeat/reattach-phase.test.ts +++ b/server/src/services/heartbeat/reattach-phase.test.ts @@ -1,6 +1,7 @@ import type { Db } from "@paperclipai/db"; -import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; -import { eq } from "drizzle-orm"; +import { agents, closeLivenessDb, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { logger } from "../../middleware/logger.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { createHeartbeatQueueTestDependencies as callbacks } from "../../__tests__/helpers/heartbeat-queue.js"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; @@ -24,6 +25,7 @@ import { } from "./run-admission.js"; import { claimableReattachRun, + type DatabaseSessionWait, handOverMoment, listLiveRunsOwnedByPreviousControllers, noteReattachRunsClaimed, @@ -265,6 +267,7 @@ describe.skipIf(!support.supported)("reattach phase discovery", () => { db = createDb(external ?? database!.connectionString, { maxConnections: 4 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { + if (db) await closeLivenessDb(db, 1); await db?.$client.end(); await database?.cleanup(); }); @@ -612,5 +615,56 @@ describe.skipIf(!support.supported)("reattach phase discovery", () => { expect(isReattachPhaseActive()).toBe(false); expect(ended).toEqual([expect.objectContaining({ reason: "nothing_to_reattach", gatedStarts: false })]); }); + + it("names the database sessions that hold an authenticated run's row when the phase ends at the cap, even with the main pool stalled", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPS"); + const stalled = await run("running", liveLease()); + const stalledDb = createDb(external ?? database!.connectionString, { maxConnections: 2 }); + const warn = vi.spyOn(logger, "warn"); + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let locked!: (pid: number) => void; + const holderPid = new Promise(resolve => { locked = resolve; }); + const holder = stalledDb.transaction(async tx => { + const [backend] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + await tx.execute(sql.raw(`select id from heartbeat_runs where id = '${stalled}' and next_event_seq >= 0 for update`)); + locked(Number(backend.pid)); + await held; + }); + const holderBackend = await holderPid; + const waiter = stalledDb.transaction(async tx => { + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, stalled)).for("no key update"); + }); + let coordinator: ReturnType | undefined; + try { + const ended: ReattachPhaseSummary[] = []; + coordinator = createDeliveryWorkCoordinator({ owner: stalledDb, canRun: () => true, onError: () => undefined }); + registerReattachPhaseWork(coordinator, stalledDb, { capMs: 300, bootId: "incoming-boot", recheckMs: 20, + onEnd: summary => ended.push(summary) }); + joinReattachPhase([{ runId: stalled, reason: "lease_expired", claimableAt: Date.now() }]); + markReattachAuthenticated(stalled); + await vi.waitFor(() => expect(ended.some(summary => summary.reason === "cap_reached")).toBe(true), { timeout: 2_000, interval: 10 }); + const stallMessage = "reattach phase ended with authenticated runs that were never reached; database sessions waiting or holding locks"; + await vi.waitFor(() => expect(warn.mock.calls.some(call => call[1] === stallMessage)).toBe(true), { timeout: 5_000, interval: 20 }); + const [fields] = warn.mock.calls.find(call => call[1] === stallMessage)! as unknown as [{ runIds: string[]; waits: DatabaseSessionWait[] }]; + expect(fields.runIds).toEqual([stalled]); + expect(fields.waits).toEqual(expect.arrayContaining([ + expect.objectContaining({ pid: holderBackend }), + expect.objectContaining({ waitEventType: "Lock", blockedBy: [holderBackend] }), + ])); + expect(JSON.stringify(fields.waits)).not.toContain(stalled); + expect(fields.waits.find(wait => wait.pid === holderBackend)?.query).toContain("where id = ? and next_event_seq >= ? for update"); + } finally { + release(); + await holder; + await waiter; + await stopReattachWatch(); + await coordinator?.stop(); + warn.mockRestore(); + await closeLivenessDb(stalledDb, 1); + await stalledDb.$client.end(); + } + }); }); }); diff --git a/server/src/services/heartbeat/reattach-phase.ts b/server/src/services/heartbeat/reattach-phase.ts index 50a03d29b3d..09f43e180f7 100644 --- a/server/src/services/heartbeat/reattach-phase.ts +++ b/server/src/services/heartbeat/reattach-phase.ts @@ -1,5 +1,5 @@ -import { and, eq, inArray, isNotNull, isNull, ne, notLike, or } from "drizzle-orm"; -import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { and, eq, inArray, isNotNull, isNull, ne, notLike, or, sql } from "drizzle-orm"; +import { dedicatedDbConnectionStats, heartbeatRuns, livenessDb, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -18,6 +18,7 @@ import { settleReattachRun, type ClaimableReattachRun, type ReattachClaimableReason, + type ReattachPhaseSummary, } from "./run-admission.js"; export const REATTACH_PHASE_DEFAULT_CAP_MS = 180_000; @@ -95,6 +96,7 @@ type ReattachWatch = { worker: ReattachWorker | null; stopListening: () => void; claim?: (runIds: string[]) => Promise; + stallReports: Set>; }; let watch: ReattachWatch | null = null; @@ -215,9 +217,18 @@ export function registerReattachPhaseWork( if (capMs <= 0) return null; if (watch) void stopReattachWatch(); enableReattachPhase(capMs); + const stopEndListener = options.onEnd ? onReattachPhaseEnd(options.onEnd) : () => undefined; const current: ReattachWatch = { db, bootId: options.bootId ?? currentControllerBootId(), initialized: false, - notified: false, watching: false, leases: new Map(), worker: null, claim: options.claim, - stopListening: options.onEnd ? onReattachPhaseEnd(options.onEnd) : () => undefined }; + notified: false, watching: false, leases: new Map(), worker: null, claim: options.claim, stallReports: new Set(), + stopListening: () => undefined }; + const stopStallReport = onReattachPhaseEnd(summary => { + if (watch !== current || !stalledReattach(summary)) return; + const report: Promise = reportStalledReattach(db, summary) + .catch(error => logger.warn({ err: error }, "reattach phase could not read database session waits")) + .finally(() => { current.stallReports.delete(report); }); + current.stallReports.add(report); + }); + current.stopListening = () => { stopEndListener(); stopStallReport(); }; watch = current; const release = () => { if (watch !== current) return; @@ -235,10 +246,72 @@ export function registerReattachPhaseWork( return current.worker; } +export type DatabaseSessionWait = { + pid: number; + applicationName: string | null; + state: string | null; + waitEventType: string | null; + waitEvent: string | null; + blockedBy: number[]; + transactionSeconds: number | null; + stateSeconds: number | null; + query: string | null; +}; + +export async function readDatabaseSessionWaits(db: Db): Promise { + return livenessDb(db).transaction(async tx => { + await tx.execute(sql`select set_config('statement_timeout', '5000', true)`); + const rows = await tx.execute>(sql` + with sessions as ( + select pid, application_name, state, wait_event_type, wait_event, pg_blocking_pids(pid) as blocked_by, + xact_start, state_change, query + from pg_stat_activity + where datname = current_database() and pid <> pg_backend_pid() and backend_type = 'client backend' + ), blockers as (select distinct unnest(blocked_by) as pid from sessions) + select pid, application_name, state, wait_event_type, wait_event, blocked_by, + extract(epoch from clock_timestamp() - xact_start)::int as transaction_seconds, + extract(epoch from clock_timestamp() - state_change)::int as state_seconds, + left(regexp_replace(regexp_replace(query, '''(?:[^'']|'''')*''', '?', 'g'), '\\m[0-9][0-9.]*\\M', '?', 'g'), 160) as query + from sessions + where cardinality(blocked_by) > 0 or pid in (select pid from blockers) + or state like 'idle in transaction%' + or (xact_start is not null and clock_timestamp() - xact_start > interval '30 seconds') + order by xact_start nulls last + limit 50`); + return rows.map(row => ({ + pid: Number(row.pid), + applicationName: (row.application_name as string | null) ?? null, + state: (row.state as string | null) ?? null, + waitEventType: (row.wait_event_type as string | null) ?? null, + waitEvent: (row.wait_event as string | null) ?? null, + blockedBy: Array.isArray(row.blocked_by) ? row.blocked_by.map(Number) : [], + transactionSeconds: row.transaction_seconds === null ? null : Number(row.transaction_seconds), + stateSeconds: row.state_seconds === null ? null : Number(row.state_seconds), + query: (row.query as string | null) ?? null, + })); + }); +} + +function authenticatedButUnreached(summary: ReattachPhaseSummary): string[] { + return summary.runs.filter(run => run.outcome === "outstanding" && run.authenticatedMs !== null).map(run => run.runId); +} + +function stalledReattach(summary: ReattachPhaseSummary): boolean { + return summary.reason === "cap_reached" && authenticatedButUnreached(summary).length > 0; +} + +async function reportStalledReattach(db: Db, summary: ReattachPhaseSummary): Promise { + const runIds = authenticatedButUnreached(summary); + const waits = await readDatabaseSessionWaits(db); + logger.warn({ runIds, pool: dedicatedDbConnectionStats(db), waits }, + "reattach phase ended with authenticated runs that were never reached; database sessions waiting or holding locks"); +} + export async function stopReattachWatch(): Promise { const current = watch; if (!current) return; watch = null; current.stopListening(); await current.worker?.stop(); + await Promise.all(current.stallReports); }