From d8a5db36e697c3ccb6e61aef7dcaddec1dd42293 Mon Sep 17 00:00:00 2001 From: Dotta <34892728+cryppadotta@users.noreply.github.com> Date: Mon, 16 Nov 2026 19:32:20 -0600 Subject: [PATCH] fix(heartbeat): name the blocking database sessions when reattached runs are never reached (#21946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Thinking Path > - Paperclip is the open source app people use to manage AI agents for work > - After a server restart, the new server reattaches to the live runs of the old server (the reattach phase) > - In a restart-chaos release swap with a hard kill, six runs authenticated in under 10 s but no run was reached > - The whole API then stopped answering for about 480 s: plain GETs timed out, CPU was idle, and no run events were saved > - The process identity write of each run started and did not commit, so the reattach phase ended at its 180 s cap with 0 of 6 reached > - The wave kept no `pg_stat_activity` or `pg_locks` data, so the head of the lock chain cannot be proven from the evidence > - This pull request records that data on the server at the moment the stall is detected > - The benefit is that the next occurrence names the blocking session and its query, so the real fix targets one proven holder ## Linked Issues or Issue Description **What happened?** In a restart-chaos K15 release swap, the outgoing server got SIGKILL. The incoming server reattached six remote runs after their leases expired. All six runs authenticated. None was reached. The reattach phase ended at the cap. The server did not answer any API request from about 22:35:24Z to 22:43:23Z. Then all buffered events arrived in the same second. The task finished without help, but the runs were silent for about 8 minutes. **Expected behavior** The runs are reached within seconds, as in the hand-off swap of the same wave (6 of 6 reached in 13 s). If a stall occurs, the server log names what holds the database. **Steps to reproduce** Run a release swap that kills the outgoing server with SIGKILL while six remote runs are live. Wait for their controller leases to expire. **Paperclip version or commit** master at 3452639a81 (restart-chaos wave w-20261116T220803-s7105). **Database mode** External Postgres on loopback, owned outside the server process tree. A follow-up check ruled out one cause: a SIGKILLed client's row lock on this loopback path is released in 4 ms, so no orphaned session from the killed server held the run rows. ## What Changed - `reattach-phase.ts`: when the reattach phase ends at its cap and some runs authenticated but were never reached, the server reads `pg_stat_activity` with `pg_blocking_pids` on the private liveness pool. It logs the waiting sessions, their blockers, long transactions, and the dedicated connection stats. - The read uses a 5 s statement timeout and a pool that a stalled main pool cannot starve. - Test: an embedded Postgres test holds a run row lock, starts a second waiting transaction, ends the phase at the cap with an authenticated run, and checks that the log names the holder and the waiter. ## Verification - `cd server && npx vitest run src/services/heartbeat/reattach-phase.test.ts` passes (36 tests). - Red/green: with the source change reverted, the new test fails (`expected false to be true`). With the change, it passes. - `npx tsc --noEmit -p server` passes. - `pnpm verify` asked for broader coverage (opaque setup dependency), so required CI gives that coverage. ## Risks - Low risk. The read runs only when a cap ends with authenticated but unreached runs. It runs on the liveness pool with a 5 s timeout. A failure is logged and ignored. - Query text in the log is cut to 160 characters. Drizzle sends parameters apart from the text, so values do not appear. - This change does not remove the stall. It makes its cause visible for the next fix. ## Model Used - Claude Opus 5.5 (`claude-opus-5-5`), Anthropic, with tool use and code execution in Claude Code. ## Checklist - [x] I have included a thinking path that traces from project context to this change - [x] I have specified the model used (with version and capability details) - [x] I have checked ROADMAP.md and confirmed this PR does not duplicate planned core work - [x] I have searched GitHub for duplicate or related PRs and linked them above - [x] I have either (a) linked existing issues with `Fixes: #` / `Closes #` / `Refs #` OR (b) described the issue in-PR following the relevant issue template - [x] I have not referenced internal/instance-local Paperclip issues or links (only public GitHub `#NNN` / `github.com/paperclipai/paperclip` URLs) - [x] My branch name describes the change (e.g. `docs/...`, `fix/...`) and contains no internal Paperclip ticket id or instance-derived details - [x] I have run tests locally and they pass - [x] I have added or updated tests where applicable - [x] I have updated relevant documentation to reflect my changes - [x] I have considered and documented any risks above - [ ] All Paperclip CI gates are green - [ ] Greptile is 5/5 with no open P2s, recommendations, or follow-ups - [x] I will address all Greptile and reviewer comments before requesting merge 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 --- .../services/heartbeat/reattach-phase.test.ts | 58 ++++++++++++- .../src/services/heartbeat/reattach-phase.ts | 81 ++++++++++++++++++- 2 files changed, 133 insertions(+), 6 deletions(-) diff --git a/server/src/services/heartbeat/reattach-phase.test.ts b/server/src/services/heartbeat/reattach-phase.test.ts index 4a6fb49d195..352502cfd8d 100644 --- a/server/src/services/heartbeat/reattach-phase.test.ts +++ b/server/src/services/heartbeat/reattach-phase.test.ts @@ -1,6 +1,7 @@ import type { Db } from "@paperclipai/db"; -import { agents, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; -import { eq } from "drizzle-orm"; +import { agents, closeLivenessDb, companies, createDb, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS, heartbeatRuns, issues, nativeRunFinalizations } from "@paperclipai/db"; +import { eq, sql } from "drizzle-orm"; +import { logger } from "../../middleware/logger.js"; import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest"; import { createHeartbeatQueueTestDependencies as callbacks } from "../../__tests__/helpers/heartbeat-queue.js"; import { resolveExternalTestDatabaseUrl } from "../../__tests__/helpers/external-test-database.js"; @@ -24,6 +25,7 @@ import { } from "./run-admission.js"; import { claimableReattachRun, + type DatabaseSessionWait, handOverMoment, listLiveRunsOwnedByPreviousControllers, noteReattachRunsClaimed, @@ -265,6 +267,7 @@ describe.skipIf(!support.supported)("reattach phase discovery", () => { db = createDb(external ?? database!.connectionString, { maxConnections: 4 }); }, EMBEDDED_POSTGRES_TEST_TIMEOUT_MS); afterAll(async () => { + if (db) await closeLivenessDb(db, 1); await db?.$client.end(); await database?.cleanup(); }); @@ -612,5 +615,56 @@ describe.skipIf(!support.supported)("reattach phase discovery", () => { expect(isReattachPhaseActive()).toBe(false); expect(ended).toEqual([expect.objectContaining({ reason: "nothing_to_reattach", gatedStarts: false })]); }); + + it("names the database sessions that hold an authenticated run's row when the phase ends at the cap, even with the main pool stalled", async () => { + await retireEarlierRuns(); + const { run } = await seed("RPS"); + const stalled = await run("running", liveLease()); + const stalledDb = createDb(external ?? database!.connectionString, { maxConnections: 2 }); + const warn = vi.spyOn(logger, "warn"); + let release!: () => void; + const held = new Promise(resolve => { release = resolve; }); + let locked!: (pid: number) => void; + const holderPid = new Promise(resolve => { locked = resolve; }); + const holder = stalledDb.transaction(async tx => { + const [backend] = await tx.execute<{ pid: number }>(sql`select pg_backend_pid() as pid`); + await tx.execute(sql.raw(`select id from heartbeat_runs where id = '${stalled}' and next_event_seq >= 0 for update`)); + locked(Number(backend.pid)); + await held; + }); + const holderBackend = await holderPid; + const waiter = stalledDb.transaction(async tx => { + await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, stalled)).for("no key update"); + }); + let coordinator: ReturnType | undefined; + try { + const ended: ReattachPhaseSummary[] = []; + coordinator = createDeliveryWorkCoordinator({ owner: stalledDb, canRun: () => true, onError: () => undefined }); + registerReattachPhaseWork(coordinator, stalledDb, { capMs: 300, bootId: "incoming-boot", recheckMs: 20, + onEnd: summary => ended.push(summary) }); + joinReattachPhase([{ runId: stalled, reason: "lease_expired", claimableAt: Date.now() }]); + markReattachAuthenticated(stalled); + await vi.waitFor(() => expect(ended.some(summary => summary.reason === "cap_reached")).toBe(true), { timeout: 2_000, interval: 10 }); + const stallMessage = "reattach phase ended with authenticated runs that were never reached; database sessions waiting or holding locks"; + await vi.waitFor(() => expect(warn.mock.calls.some(call => call[1] === stallMessage)).toBe(true), { timeout: 5_000, interval: 20 }); + const [fields] = warn.mock.calls.find(call => call[1] === stallMessage)! as unknown as [{ runIds: string[]; waits: DatabaseSessionWait[] }]; + expect(fields.runIds).toEqual([stalled]); + expect(fields.waits).toEqual(expect.arrayContaining([ + expect.objectContaining({ pid: holderBackend }), + expect.objectContaining({ waitEventType: "Lock", blockedBy: [holderBackend] }), + ])); + expect(JSON.stringify(fields.waits)).not.toContain(stalled); + expect(fields.waits.find(wait => wait.pid === holderBackend)?.query).toContain("where id = ? and next_event_seq >= ? for update"); + } finally { + release(); + await holder; + await waiter; + await stopReattachWatch(); + await coordinator?.stop(); + warn.mockRestore(); + await closeLivenessDb(stalledDb, 1); + await stalledDb.$client.end(); + } + }); }); }); diff --git a/server/src/services/heartbeat/reattach-phase.ts b/server/src/services/heartbeat/reattach-phase.ts index 50a03d29b3d..09f43e180f7 100644 --- a/server/src/services/heartbeat/reattach-phase.ts +++ b/server/src/services/heartbeat/reattach-phase.ts @@ -1,5 +1,5 @@ -import { and, eq, inArray, isNotNull, isNull, ne, notLike, or } from "drizzle-orm"; -import { heartbeatRuns, nativeRunFinalizations, type Db } from "@paperclipai/db"; +import { and, eq, inArray, isNotNull, isNull, ne, notLike, or, sql } from "drizzle-orm"; +import { dedicatedDbConnectionStats, heartbeatRuns, livenessDb, nativeRunFinalizations, type Db } from "@paperclipai/db"; import { logger } from "../../middleware/logger.js"; import type { createDeliveryWorkCoordinator } from "../delivery-work-coordinator.js"; import { DELIVERY_QUEUES, notifyDeliveryWork } from "../delivery-work-notifications.js"; @@ -18,6 +18,7 @@ import { settleReattachRun, type ClaimableReattachRun, type ReattachClaimableReason, + type ReattachPhaseSummary, } from "./run-admission.js"; export const REATTACH_PHASE_DEFAULT_CAP_MS = 180_000; @@ -95,6 +96,7 @@ type ReattachWatch = { worker: ReattachWorker | null; stopListening: () => void; claim?: (runIds: string[]) => Promise; + stallReports: Set>; }; let watch: ReattachWatch | null = null; @@ -215,9 +217,18 @@ export function registerReattachPhaseWork( if (capMs <= 0) return null; if (watch) void stopReattachWatch(); enableReattachPhase(capMs); + const stopEndListener = options.onEnd ? onReattachPhaseEnd(options.onEnd) : () => undefined; const current: ReattachWatch = { db, bootId: options.bootId ?? currentControllerBootId(), initialized: false, - notified: false, watching: false, leases: new Map(), worker: null, claim: options.claim, - stopListening: options.onEnd ? onReattachPhaseEnd(options.onEnd) : () => undefined }; + notified: false, watching: false, leases: new Map(), worker: null, claim: options.claim, stallReports: new Set(), + stopListening: () => undefined }; + const stopStallReport = onReattachPhaseEnd(summary => { + if (watch !== current || !stalledReattach(summary)) return; + const report: Promise = reportStalledReattach(db, summary) + .catch(error => logger.warn({ err: error }, "reattach phase could not read database session waits")) + .finally(() => { current.stallReports.delete(report); }); + current.stallReports.add(report); + }); + current.stopListening = () => { stopEndListener(); stopStallReport(); }; watch = current; const release = () => { if (watch !== current) return; @@ -235,10 +246,72 @@ export function registerReattachPhaseWork( return current.worker; } +export type DatabaseSessionWait = { + pid: number; + applicationName: string | null; + state: string | null; + waitEventType: string | null; + waitEvent: string | null; + blockedBy: number[]; + transactionSeconds: number | null; + stateSeconds: number | null; + query: string | null; +}; + +export async function readDatabaseSessionWaits(db: Db): Promise { + return livenessDb(db).transaction(async tx => { + await tx.execute(sql`select set_config('statement_timeout', '5000', true)`); + const rows = await tx.execute>(sql` + with sessions as ( + select pid, application_name, state, wait_event_type, wait_event, pg_blocking_pids(pid) as blocked_by, + xact_start, state_change, query + from pg_stat_activity + where datname = current_database() and pid <> pg_backend_pid() and backend_type = 'client backend' + ), blockers as (select distinct unnest(blocked_by) as pid from sessions) + select pid, application_name, state, wait_event_type, wait_event, blocked_by, + extract(epoch from clock_timestamp() - xact_start)::int as transaction_seconds, + extract(epoch from clock_timestamp() - state_change)::int as state_seconds, + left(regexp_replace(regexp_replace(query, '''(?:[^'']|'''')*''', '?', 'g'), '\\m[0-9][0-9.]*\\M', '?', 'g'), 160) as query + from sessions + where cardinality(blocked_by) > 0 or pid in (select pid from blockers) + or state like 'idle in transaction%' + or (xact_start is not null and clock_timestamp() - xact_start > interval '30 seconds') + order by xact_start nulls last + limit 50`); + return rows.map(row => ({ + pid: Number(row.pid), + applicationName: (row.application_name as string | null) ?? null, + state: (row.state as string | null) ?? null, + waitEventType: (row.wait_event_type as string | null) ?? null, + waitEvent: (row.wait_event as string | null) ?? null, + blockedBy: Array.isArray(row.blocked_by) ? row.blocked_by.map(Number) : [], + transactionSeconds: row.transaction_seconds === null ? null : Number(row.transaction_seconds), + stateSeconds: row.state_seconds === null ? null : Number(row.state_seconds), + query: (row.query as string | null) ?? null, + })); + }); +} + +function authenticatedButUnreached(summary: ReattachPhaseSummary): string[] { + return summary.runs.filter(run => run.outcome === "outstanding" && run.authenticatedMs !== null).map(run => run.runId); +} + +function stalledReattach(summary: ReattachPhaseSummary): boolean { + return summary.reason === "cap_reached" && authenticatedButUnreached(summary).length > 0; +} + +async function reportStalledReattach(db: Db, summary: ReattachPhaseSummary): Promise { + const runIds = authenticatedButUnreached(summary); + const waits = await readDatabaseSessionWaits(db); + logger.warn({ runIds, pool: dedicatedDbConnectionStats(db), waits }, + "reattach phase ended with authenticated runs that were never reached; database sessions waiting or holding locks"); +} + export async function stopReattachWatch(): Promise { const current = watch; if (!current) return; watch = null; current.stopListening(); await current.worker?.stop(); + await Promise.all(current.stallReports); }