From 2d6c3dc597931f97f5dcb5efc141e65cc1c1ded4 Mon Sep 17 00:00:00 2001 From: Priya Raman Date: Thu, 27 Aug 2026 21:05:38 +0000 Subject: [PATCH] feat(server): measure the Postgres driver null-socket crash on fatal errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a process-level handler for uncaughtException and unhandledRejection. It reports a fixed-cardinality verdict to first-party telemetry — whether the crash matches the known postgres null-socket write defect — and calls process.exit(1) on every path, so it does not change today's crash behavior. This adds measurement only; it does not guard against the defect. Co-authored-by: Paperclip --- .../shared/src/telemetry/retention.test.ts | 10 + packages/shared/src/telemetry/retention.ts | 9 + .../src/__tests__/fatal-error-handler.test.ts | Bin 0 -> 7504 bytes server/src/fatal-error-handler.ts | 182 ++++++++++++++++++ server/src/index.ts | 6 + 5 files changed, 207 insertions(+) create mode 100644 server/src/__tests__/fatal-error-handler.test.ts create mode 100644 server/src/fatal-error-handler.ts diff --git a/packages/shared/src/telemetry/retention.test.ts b/packages/shared/src/telemetry/retention.test.ts index 9d54db1371b..d11cad1974b 100644 --- a/packages/shared/src/telemetry/retention.test.ts +++ b/packages/shared/src/telemetry/retention.test.ts @@ -23,4 +23,14 @@ describe("telemetry retention contract", () => { expect(EVENT_RETENTION_CLASS["interaction.created"]).toBe("operational_enum_count"); expect(EVENT_RETENTION_CLASS["interaction.resolved"]).toBe("operational_enum_count"); }); + + it("process.fatal_error is assigned the operational_enum_count class", () => { + expect(EVENT_RETENTION_CLASS["process.fatal_error"]).toBe("operational_enum_count"); + }); + + it("process.fatal_error resolves to 90 days", () => { + const cls = EVENT_RETENTION_CLASS["process.fatal_error"]; + expect(cls).toBeDefined(); + expect(RETENTION_DAYS[cls!]).toBe(90); + }); }); diff --git a/packages/shared/src/telemetry/retention.ts b/packages/shared/src/telemetry/retention.ts index 543e1085298..4c881226b7d 100644 --- a/packages/shared/src/telemetry/retention.ts +++ b/packages/shared/src/telemetry/retention.ts @@ -42,9 +42,18 @@ export type RetentionClass = keyof typeof RETENTION_DAYS; * counts (rotations detected). No token material and no PII. * External-system event (Codex CLI) — not yet in PaperclipEventName. * Class: operational_enum_count — 90-day window. + * + * process.fatal_error + * Emits fatal-crash observability fields: enums (crash source, error + * class), a boolean (matches the known Postgres driver null-socket write + * defect), and the installed Postgres driver version. No error message, no + * stack, no query text, no connection URL, no host name, no database name. + * Not yet in PaperclipEventName. + * Class: operational_enum_count — 90-day window. */ export const EVENT_RETENTION_CLASS: Partial> = { "codex.credential_health": "operational_enum_count", "interaction.created": "operational_enum_count", "interaction.resolved": "operational_enum_count", + "process.fatal_error": "operational_enum_count", }; diff --git a/server/src/__tests__/fatal-error-handler.test.ts b/server/src/__tests__/fatal-error-handler.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..71a2ca153ac5b386613f1d02b0f153177aa34c00 GIT binary patch literal 7504 zcmeHMZEqXL5$VcYHM^$7+fIHDFs>?%T{f^UO0#Rn^9NaV_N3gMBIU`9NI2)K~~V7vOTM zE?^+wT@AS(2!(&Il(-R7YbudkDG$ykPkX)GXy-*`^2@KmpF;^1c)L6;6=*M>#Eso5IBJL;_= zu-4eEv8nX3+!(u|qvzh&e`5pJJ`4U0P`MU!snHd$X#Gpt7}4dvvg!)#cTHKo#?XTQ z(IUc5t#ot5+;brpQhA{mwj6gp{fO^%6!v3>QW%F1ANIsU@uEQoIw8g6d|3k@G!cur zaUh(R`K8FEwF-EKhIiq_3F2E;Ae5#pCbe;XX2A`e&4+k>jk$pM#L!I!qLLoNp=aoQ zz(B}Sk>GJgX9zY22;-+=A-XWM>o8bC@Qf@( zOb` z%jS&M2fI-o9Y1+8I)2puNhe!}*Vg1{eRl_u(xD@zkx#WOhh9}+-RRTLJ{cW9?juol zxylAV?cMZxEz73KA^dl&Mmev1nlSDVw*)&=7GT8>pNZ%u;c6&L0_8&aIZr7#vjVO+ z_Ck(f?rvkWH^j-Q)Y^E$cYzQ#HCVqC4;GZ34}_TtjVYUOBr%p;EYd^Pq$t z=@b(at7?q({RDHrwNqmr{CZob5@}>Ru3N@3X{y=0GbbpE4IeNUA2f55SN4H*zkLC} zD+{R%ZIF-(c|OMlRzQABE{|!3yC1qN1eSZ?r>BKTWY~_bO#oph`c0sruQ2S-t&GU(UJDy z!f|_9U|Rp^m@xOPYQeTT+Q5Ga9$8lT*Y$EF&RG0W*q(X(2RRFsv8$Kg>l0?vm7mf4CiJ zbb+!+L(vz~A^lxyv(Wsr2F!|e66t!C4Qt~0?Zrq2wFaO!8V6^n%+9$Qhw-&iLu-kE z_0d1T8sy)CMU%xrqNgC|upOaaw6PURJW_KQ5$TpXkKIP_YN(|oyDz_5YFVi~xP+P= zs{U=Yot6!1T5gQ!qxRmWyn^BfSvHWaToTsp@2%oOlK)}ZwOBuVAi_ap?dtFkF7*dl zj%p!q9@P0Za6_p_6kzir8)W)1v<+UjGAf3BP$P-(S9t|50Q4#9UMPMTpkvOcEm25E(ClxiK@4S|UnD3GooOsJ=KGQ&bndYw8t2qctf`Qs z{Wzd)Ou~EYxLF>voEf1_4TM>3+DFvK@<12M+Ojr|`$W{X=7m;nk45iWn3bSseje6; zjd~e7pmXGc^bu#?PBO8=3q4HHWm4#I*W@pIH#CR8{|q8cbS9BMqA+p>WtBnZup-!r zFfA0-#f}(nrPRQ-#n=ML%Jrpf%-2d01!t2>_om5*XU83(zI)jg%$z?h)>l5d1K3&E zuA_YiY1&Y>gCuab~JD3x#IYX;v(xFE|A_(=1Yb3Hp<(;p}}3@B`3gm*y35>wm~a1Px`Be)R;7{`5`Oi<@iW}y*XyisX^7M zbh>RY@p-5I53XeA#s2d1&qeYF>hJK?U0#&XQtb^_$9Ruu#w9FoHzITRmX9y$c+;1^ zG_p+J@J`ryzOPbr*_H6cIQ>>*ElRcu#A-sQVe$93J1oB`s_`{32O(r0ppAdy3x}7U7zO(SQyNCcf&v8o8y1qN@g_fMJrwmDG6*(~BjA>q}}= z)8p~uM?s9Wv+1(a*#H0l literal 0 HcmV?d00001 diff --git a/server/src/fatal-error-handler.ts b/server/src/fatal-error-handler.ts new file mode 100644 index 00000000000..16aad64e00c --- /dev/null +++ b/server/src/fatal-error-handler.ts @@ -0,0 +1,182 @@ +// Process-level fatal-error handler for `uncaughtException` and +// `unhandledRejection`. +// +// The server registers neither handler today, so Node ends the process on +// either event by default (its built-in crash behavior). Registering a +// handler cancels that default, so this module must call `process.exit(1)` +// on every path — the normal path, the path where the telemetry flush +// rejects, and the path where the flush times out. `sentry.ts` documents the +// same hazard for `unhandledRejection` and keeps `mode: "strict"` for the +// same reason; this module does not weaken that setting. +// +// This module adds measurement only. It reports one classifier — whether the +// crash matches the known Postgres driver null-socket write defect — and one +// denominator (every fatal error, matched or not). It does not change the +// crash-and-restart behavior, and it does not guard against the defect. + +import { createRequire } from "node:module"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { TelemetryClient } from "@paperclipai/shared/telemetry"; +import { captureException } from "./sentry.js"; +import { getTelemetryClient } from "./telemetry.js"; +import { logger } from "./middleware/logger.js"; + +const FLUSH_TIMEOUT_MS = 2_000; + +export type FatalErrorSource = "uncaught_exception" | "unhandled_rejection"; + +// Both V8 message spellings for a read of a property on `null`. The wording +// changed between an older V8 (`Cannot read property 'write' of null`) and a +// newer V8 (`Cannot read properties of null (reading 'write')`). +const NULL_WRITE_MESSAGE_PATTERN = + /Cannot read propert(?:y 'write' of null|ies of null \(reading 'write'\))/; + +// Matches the frame, not an absolute path, so the check survives any +// install-path prefix (a package-store path, a different path separator). +const NEXT_WRITE_FRAME_PATTERN = /\bnextWrite\b.*postgres[\\/]src[\\/]connection\.js/; + +/** + * Verdict-only fingerprint for the known Postgres driver defect: `postgres` + * reserves a physical connection for a transaction. When the database + * backend closes that connection, `nextWrite` still calls `socket.write` on + * the now-null socket. This matches on three facts together — the error + * class, the message, and the crashing frame — so an unrelated `TypeError` + * never counts. + */ +export function isDriverNullSocketWriteCrash(error: unknown): boolean { + if (!(error instanceof Error) || error.name !== "TypeError") return false; + if (!NULL_WRITE_MESSAGE_PATTERN.test(error.message)) return false; + const stack = error.stack ?? ""; + return stack.split("\n").some((line) => NEXT_WRITE_FRAME_PATTERN.test(line)); +} + +/** + * Reads the installed `postgres` driver's own declared version, without + * importing or running the package. The server does not depend on + * `postgres` directly — `@paperclipai/db` does — so this walks through that + * package's own dependency graph, the same install the crash came from. + * Returns "unknown" when the version cannot be read. The field is a + * fixed-cardinality dimension: it never carries a raw path or an error. + */ +function readPostgresDriverVersion(): string { + try { + const requireFromHere = createRequire(import.meta.url); + const dbEntry = requireFromHere.resolve("@paperclipai/db"); + const requireFromDb = createRequire(dbEntry); + let dir = dirname(requireFromDb.resolve("postgres")); + for (;;) { + const candidate = join(dir, "package.json"); + if (existsSync(candidate)) { + const parsed = JSON.parse(readFileSync(candidate, "utf8")) as { + name?: unknown; + version?: unknown; + }; + if (parsed.name === "postgres" && typeof parsed.version === "string") { + return parsed.version; + } + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } catch { + // Fall through to the sentinel below. + } + return "unknown"; +} + +/** + * Emits the measurement event with a dynamic event name and dimension set. + * The generated telemetry catalogue has no generator in this repository (see + * `generated/paperclip-telemetry.ts`), so this follows the shipped + * `codex.credential_health` precedent — `trackDynamic`, registered in + * `EVENT_RETENTION_CLASS` — instead of `client.track()`. + * + * Emits only fixed-cardinality classifiers. Never the error message, the + * stack, query text, a connection URL, a host name, or a database name. + */ +function trackProcessFatalError( + client: TelemetryClient, + dims: { + source: FatalErrorSource; + errorCode: string; + driverNullSocketWrite: boolean; + postgresDriverVersion: string; + }, +): void { + client.trackDynamic("process.fatal_error", { + source: dims.source, + error_code: dims.errorCode, + driver_null_socket_write: dims.driverNullSocketWrite, + postgres_driver_version: dims.postgresDriverVersion, + }); +} + +/** + * Awaits `client.flush()` up to `timeoutMs`, then resolves either way. Never + * rejects: a fatal-error handler that itself throws or hangs would turn a + * crash into a silent, undefined-state survival, the exact outcome this + * module exists to prevent. + * + * The flush keeps running in the background after a timeout wins the race. + * Attaching both branches of `.then` here, instead of leaving the promise + * dangling, keeps a late rejection from firing as a second, unhandled + * rejection. + */ +function flushWithTimeout(client: TelemetryClient, timeoutMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + resolve(); + }; + const timer = setTimeout(finish, timeoutMs); + timer.unref?.(); + client.flush().then(finish, finish); + }); +} + +/** + * Handles one fatal error end to end: local log, Sentry report, the + * measurement event, a bounded telemetry flush, then process exit. Exported + * so a test can drive it directly with a fake `unhandledRejection` reason or + * a fake telemetry client, instead of dispatching a real process event. + */ +export async function handleFatalError(error: unknown, source: FatalErrorSource): Promise { + const rootError = error instanceof Error ? error : new Error(String(error)); + try { + logger.fatal({ err: rootError, source }, "fatal error; process exiting"); + captureException(rootError); + const client = getTelemetryClient(); + if (client) { + trackProcessFatalError(client, { + source, + errorCode: rootError.name, + driverNullSocketWrite: isDriverNullSocketWriteCrash(rootError), + postgresDriverVersion: readPostgresDriverVersion(), + }); + await flushWithTimeout(client, FLUSH_TIMEOUT_MS); + } + } catch { + // An observability failure must never change the exit path below. + } finally { + process.exit(1); + } +} + +/** + * Registers the process-level fatal-error handlers. Node ends the process on + * an uncaught exception or an unhandled rejection today because the server + * registers neither handler. This function measures the crash, then ends the + * process itself on every path — see the module comment above. + */ +export function registerFatalErrorHandlers(): void { + process.on("uncaughtException", (error) => { + void handleFatalError(error, "uncaught_exception"); + }); + process.on("unhandledRejection", (reason) => { + void handleFatalError(reason, "unhandled_rejection"); + }); +} diff --git a/server/src/index.ts b/server/src/index.ts index e668adf7a8f..2004ce56f56 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -5,6 +5,7 @@ // HTTP server, so trace coverage does not depend on incidental timing. import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js"; import { sentryReady, shutdownSentry, captureException } from "./sentry.js"; +import { registerFatalErrorHandlers } from "./fatal-error-handler.js"; import { existsSync, readFileSync, rmSync } from "node:fs"; import { createServer } from "node:http"; import { resolve } from "node:path"; @@ -166,6 +167,11 @@ export async function startServer(): Promise { ensureDecisionSigningSecret(); let config = loadConfig(); initTelemetry({ enabled: config.telemetryEnabled }); + // Registered after Sentry and telemetry are ready, so the handler's own + // reporting calls have a live client on their first possible invocation. + // See fatal-error-handler.ts for why this handler must call + // `process.exit(1)` on every path. + registerFatalErrorHandlers(); if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) { process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider; }