feat(server): measure the Postgres driver null-socket crash on fatal errors

Add a process-level handler for uncaughtException and unhandledRejection.
It reports a fixed-cardinality verdict to first-party telemetry — whether
the crash matches the known postgres null-socket write defect — and calls
process.exit(1) on every path, so it does not change today's crash
behavior. This adds measurement only; it does not guard against the
defect.

Co-authored-by: Paperclip <noreply@paperclip.ing>

packages/shared/src/telemetry/retention.test.ts

@@ -23,4 +23,14 @@ describe("telemetry retention contract", () => {
     expect(EVENT_RETENTION_CLASS["interaction.created"]).toBe("operational_enum_count");
     expect(EVENT_RETENTION_CLASS["interaction.resolved"]).toBe("operational_enum_count");
   });
+
+  it("process.fatal_error is assigned the operational_enum_count class", () => {
+    expect(EVENT_RETENTION_CLASS["process.fatal_error"]).toBe("operational_enum_count");
+  });
+
+  it("process.fatal_error resolves to 90 days", () => {
+    const cls = EVENT_RETENTION_CLASS["process.fatal_error"];
+    expect(cls).toBeDefined();
+    expect(RETENTION_DAYS[cls!]).toBe(90);
+  });
 });

packages/shared/src/telemetry/retention.ts

@@ -42,9 +42,18 @@ export type RetentionClass = keyof typeof RETENTION_DAYS;
  *   counts (rotations detected). No token material and no PII.
  *   External-system event (Codex CLI) — not yet in PaperclipEventName.
  *   Class: operational_enum_count — 90-day window.
+ *
+ * process.fatal_error
+ *   Emits fatal-crash observability fields: enums (crash source, error
+ *   class), a boolean (matches the known Postgres driver null-socket write
+ *   defect), and the installed Postgres driver version. No error message, no
+ *   stack, no query text, no connection URL, no host name, no database name.
+ *   Not yet in PaperclipEventName.
+ *   Class: operational_enum_count — 90-day window.
  */
 export const EVENT_RETENTION_CLASS: Partial<Record<string, RetentionClass>> = {
   "codex.credential_health": "operational_enum_count",
   "interaction.created": "operational_enum_count",
   "interaction.resolved": "operational_enum_count",
+  "process.fatal_error": "operational_enum_count",
 };

server/src/fatal-error-handler.ts

@@ -0,0 +1,182 @@
+// Process-level fatal-error handler for `uncaughtException` and
+// `unhandledRejection`.
+//
+// The server registers neither handler today, so Node ends the process on
+// either event by default (its built-in crash behavior). Registering a
+// handler cancels that default, so this module must call `process.exit(1)`
+// on every path — the normal path, the path where the telemetry flush
+// rejects, and the path where the flush times out. `sentry.ts` documents the
+// same hazard for `unhandledRejection` and keeps `mode: "strict"` for the
+// same reason; this module does not weaken that setting.
+//
+// This module adds measurement only. It reports one classifier — whether the
+// crash matches the known Postgres driver null-socket write defect — and one
+// denominator (every fatal error, matched or not). It does not change the
+// crash-and-restart behavior, and it does not guard against the defect.
+
+import { createRequire } from "node:module";
+import { existsSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import type { TelemetryClient } from "@paperclipai/shared/telemetry";
+import { captureException } from "./sentry.js";
+import { getTelemetryClient } from "./telemetry.js";
+import { logger } from "./middleware/logger.js";
+
+const FLUSH_TIMEOUT_MS = 2_000;
+
+export type FatalErrorSource = "uncaught_exception" | "unhandled_rejection";
+
+// Both V8 message spellings for a read of a property on `null`. The wording
+// changed between an older V8 (`Cannot read property 'write' of null`) and a
+// newer V8 (`Cannot read properties of null (reading 'write')`).
+const NULL_WRITE_MESSAGE_PATTERN =
+  /Cannot read propert(?:y 'write' of null|ies of null \(reading 'write'\))/;
+
+// Matches the frame, not an absolute path, so the check survives any
+// install-path prefix (a package-store path, a different path separator).
+const NEXT_WRITE_FRAME_PATTERN = /\bnextWrite\b.*postgres[\\/]src[\\/]connection\.js/;
+
+/**
+ * Verdict-only fingerprint for the known Postgres driver defect: `postgres`
+ * reserves a physical connection for a transaction. When the database
+ * backend closes that connection, `nextWrite` still calls `socket.write` on
+ * the now-null socket. This matches on three facts together — the error
+ * class, the message, and the crashing frame — so an unrelated `TypeError`
+ * never counts.
+ */
+export function isDriverNullSocketWriteCrash(error: unknown): boolean {
+  if (!(error instanceof Error) || error.name !== "TypeError") return false;
+  if (!NULL_WRITE_MESSAGE_PATTERN.test(error.message)) return false;
+  const stack = error.stack ?? "";
+  return stack.split("\n").some((line) => NEXT_WRITE_FRAME_PATTERN.test(line));
+}
+
+/**
+ * Reads the installed `postgres` driver's own declared version, without
+ * importing or running the package. The server does not depend on
+ * `postgres` directly — `@paperclipai/db` does — so this walks through that
+ * package's own dependency graph, the same install the crash came from.
+ * Returns "unknown" when the version cannot be read. The field is a
+ * fixed-cardinality dimension: it never carries a raw path or an error.
+ */
+function readPostgresDriverVersion(): string {
+  try {
+    const requireFromHere = createRequire(import.meta.url);
+    const dbEntry = requireFromHere.resolve("@paperclipai/db");
+    const requireFromDb = createRequire(dbEntry);
+    let dir = dirname(requireFromDb.resolve("postgres"));
+    for (;;) {
+      const candidate = join(dir, "package.json");
+      if (existsSync(candidate)) {
+        const parsed = JSON.parse(readFileSync(candidate, "utf8")) as {
+          name?: unknown;
+          version?: unknown;
+        };
+        if (parsed.name === "postgres" && typeof parsed.version === "string") {
+          return parsed.version;
+        }
+      }
+      const parent = dirname(dir);
+      if (parent === dir) break;
+      dir = parent;
+    }
+  } catch {
+    // Fall through to the sentinel below.
+  }
+  return "unknown";
+}
+
+/**
+ * Emits the measurement event with a dynamic event name and dimension set.
+ * The generated telemetry catalogue has no generator in this repository (see
+ * `generated/paperclip-telemetry.ts`), so this follows the shipped
+ * `codex.credential_health` precedent — `trackDynamic`, registered in
+ * `EVENT_RETENTION_CLASS` — instead of `client.track()`.
+ *
+ * Emits only fixed-cardinality classifiers. Never the error message, the
+ * stack, query text, a connection URL, a host name, or a database name.
+ */
+function trackProcessFatalError(
+  client: TelemetryClient,
+  dims: {
+    source: FatalErrorSource;
+    errorCode: string;
+    driverNullSocketWrite: boolean;
+    postgresDriverVersion: string;
+  },
+): void {
+  client.trackDynamic("process.fatal_error", {
+    source: dims.source,
+    error_code: dims.errorCode,
+    driver_null_socket_write: dims.driverNullSocketWrite,
+    postgres_driver_version: dims.postgresDriverVersion,
+  });
+}
+
+/**
+ * Awaits `client.flush()` up to `timeoutMs`, then resolves either way. Never
+ * rejects: a fatal-error handler that itself throws or hangs would turn a
+ * crash into a silent, undefined-state survival, the exact outcome this
+ * module exists to prevent.
+ *
+ * The flush keeps running in the background after a timeout wins the race.
+ * Attaching both branches of `.then` here, instead of leaving the promise
+ * dangling, keeps a late rejection from firing as a second, unhandled
+ * rejection.
+ */
+function flushWithTimeout(client: TelemetryClient, timeoutMs: number): Promise<void> {
+  return new Promise((resolve) => {
+    let settled = false;
+    const finish = () => {
+      if (settled) return;
+      settled = true;
+      resolve();
+    };
+    const timer = setTimeout(finish, timeoutMs);
+    timer.unref?.();
+    client.flush().then(finish, finish);
+  });
+}
+
+/**
+ * Handles one fatal error end to end: local log, Sentry report, the
+ * measurement event, a bounded telemetry flush, then process exit. Exported
+ * so a test can drive it directly with a fake `unhandledRejection` reason or
+ * a fake telemetry client, instead of dispatching a real process event.
+ */
+export async function handleFatalError(error: unknown, source: FatalErrorSource): Promise<void> {
+  const rootError = error instanceof Error ? error : new Error(String(error));
+  try {
+    logger.fatal({ err: rootError, source }, "fatal error; process exiting");
+    captureException(rootError);
+    const client = getTelemetryClient();
+    if (client) {
+      trackProcessFatalError(client, {
+        source,
+        errorCode: rootError.name,
+        driverNullSocketWrite: isDriverNullSocketWriteCrash(rootError),
+        postgresDriverVersion: readPostgresDriverVersion(),
+      });
+      await flushWithTimeout(client, FLUSH_TIMEOUT_MS);
+    }
+  } catch {
+    // An observability failure must never change the exit path below.
+  } finally {
+    process.exit(1);
+  }
+}
+
+/**
+ * Registers the process-level fatal-error handlers. Node ends the process on
+ * an uncaught exception or an unhandled rejection today because the server
+ * registers neither handler. This function measures the crash, then ends the
+ * process itself on every path — see the module comment above.
+ */
+export function registerFatalErrorHandlers(): void {
+  process.on("uncaughtException", (error) => {
+    void handleFatalError(error, "uncaught_exception");
+  });
+  process.on("unhandledRejection", (reason) => {
+    void handleFatalError(reason, "unhandled_rejection");
+  });
+}

server/src/index.ts

@@ -5,6 +5,7 @@
 // HTTP server, so trace coverage does not depend on incidental timing.
 import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js";
 import { sentryReady, shutdownSentry, captureException } from "./sentry.js";
+import { registerFatalErrorHandlers } from "./fatal-error-handler.js";
 import { existsSync, readFileSync, rmSync } from "node:fs";
 import { createServer } from "node:http";
 import { resolve } from "node:path";
@@ -166,6 +167,11 @@ export async function startServer(): Promise<StartedServer> {
   ensureDecisionSigningSecret();
   let config = loadConfig();
   initTelemetry({ enabled: config.telemetryEnabled });
+  // Registered after Sentry and telemetry are ready, so the handler's own
+  // reporting calls have a live client on their first possible invocation.
+  // See fatal-error-handler.ts for why this handler must call
+  // `process.exit(1)` on every path.
+  registerFatalErrorHandlers();
   if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) {
     process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider;
   }