diff --git a/packages/shared/src/telemetry/retention.test.ts b/packages/shared/src/telemetry/retention.test.ts index 9d54db1371b..d11cad1974b 100644 --- a/packages/shared/src/telemetry/retention.test.ts +++ b/packages/shared/src/telemetry/retention.test.ts @@ -23,4 +23,14 @@ describe("telemetry retention contract", () => { expect(EVENT_RETENTION_CLASS["interaction.created"]).toBe("operational_enum_count"); expect(EVENT_RETENTION_CLASS["interaction.resolved"]).toBe("operational_enum_count"); }); + + it("process.fatal_error is assigned the operational_enum_count class", () => { + expect(EVENT_RETENTION_CLASS["process.fatal_error"]).toBe("operational_enum_count"); + }); + + it("process.fatal_error resolves to 90 days", () => { + const cls = EVENT_RETENTION_CLASS["process.fatal_error"]; + expect(cls).toBeDefined(); + expect(RETENTION_DAYS[cls!]).toBe(90); + }); }); diff --git a/packages/shared/src/telemetry/retention.ts b/packages/shared/src/telemetry/retention.ts index 543e1085298..4c881226b7d 100644 --- a/packages/shared/src/telemetry/retention.ts +++ b/packages/shared/src/telemetry/retention.ts @@ -42,9 +42,18 @@ export type RetentionClass = keyof typeof RETENTION_DAYS; * counts (rotations detected). No token material and no PII. * External-system event (Codex CLI) — not yet in PaperclipEventName. * Class: operational_enum_count — 90-day window. + * + * process.fatal_error + * Emits fatal-crash observability fields: enums (crash source, error + * class), a boolean (matches the known Postgres driver null-socket write + * defect), and the installed Postgres driver version. No error message, no + * stack, no query text, no connection URL, no host name, no database name. + * Not yet in PaperclipEventName. + * Class: operational_enum_count — 90-day window. */ export const EVENT_RETENTION_CLASS: Partial> = { "codex.credential_health": "operational_enum_count", "interaction.created": "operational_enum_count", "interaction.resolved": "operational_enum_count", + "process.fatal_error": "operational_enum_count", }; diff --git a/server/src/__tests__/fatal-error-handler.test.ts b/server/src/__tests__/fatal-error-handler.test.ts new file mode 100644 index 00000000000..71a2ca153ac Binary files /dev/null and b/server/src/__tests__/fatal-error-handler.test.ts differ diff --git a/server/src/fatal-error-handler.ts b/server/src/fatal-error-handler.ts new file mode 100644 index 00000000000..16aad64e00c --- /dev/null +++ b/server/src/fatal-error-handler.ts @@ -0,0 +1,182 @@ +// Process-level fatal-error handler for `uncaughtException` and +// `unhandledRejection`. +// +// The server registers neither handler today, so Node ends the process on +// either event by default (its built-in crash behavior). Registering a +// handler cancels that default, so this module must call `process.exit(1)` +// on every path — the normal path, the path where the telemetry flush +// rejects, and the path where the flush times out. `sentry.ts` documents the +// same hazard for `unhandledRejection` and keeps `mode: "strict"` for the +// same reason; this module does not weaken that setting. +// +// This module adds measurement only. It reports one classifier — whether the +// crash matches the known Postgres driver null-socket write defect — and one +// denominator (every fatal error, matched or not). It does not change the +// crash-and-restart behavior, and it does not guard against the defect. + +import { createRequire } from "node:module"; +import { existsSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import type { TelemetryClient } from "@paperclipai/shared/telemetry"; +import { captureException } from "./sentry.js"; +import { getTelemetryClient } from "./telemetry.js"; +import { logger } from "./middleware/logger.js"; + +const FLUSH_TIMEOUT_MS = 2_000; + +export type FatalErrorSource = "uncaught_exception" | "unhandled_rejection"; + +// Both V8 message spellings for a read of a property on `null`. The wording +// changed between an older V8 (`Cannot read property 'write' of null`) and a +// newer V8 (`Cannot read properties of null (reading 'write')`). +const NULL_WRITE_MESSAGE_PATTERN = + /Cannot read propert(?:y 'write' of null|ies of null \(reading 'write'\))/; + +// Matches the frame, not an absolute path, so the check survives any +// install-path prefix (a package-store path, a different path separator). +const NEXT_WRITE_FRAME_PATTERN = /\bnextWrite\b.*postgres[\\/]src[\\/]connection\.js/; + +/** + * Verdict-only fingerprint for the known Postgres driver defect: `postgres` + * reserves a physical connection for a transaction. When the database + * backend closes that connection, `nextWrite` still calls `socket.write` on + * the now-null socket. This matches on three facts together — the error + * class, the message, and the crashing frame — so an unrelated `TypeError` + * never counts. + */ +export function isDriverNullSocketWriteCrash(error: unknown): boolean { + if (!(error instanceof Error) || error.name !== "TypeError") return false; + if (!NULL_WRITE_MESSAGE_PATTERN.test(error.message)) return false; + const stack = error.stack ?? ""; + return stack.split("\n").some((line) => NEXT_WRITE_FRAME_PATTERN.test(line)); +} + +/** + * Reads the installed `postgres` driver's own declared version, without + * importing or running the package. The server does not depend on + * `postgres` directly — `@paperclipai/db` does — so this walks through that + * package's own dependency graph, the same install the crash came from. + * Returns "unknown" when the version cannot be read. The field is a + * fixed-cardinality dimension: it never carries a raw path or an error. + */ +function readPostgresDriverVersion(): string { + try { + const requireFromHere = createRequire(import.meta.url); + const dbEntry = requireFromHere.resolve("@paperclipai/db"); + const requireFromDb = createRequire(dbEntry); + let dir = dirname(requireFromDb.resolve("postgres")); + for (;;) { + const candidate = join(dir, "package.json"); + if (existsSync(candidate)) { + const parsed = JSON.parse(readFileSync(candidate, "utf8")) as { + name?: unknown; + version?: unknown; + }; + if (parsed.name === "postgres" && typeof parsed.version === "string") { + return parsed.version; + } + } + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + } catch { + // Fall through to the sentinel below. + } + return "unknown"; +} + +/** + * Emits the measurement event with a dynamic event name and dimension set. + * The generated telemetry catalogue has no generator in this repository (see + * `generated/paperclip-telemetry.ts`), so this follows the shipped + * `codex.credential_health` precedent — `trackDynamic`, registered in + * `EVENT_RETENTION_CLASS` — instead of `client.track()`. + * + * Emits only fixed-cardinality classifiers. Never the error message, the + * stack, query text, a connection URL, a host name, or a database name. + */ +function trackProcessFatalError( + client: TelemetryClient, + dims: { + source: FatalErrorSource; + errorCode: string; + driverNullSocketWrite: boolean; + postgresDriverVersion: string; + }, +): void { + client.trackDynamic("process.fatal_error", { + source: dims.source, + error_code: dims.errorCode, + driver_null_socket_write: dims.driverNullSocketWrite, + postgres_driver_version: dims.postgresDriverVersion, + }); +} + +/** + * Awaits `client.flush()` up to `timeoutMs`, then resolves either way. Never + * rejects: a fatal-error handler that itself throws or hangs would turn a + * crash into a silent, undefined-state survival, the exact outcome this + * module exists to prevent. + * + * The flush keeps running in the background after a timeout wins the race. + * Attaching both branches of `.then` here, instead of leaving the promise + * dangling, keeps a late rejection from firing as a second, unhandled + * rejection. + */ +function flushWithTimeout(client: TelemetryClient, timeoutMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const finish = () => { + if (settled) return; + settled = true; + resolve(); + }; + const timer = setTimeout(finish, timeoutMs); + timer.unref?.(); + client.flush().then(finish, finish); + }); +} + +/** + * Handles one fatal error end to end: local log, Sentry report, the + * measurement event, a bounded telemetry flush, then process exit. Exported + * so a test can drive it directly with a fake `unhandledRejection` reason or + * a fake telemetry client, instead of dispatching a real process event. + */ +export async function handleFatalError(error: unknown, source: FatalErrorSource): Promise { + const rootError = error instanceof Error ? error : new Error(String(error)); + try { + logger.fatal({ err: rootError, source }, "fatal error; process exiting"); + captureException(rootError); + const client = getTelemetryClient(); + if (client) { + trackProcessFatalError(client, { + source, + errorCode: rootError.name, + driverNullSocketWrite: isDriverNullSocketWriteCrash(rootError), + postgresDriverVersion: readPostgresDriverVersion(), + }); + await flushWithTimeout(client, FLUSH_TIMEOUT_MS); + } + } catch { + // An observability failure must never change the exit path below. + } finally { + process.exit(1); + } +} + +/** + * Registers the process-level fatal-error handlers. Node ends the process on + * an uncaught exception or an unhandled rejection today because the server + * registers neither handler. This function measures the crash, then ends the + * process itself on every path — see the module comment above. + */ +export function registerFatalErrorHandlers(): void { + process.on("uncaughtException", (error) => { + void handleFatalError(error, "uncaught_exception"); + }); + process.on("unhandledRejection", (reason) => { + void handleFatalError(reason, "unhandled_rejection"); + }); +} diff --git a/server/src/index.ts b/server/src/index.ts index e668adf7a8f..2004ce56f56 100644 --- a/server/src/index.ts +++ b/server/src/index.ts @@ -5,6 +5,7 @@ // HTTP server, so trace coverage does not depend on incidental timing. import { instrumentationReady, shutdownInstrumentation } from "./instrumentation.js"; import { sentryReady, shutdownSentry, captureException } from "./sentry.js"; +import { registerFatalErrorHandlers } from "./fatal-error-handler.js"; import { existsSync, readFileSync, rmSync } from "node:fs"; import { createServer } from "node:http"; import { resolve } from "node:path"; @@ -166,6 +167,11 @@ export async function startServer(): Promise { ensureDecisionSigningSecret(); let config = loadConfig(); initTelemetry({ enabled: config.telemetryEnabled }); + // Registered after Sentry and telemetry are ready, so the handler's own + // reporting calls have a live client on their first possible invocation. + // See fatal-error-handler.ts for why this handler must call + // `process.exit(1)` on every path. + registerFatalErrorHandlers(); if (process.env.PAPERCLIP_SECRETS_PROVIDER === undefined) { process.env.PAPERCLIP_SECRETS_PROVIDER = config.secretsProvider; }