Files
PaperClipAI/server/src/__tests__/fatal-error-handler.test.ts
T
Priya RamanandPaperclip 2d6c3dc597 feat(server): measure the Postgres driver null-socket crash on fatal errors
Add a process-level handler for uncaughtException and unhandledRejection.
It reports a fixed-cardinality verdict to first-party telemetry — whether
the crash matches the known postgres null-socket write defect — and calls
process.exit(1) on every path, so it does not change today's crash
behavior. This adds measurement only; it does not guard against the
defect.

Co-authored-by: Paperclip <noreply@paperclip.ing>
2026-08-27 21:05:38 +00:00

202 lines
7.3 KiB
TypeScript

import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mockGetTelemetryClient = vi.hoisted(() => vi.fn());
const mockCaptureException = vi.hoisted(() => vi.fn());
vi.mock("../telemetry.js", () => ({
getTelemetryClient: mockGetTelemetryClient,
}));
vi.mock("../sentry.js", () => ({
captureException: mockCaptureException,
}));
vi.mock("../middleware/logger.js", () => ({
logger: {
child: vi.fn(function child() {
return this;
}),
trace: vi.fn(),
debug: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
fatal: vi.fn(),
},
httpLogger: vi.fn(),
}));
const { handleFatalError, isDriverNullSocketWriteCrash } = await import("../fatal-error-handler.js");
/**
* Builds a `TypeError` whose stack carries a `nextWrite` frame in
* `postgres/src/connection.js`, matching the real crash signature. The path
* prefix before `postgres/src/connection.js` is a stand-in for any install
* layout (a package-store path in this repository, a plain
* `node_modules/postgres` path elsewhere).
*/
function makeDriverCrashError(message: string): Error {
const error = new TypeError(message);
error.stack =
`TypeError: ${message}\n` +
" at Immediate.nextWrite (/app/node_modules/.pnpm/postgres@3.4.9/node_modules/postgres/src/connection.js:255:20)\n" +
" at processImmediate (node:internal/timers:483:21)";
return error;
}
describe("isDriverNullSocketWriteCrash", () => {
it("matches the older V8 message spelling with the driver frame", () => {
const error = makeDriverCrashError("Cannot read property 'write' of null");
expect(isDriverNullSocketWriteCrash(error)).toBe(true);
});
it("matches the newer V8 message spelling with the driver frame", () => {
const error = makeDriverCrashError("Cannot read properties of null (reading 'write')");
expect(isDriverNullSocketWriteCrash(error)).toBe(true);
});
it("does not match an unrelated TypeError with a different message", () => {
const error = makeDriverCrashError("Cannot read properties of null (reading 'foo')");
expect(isDriverNullSocketWriteCrash(error)).toBe(false);
});
it("does not match the right message from an unrelated frame", () => {
const error = new TypeError("Cannot read properties of null (reading 'write')");
error.stack =
"TypeError: Cannot read properties of null (reading 'write')\n" +
" at Object.<anonymous> (/app/server/src/services/unrelated.js:12:5)";
expect(isDriverNullSocketWriteCrash(error)).toBe(false);
});
it("does not match a non-TypeError with the same message and frame", () => {
const error = new Error("Cannot read properties of null (reading 'write')");
error.stack =
"Error: Cannot read properties of null (reading 'write')\n" +
" at Immediate.nextWrite (/app/node_modules/postgres/src/connection.js:255:20)";
expect(isDriverNullSocketWriteCrash(error)).toBe(false);
});
});
describe("handleFatalError", () => {
let exitSpy: ReturnType<typeof vi.spyOn>;
beforeEach(() => {
mockGetTelemetryClient.mockReset();
mockCaptureException.mockReset();
exitSpy = vi.spyOn(process, "exit").mockImplementation(((() => undefined) as unknown) as never);
});
afterEach(() => {
exitSpy.mockRestore();
vi.useRealTimers();
});
it("exits non-zero and reports the crash on the normal path", async () => {
const trackDynamic = vi.fn();
const flush = vi.fn().mockResolvedValue(undefined);
mockGetTelemetryClient.mockReturnValue({ trackDynamic, flush });
const error = makeDriverCrashError("Cannot read properties of null (reading 'write')");
await handleFatalError(error, "uncaught_exception");
expect(mockCaptureException).toHaveBeenCalledWith(error);
expect(flush).toHaveBeenCalledTimes(1);
expect(exitSpy).toHaveBeenCalledWith(1);
});
it("emits the fingerprint verdict and the driver version, matched case", async () => {
const trackDynamic = vi.fn();
const flush = vi.fn().mockResolvedValue(undefined);
mockGetTelemetryClient.mockReturnValue({ trackDynamic, flush });
const error = makeDriverCrashError("Cannot read properties of null (reading 'write')");
await handleFatalError(error, "uncaught_exception");
expect(trackDynamic).toHaveBeenCalledTimes(1);
const [eventName, dims] = trackDynamic.mock.calls[0]!;
expect(eventName).toBe("process.fatal_error");
expect(dims).toMatchObject({
source: "uncaught_exception",
error_code: "TypeError",
driver_null_socket_write: true,
});
expect(typeof dims.postgres_driver_version).toBe("string");
expect(dims.postgres_driver_version.length).toBeGreaterThan(0);
});
it("emits an unmatched verdict for an unrelated fatal error", async () => {
const trackDynamic = vi.fn();
const flush = vi.fn().mockResolvedValue(undefined);
mockGetTelemetryClient.mockReturnValue({ trackDynamic, flush });
const error = new RangeError("some other fatal condition");
await handleFatalError(error, "unhandled_rejection");
expect(trackDynamic).toHaveBeenCalledTimes(1);
const [, dims] = trackDynamic.mock.calls[0]!;
expect(dims).toMatchObject({
source: "unhandled_rejection",
error_code: "RangeError",
driver_null_socket_write: false,
});
});
it("emits no dimension that carries the error message or the stack", async () => {
const trackDynamic = vi.fn();
const flush = vi.fn().mockResolvedValue(undefined);
mockGetTelemetryClient.mockReturnValue({ trackDynamic, flush });
const secretMessage = "connection to postgres://user:hunter2@db.internal/prod failed";
const error = new Error(secretMessage);
await handleFatalError(error, "uncaught_exception");
const [, dims] = trackDynamic.mock.calls[0]!;
for (const value of Object.values(dims as Record<string, unknown>)) {
expect(String(value)).not.toContain(secretMessage);
expect(String(value)).not.toContain(error.stack ?? "unreachable");
}
expect(Object.keys(dims as Record<string, unknown>).sort()).toEqual([
"driver_null_socket_write",
"error_code",
"postgres_driver_version",
"source",
]);
});
it("exits non-zero when the telemetry flush rejects", async () => {
const trackDynamic = vi.fn();
const flush = vi.fn().mockRejectedValue(new Error("network down"));
mockGetTelemetryClient.mockReturnValue({ trackDynamic, flush });
const error = new Error("boom");
await handleFatalError(error, "uncaught_exception");
expect(exitSpy).toHaveBeenCalledWith(1);
});
it("exits non-zero when the telemetry flush times out", async () => {
vi.useFakeTimers();
const trackDynamic = vi.fn();
// A flush that never settles — the hard timeout must still resolve the
// handler and drive it to `process.exit(1)`.
const flush = vi.fn(() => new Promise<void>(() => {}));
mockGetTelemetryClient.mockReturnValue({ trackDynamic, flush });
const error = new Error("boom");
const done = handleFatalError(error, "uncaught_exception");
await vi.advanceTimersByTimeAsync(2_100);
await done;
expect(exitSpy).toHaveBeenCalledWith(1);
});
it("exits non-zero even when there is no telemetry client", async () => {
mockGetTelemetryClient.mockReturnValue(null);
const error = new Error("boom");
await handleFatalError(error, "uncaught_exception");
expect(exitSpy).toHaveBeenCalledWith(1);
});
});