diff --git a/doc/DATABASE.md b/doc/DATABASE.md index bc1292bbe94..e6211773d87 100644 --- a/doc/DATABASE.md +++ b/doc/DATABASE.md @@ -332,12 +332,21 @@ The diagnostic measures awaited Postgres.js `unsafe` responses before Drizzle column decoding. It retains the Drizzle builder caller for normal, prepared, transaction, and dedicated-pool execution. It issues no queries. It logs the public server caller locations, static schema table, operation, -query/row counts, and response value sizes. It emits at most 128 aggregate +query/row counts, and response value sizes. It emits at most 512 aggregate groups and a summary at the deadline. It never logs SQL, parameters, result values, connection strings, or query errors. In-flight queries continue normally. +For a bundled server, keep the linked source maps and enable Node's +`--enable-source-maps` option so the diagnostic can resolve public caller +locations. Restore the prior Node options after capture. Sizes still count when +caller locations are unavailable. + Look for `paperclip.db_response_capture.entry.v1` and -`paperclip.db_response_capture.summary.v1` in the process logs. Tagged raw-driver calls, streams, unawaited raw-driver +`paperclip.db_response_capture.summary.v1` in the process logs. A log provider +can store `event` as a structured field instead of message text. Filter by that +field or read the bounded window from the exact deployment. Check the summary +and group totals, including the unattributed overflow group. +Tagged raw-driver calls, streams, unawaited raw-driver dispatch, and protocol overhead are not measured. Relational queries use a driver-level caller fallback. Queries still in flight at the deadline are reported separately. Sizes describe driver values before column decoding, not billed network bytes. Complex parsed diff --git a/packages/db/src/response-capture.test.ts b/packages/db/src/response-capture.test.ts index 65f58d10883..4b87c7924a4 100644 --- a/packages/db/src/response-capture.test.ts +++ b/packages/db/src/response-capture.test.ts @@ -107,11 +107,12 @@ describe("bounded database response capture", () => { }); it("bounds cardinality without dropping response-byte totals", () => { const f = fixture(); - for (let i = 0; i < 200; i += 1) f.capture.execute(() => [i], { ...metadata, callers: [`services/f${i}.ts:1:1`] }); + for (let i = 0; i < 600; i += 1) f.capture.execute(() => [i], { ...metadata, callers: [`services/f${i}.ts:1:1`] }); f.capture.flush(); const summary = f.events.at(-1)!; - expect(summary).toMatchObject({ startedQueries: 200, completedQueries: 200, groups: 128, overflowCalls: 73 }); - expect(summary.responseValueBytes).toBe(Array.from({ length: 200 }, (_, i) => Buffer.byteLength(JSON.stringify([i]))).reduce((a, b) => a + b, 0)); + expect(summary).toMatchObject({ startedQueries: 600, completedQueries: 600, groups: 512, overflowCalls: 89 }); + expect(f.events).toHaveLength(513); + expect(summary.responseValueBytes).toBe(Array.from({ length: 600 }, (_, i) => Buffer.byteLength(JSON.stringify([i]))).reduce((a, b) => a + b, 0)); }); it("does not execute builders or prepared statements until requested", async () => { const f = fixture(), rows = [{ id: 1 }], execute = vi.fn(async () => rows); diff --git a/packages/db/src/response-capture.ts b/packages/db/src/response-capture.ts index 8f738043f4d..f8e6da8e87a 100644 --- a/packages/db/src/response-capture.ts +++ b/packages/db/src/response-capture.ts @@ -99,7 +99,7 @@ export function createDatabaseResponseCapture( completed += 1; try { let key = JSON.stringify(metadata); - if (!entries.has(key) && entries.size >= 127) { + if (!entries.has(key) && entries.size >= 511) { overflowCalls += 1; metadata = { operation: "other", table: "other", callers: [] }; key = "overflow"; diff --git a/server/scripts/bundle-runtime.test.mjs b/server/scripts/bundle-runtime.test.mjs index 14b0d46c681..6e9375f61c4 100644 --- a/server/scripts/bundle-runtime.test.mjs +++ b/server/scripts/bundle-runtime.test.mjs @@ -2,7 +2,7 @@ import { execFile } from "node:child_process"; import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; @@ -85,6 +85,33 @@ describe("bundleRuntime", () => { expect(output.entryDir).toBe(path.join(root, "server", "dist")); }); + it("keeps database caller locations public in a source-mapped runtime bundle", async () => { + const captureModule = fileURLToPath(new URL("../../packages/db/src/response-capture.ts", import.meta.url)); + write(path.join(root, "server", "dist", "services", "response-caller.js"), [ + `import { publicDatabaseCallers } from ${JSON.stringify(captureModule)};`, + 'export function callers() { return publicDatabaseCallers(new Error("private-capture-message").stack); }', + ].join("\n")); + write(path.join(root, "server", "dist", "index.js"), [ + 'import { callers } from "./services/response-caller.js";', + 'console.log(JSON.stringify({ callers: callers() }));', + ].join("\n")); + const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir: path.join(root, "server", "dist-runtime") }); + const env = { ...process.env }; + delete env.NODE_OPTIONS; + delete env.PAPERCLIP_DB_RESPONSE_CAPTURE_WINDOW; + const options = { cwd: path.join(root, "server"), env, timeout: 30_000 }; + const plain = JSON.parse((await execFileAsync(process.execPath, [summary.outfile], options)).stdout); + const mapped = JSON.parse((await execFileAsync(process.execPath, ["--enable-source-maps", summary.outfile], options)).stdout); + expect(plain.callers).toEqual([]); + expect(mapped.callers).toEqual([expect.stringMatching(/^services\/response-caller\.js:\d+:\d+$/)]); + for (const result of [plain, mapped]) { + const output = JSON.stringify(result); + expect(output).not.toContain(root); + expect(output).not.toContain(captureModule); + expect(output).not.toContain("private-capture-message"); + } + }); + it("emits a server module that loads without starting the server", async () => { const outdir = path.join(root, "server", "dist-runtime"); const summary = await bundleRuntime({ entry: path.join(root, "server", "dist", "index.js"), outdir });