diff --git a/server/src/__tests__/helpers/maintenance/database-startup.ts b/server/src/__tests__/helpers/maintenance/database-startup.ts new file mode 100644 index 00000000000..64134837a6c --- /dev/null +++ b/server/src/__tests__/helpers/maintenance/database-startup.ts @@ -0,0 +1,53 @@ +import { channel } from "node:diagnostics_channel"; +import type { Socket } from "node:net"; +import { createPostgresProtocolObserver } from "./postgres-proxy.js"; + +const realTimeout = globalThis.setTimeout; + +export function databasePort(databaseUrl: string) { + return Number(new URL(databaseUrl).port) || 5432; +} + +export function trackDatabaseConnectionStartup(port: number) { + const starting = new Set(); + let errors: Error[] = []; + const subscriber = (message: unknown) => { + const socket = (message as { socket: Socket }).socket; + starting.add(socket); + socket.once("close", () => starting.delete(socket)); + socket.once("connect", () => { + if (socket.remotePort !== port) { starting.delete(socket); return; } + const observer = createPostgresProtocolObserver(() => {}); + const onData = (chunk: Buffer) => { + try { observer.backend(chunk); } + catch (error) { errors.push(error as Error); starting.delete(socket); socket.off("data", onData); return; } + if (observer.ready) { starting.delete(socket); socket.off("data", onData); } + }; + socket.on("data", onData); + }); + }; + channel("net.client.socket").subscribe(subscriber); + function failIfObservationFailed() { + if (!errors.length) return; + const failed = errors; + errors = []; + throw new AggregateError(failed, "Database connection startup could not be observed"); + } + return { + get starting() { return starting.size; }, + async waitUntilStarted(timeoutMs = 30_000) { + const deadline = performance.now() + timeoutMs; + while (starting.size > 0) { + failIfObservationFailed(); + if (performance.now() >= deadline) throw new Error(`Database connection startup did not finish within ${timeoutMs}ms`); + await new Promise(resolve => realTimeout(resolve, 5)); + } + failIfObservationFailed(); + }, + async advanceAfterStartup(tick: (ms: number) => void, ms: number, timeoutMs = 30_000) { + do await this.waitUntilStarted(timeoutMs); while (starting.size > 0); + tick(ms); + }, + dispose() { channel("net.client.socket").unsubscribe(subscriber); }, + }; +} diff --git a/server/src/__tests__/helpers/maintenance/server-child.ts b/server/src/__tests__/helpers/maintenance/server-child.ts index 833726d930e..5418246708b 100644 --- a/server/src/__tests__/helpers/maintenance/server-child.ts +++ b/server/src/__tests__/helpers/maintenance/server-child.ts @@ -2,6 +2,7 @@ import { channel } from "node:diagnostics_channel"; import { mock } from "node:test"; import type { Db } from "@paperclipai/db"; import type { Socket } from "node:net"; +import { databasePort, trackDatabaseConnectionStartup } from "./database-startup.js"; import { observeMaintenanceIncomingHttp } from "./incoming-http-observation.js"; const realTimeout = globalThis.setTimeout, realInterval = globalThis.setInterval, clearRealInterval = globalThis.clearInterval, realNow = Date.now; @@ -26,11 +27,12 @@ if (process.env.PAPERCLIP_MAINTENANCE_GITHUB_ORIGIN) { }; } +const databaseStartup = trackDatabaseConnectionStartup(databasePort(process.env.DATABASE_URL!)); let outgoingHttpRequests = 0; const sockets: Array<{ socket: Socket; database: boolean }> = []; channel("net.client.socket").subscribe(message => { const socket = (message as { socket: Socket }).socket; - socket.once("connect", () => sockets.push({ socket, database: socket.remotePort === Number(new URL(process.env.DATABASE_URL!).port) })); + socket.once("connect", () => sockets.push({ socket, database: socket.remotePort === databasePort(process.env.DATABASE_URL!) })); }); function network() { return { nonDatabaseConnections: sockets.filter(entry => !entry.database).length, @@ -83,7 +85,7 @@ try { if (message.command === "realtime" && fakeClock) runClock(); if (message.command === "advance") { if (!fakeClock || typeof message.ms !== "number" || !Number.isFinite(message.ms) || message.ms < 0 || message.ms > 30_000) throw new Error("Invalid maintenance clock advance"); - mock.timers.tick(message.ms); + await databaseStartup.advanceAfterStartup(ms => mock.timers.tick(ms), message.ms); } if (message.command === "drain") startTaskDrain({ purpose: "idle" }); if (message.command === "resume") stopTaskDrain(); diff --git a/server/src/__tests__/maintenance-clock-database-startup.test.ts b/server/src/__tests__/maintenance-clock-database-startup.test.ts new file mode 100644 index 00000000000..ecce084467a --- /dev/null +++ b/server/src/__tests__/maintenance-clock-database-startup.test.ts @@ -0,0 +1,121 @@ +import { createRequire } from "node:module"; +import { once } from "node:events"; +import { createConnection, createServer, type Server, type Socket } from "node:net"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { databasePort, trackDatabaseConnectionStartup } from "./helpers/maintenance/database-startup.js"; + +const databaseRequire = createRequire(createRequire(import.meta.url).resolve("@paperclipai/db")); +type PostgresClient = { + (strings: TemplateStringsArray): { simple(): { execute(): Promise } }; + end(options: { timeout: number }): Promise; +}; +const postgres = (await import(pathToFileURL(databaseRequire.resolve("postgres")).href)).default as (options: Record) => PostgresClient; +const realTimeout = globalThis.setTimeout; +const realImmediate = globalThis.setImmediate; + +function frame(type: string, body: Buffer) { + const header = Buffer.alloc(5); + header[0] = type.charCodeAt(0); header.writeInt32BE(body.length + 4, 1); + return Buffer.concat([header, body]); +} + +async function startHeldPostgres(startupReply = Buffer.concat([frame("R", Buffer.alloc(4)), frame("Z", Buffer.from("I"))])) { + const held: Socket[] = [], sockets = new Set(); + const server: Server = createServer(socket => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + socket.once("data", () => { + held.push(socket); + socket.on("data", () => socket.write(Buffer.concat([frame("C", Buffer.from("SELECT 0\0")), frame("Z", Buffer.from("I"))]))); + }); + socket.on("error", () => {}); + }); + await new Promise((resolve, reject) => { server.once("error", reject); server.listen(0, "127.0.0.1", resolve); }); + const port = (server.address() as { port: number }).port; + return { + port, + held, + release() { + for (const socket of held.splice(0)) { + if (!socket.destroyed) socket.write(startupReply); + } + }, + async stop() { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => server.close(error => error ? reject(error) : resolve())); + }, + }; +} + +async function startConnection(port: number, tracker: ReturnType, upstream: { held: Socket[] }, cleanup: Array<() => unknown>) { + const sql = postgres({ host: "127.0.0.1", port, username: "maintenance", database: "maintenance", password: "unused", + ssl: false, fetch_types: false, connect_timeout: 30, max: 1, onnotice: () => {} }); + const query = sql`select 1`.simple().execute(); + query.catch(() => {}); + cleanup.push(() => sql.end({ timeout: 0 })); + const deadline = performance.now() + 5_000; + while (tracker.starting === 0 || upstream.held.length === 0) { + if (performance.now() > deadline) throw new Error("PostgreSQL client did not start its connection"); + vi.advanceTimersByTime(0); + await new Promise(resolve => realImmediate(resolve)); + } + return { query }; +} + +describe("maintenance fake clock and database connection startup", () => { + const cleanup: Array<() => unknown> = []; + afterEach(async () => { + vi.useRealTimers(); + for (const step of cleanup.splice(0).reverse()) await step(); + }); + + it("fires the driver connect timeout when the clock advances during connection startup", async () => { + const upstream = await startHeldPostgres(); cleanup.push(() => upstream.stop()); + const tracker = trackDatabaseConnectionStartup(upstream.port); cleanup.push(() => tracker.dispose()); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const { query } = await startConnection(upstream.port, tracker, upstream, cleanup); + vi.advanceTimersByTime(30_000); + upstream.release(); + await expect(query).rejects.toMatchObject({ code: "CONNECT_TIMEOUT" }); + }); + + it("advances the clock only after every database connection has finished startup", async () => { + const upstream = await startHeldPostgres(); cleanup.push(() => upstream.stop()); + const tracker = trackDatabaseConnectionStartup(upstream.port); cleanup.push(() => tracker.dispose()); + vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + const { query } = await startConnection(upstream.port, tracker, upstream, cleanup); + expect(tracker.starting).toBe(1); + realTimeout(() => upstream.release(), 50); + const ticks: number[] = []; + await tracker.advanceAfterStartup(ms => { ticks.push(ms); expect(tracker.starting).toBe(0); vi.advanceTimersByTime(ms); }, 30_000, 5_000); + expect(ticks).toEqual([30_000]); + await expect(query).resolves.toHaveLength(0); + }); + + it("reports an unreadable startup reply once and keeps later advances usable", async () => { + const upstream = await startHeldPostgres(Buffer.from([0x52, 0, 0, 0, 0])); cleanup.push(() => upstream.stop()); + const tracker = trackDatabaseConnectionStartup(upstream.port); cleanup.push(() => tracker.dispose()); + const client = createConnection({ host: "127.0.0.1", port: upstream.port }); + let clientError: Error | undefined; + client.on("error", error => { clientError = error; }); cleanup.push(() => client.destroy()); + await once(client, "connect"); + client.write(Buffer.from([0, 0, 0, 8, 0, 3, 0, 0])); + const deadline = performance.now() + 5_000; + while (upstream.held.length === 0) { + if (clientError) throw clientError; + if (performance.now() > deadline) throw new Error("PostgreSQL stub did not receive the startup message"); + await new Promise(resolve => realTimeout(resolve, 5)); + } + expect(tracker.starting).toBe(1); + upstream.release(); + await expect(tracker.waitUntilStarted(5_000)).rejects.toThrow("Database connection startup could not be observed"); + expect(tracker.starting).toBe(0); + await expect(tracker.waitUntilStarted(5_000)).resolves.toBeUndefined(); + }); + + it("uses the PostgreSQL default port when the database URL omits it", () => { + expect(databasePort("postgres://maintenance@127.0.0.1/maintenance_test")).toBe(5432); + expect(databasePort("postgres://maintenance@127.0.0.1:6543/maintenance_test")).toBe(6543); + }); +});