diff --git a/packages/db/src/work-signals.test.ts b/packages/db/src/work-signals.test.ts index fdfb9202233..aabf0529911 100644 --- a/packages/db/src/work-signals.test.ts +++ b/packages/db/src/work-signals.test.ts @@ -4,7 +4,7 @@ import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; import { createDb, withDedicatedDbConnection } from "./client.js"; import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./test-embedded-postgres.js"; import { workSignalDebug } from "./work-signal-debug.js"; -import { DATABASE_WORK_CHANNEL, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, publishDatabaseWorkSettled, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; +import { DATABASE_WORK_CHANNEL, DATABASE_WORK_NOTIFICATION_MAX_BYTES, DATABASE_WORK_TOPIC_NOTIFICATIONS_MAX_BYTES, databaseWorkScopeKey, databaseWorkPending, enableDatabaseWorkNotifications, installDatabaseWorkSignals, notifyDatabaseWorkOnCommit, primeDatabaseWorkTransactionId, publishDatabaseWorkSettled, receiveDatabaseWorkNotification, reconcileDatabaseWork, signalDatabaseWork, subscribeDatabaseWork } from "./work-signals.js"; const support = await getEmbeddedPostgresTestSupport(); (support.supported ? describe : describe.skip)("database work signals", () => { @@ -339,8 +339,11 @@ describe("database work scope identity", () => { expect(notifyStatements).toBe(0); })); expect(result).toMatchObject({ statements: 1, sent: ["a", "b", "c"], delivered: ["a", "b", "c"], failure: null }); - expect(result.payloads.map(payload => JSON.parse(payload))).toEqual(["a", "b", "c"].map(name => - ({ version: 1, origin: expect.any(String), topic: name }))); + expect(result.payloads.map(payload => JSON.parse(payload))).toEqual([ + { version: 1, origin: expect.any(String), topic: "a", hints: ["one", "two"] }, + { version: 1, origin: expect.any(String), topic: "b" }, + { version: 1, origin: expect.any(String), topic: "c" }, + ]); }); it("sends nothing for a transaction that signals nothing", async () => { @@ -424,4 +427,133 @@ describe("database work scope identity", () => { await root.transaction(callback); })).toMatchObject({ statements: 5, delivered: ["a", "a", "b", "b", "c"] }); }); + + function legacyReceive(payload: string): string | null { + if (payload.length > 1024) return null; + let value: { version?: unknown; origin?: unknown; topic?: unknown }; + try { value = JSON.parse(payload); } catch { return null; } + if (!value || value.version !== 1 || typeof value.origin !== "string" + || typeof value.topic !== "string" || !/^[a-zA-Z0-9:_-]{1,200}$/.test(value.topic)) return null; + return value.topic; + } + function receiving(topics: string[]) { + const receiver = createDb(temporary.connectionString, { maxConnections: 1 }); + const wakes: Array<[string, string[] | null]> = []; + const stops = topics.map(name => subscribeDatabaseWork(receiver, name, (event, hints) => { + wakes.push([name, hints ? [...hints] : null]); + expect(event).toBe("settled"); + })); + return { + receiver, wakes, + deliver(payloads: string[]) { for (const payload of payloads) receiveDatabaseWorkNotification(receiver, payload); }, + async close() { for (const stop of stops) stop(); await receiver.$client.end({ timeout: 1 }); }, + }; + } + + it("carries a commit's deduplicated hints, so another process wakes only those scopes", async () => { + const other = receiving(["hinted", "unhinted", "mixed"]); + try { + const result = await run(root => root.transaction(async tx => { + await signalDatabaseWork(tx, "hinted", "company-1"); + await tx.transaction(async nested => { + await signalDatabaseWork(nested, "hinted", "company-2"); + publishDatabaseWorkSettled(nested, "hinted", "company-1"); + }); + await tx.transaction(async nested => { await signalDatabaseWork(nested, "hinted", "rolled-back"); rollback(); }).catch(() => {}); + await signalDatabaseWork(tx, "unhinted"); + await signalDatabaseWork(tx, "mixed", "company-3"); + publishDatabaseWorkSettled(tx, "mixed"); + })); + expect(result).toMatchObject({ statements: 1, sent: ["hinted", "unhinted", "mixed"], failure: null }); + expect(result.payloads.map(payload => JSON.parse(payload).hints)).toEqual([["company-1", "company-2"], undefined, undefined]); + other.deliver(result.payloads); + expect(other.wakes).toEqual([["hinted", ["company-1", "company-2"]], ["unhinted", null], ["mixed", null]]); + expect(result.payloads.map(legacyReceive)).toEqual(["hinted", "unhinted", "mixed"]); + } finally { await other.close(); } + }); + + it("splits many hints into payloads an older receiver still reads, within the per-topic byte cap", async () => { + const hints = Array.from({ length: 80 }, (_, index) => JSON.stringify({ companyId: `company-${String(index).padStart(4, "0")}` })); + const other = receiving(["many"]); + try { + const result = await run(root => root.transaction(async tx => { for (const hint of hints) await signalDatabaseWork(tx, "many", hint); })); + expect(result.statements).toBe(1); + expect(result.payloads.length).toBeGreaterThan(1); + for (const payload of result.payloads) expect(Buffer.byteLength(payload)).toBeLessThanOrEqual(DATABASE_WORK_NOTIFICATION_MAX_BYTES); + expect(result.payloads.reduce((total, payload) => total + Buffer.byteLength(payload), 0)) + .toBeLessThanOrEqual(DATABASE_WORK_TOPIC_NOTIFICATIONS_MAX_BYTES); + expect(result.payloads.flatMap(payload => JSON.parse(payload).hints)).toEqual(hints); + expect(result.payloads.map(legacyReceive)).toEqual(result.payloads.map(() => "many")); + other.deliver(result.payloads); + expect(other.wakes.flatMap(([, received]) => received ?? ["*"])).toEqual(hints); + } finally { await other.close(); } + }); + + it("falls back to one unscoped notification when a topic's hints exceed the byte cap", async () => { + const tooMany = Array.from({ length: 400 }, (_, index) => `scope-${index}-${"x".repeat(20)}`); + const tooLong = "y".repeat(DATABASE_WORK_NOTIFICATION_MAX_BYTES); + const other = receiving(["overflow", "long-hint"]); + try { + const result = await run(root => root.transaction(async tx => { + for (const hint of tooMany) await signalDatabaseWork(tx, "overflow", hint); + await signalDatabaseWork(tx, "long-hint", "short"); + await signalDatabaseWork(tx, "long-hint", tooLong); + })); + expect(result.payloads.map(payload => JSON.parse(payload))).toEqual([ + { version: 1, origin: expect.any(String), topic: "overflow" }, + { version: 1, origin: expect.any(String), topic: "long-hint" }, + ]); + other.deliver(result.payloads); + expect(other.wakes).toEqual([["overflow", null], ["long-hint", null]]); + } finally { await other.close(); } + }); + + it("treats a malformed or unknown-version hint list as unscoped and ignores unusable or own-origin payloads", async () => { + const other = receiving(["remote"]); + const warn = vi.spyOn(process, "emitWarning"); + try { + const own = await run(root => root.transaction(async tx => { await signalDatabaseWork(tx, "remote", "mine"); })); + const ownEvents: string[] = []; + const stopOwn = subscribeDatabaseWork(db, "remote", event => ownEvents.push(event)); + try { receiveDatabaseWorkNotification(db, own.payloads[0]!); } finally { stopOwn(); } + expect(ownEvents).toEqual([]); + const origin = "another-process"; + const payload = (value: Record) => JSON.stringify({ version: 1, origin, topic: "remote", ...value }); + other.deliver([ + payload({ hints: "company-1" }), + payload({ hints: ["company-1", 7] }), + payload({ hints: [] }), + payload({ hints: null }), + payload({ version: 2, hints: ["company-1"] }), + payload({}), + ]); + expect(other.wakes).toEqual(Array.from({ length: 6 }, () => ["remote", null])); + other.wakes.length = 0; + expect(() => other.deliver([ + "not json", "null", "[]", "7", JSON.stringify({ origin, topic: "remote", hints: ["a"] }), + payload({ version: "1" }), payload({ version: 1.5 }), payload({ version: 0 }), payload({ origin: 4 }), + payload({ topic: "bad topic" }), payload({ hints: ["a".repeat(9000)] }), + ])).not.toThrow(); + expect(other.wakes).toEqual([]); + other.deliver([payload({ hints: ["company-9"] })]); + expect(other.wakes).toEqual([["remote", ["company-9"]]]); + expect(warn).not.toHaveBeenCalled(); + } finally { warn.mockRestore(); await other.close(); } + }); + + it("logs the received hints in the remote notification debug event", async () => { + const enabled = vi.spyOn(workSignalDebug, "enabled", "get").mockReturnValue(true); + const event = vi.spyOn(workSignalDebug, "event").mockImplementation(() => {}); + const other = receiving([]); + try { + other.deliver([ + JSON.stringify({ version: 1, origin: "another-process", topic: "debugged", hints: ["company-1", "company-2"] }), + JSON.stringify({ version: 1, origin: "another-process", topic: "debugged" }), + ]); + expect(event.mock.calls.filter(([, name]) => name === "remote_notification")).toEqual([ + ["debugged", "remote_notification", { hints: JSON.stringify(["company-1", "company-2"]) }], + ["debugged", "remote_notification", { hints: null }], + ]); + } finally { enabled.mockRestore(); event.mockRestore(); await other.close(); } + }); }); diff --git a/packages/db/src/work-signals.ts b/packages/db/src/work-signals.ts index 42be2669622..69b3d2159a0 100644 --- a/packages/db/src/work-signals.ts +++ b/packages/db/src/work-signals.ts @@ -8,7 +8,8 @@ import { workSignalDebug } from "./work-signal-debug.js"; type Listener = (event: "intent" | "settled", hints?: ReadonlySet) => void; type Transaction = { topics: Set; hints: Map>; xid?: Promise; uncertain: boolean }; type Scope = { key: object; listeners: Map>; transactions: Set; root: Connection; notificationOrigin: string; notify: boolean }; -type Context = { scope: Scope; transaction?: Transaction; pending?: Map; settled?: Array<[string, string | undefined]>; ended?: boolean }; +type PendingNotification = { hints: Set; unscoped: boolean }; +type Context = { scope: Scope; transaction?: Transaction; pending?: Map; settled?: Array<[string, string | undefined]>; ended?: boolean }; type Connection = { transaction: (...args: any[]) => Promise; execute: (...args: any[]) => Promise }; const contexts = new WeakMap(); @@ -42,12 +43,12 @@ function install(connection: Connection, context: Context) { try { const value = await callback(tx); current.ended = true; - if (!context.transaction) await flushNotifications(tx, current.pending!); + if (!context.transaction) await flushNotifications(tx, context.scope, current.pending!); return value; } finally { current.ended = true; } }, ...options); if (context.pending && level?.pending) { - for (const [key, payload] of level.pending) if (!context.pending.has(key)) context.pending.set(key, payload); + for (const [topic, notification] of level.pending) mergeNotification(context.pending, topic, notification); } if (context.transaction) context.settled?.push(...level?.settled ?? []); else for (const [topic, hint] of level?.settled ?? []) publish(context.scope, topic, "settled", hint === undefined ? undefined : new Set([hint])); @@ -89,7 +90,7 @@ export async function signalDatabaseWork(connection: object, topic: string, hint // Keep a failed acquisition distinguishable from an uncertain queue write. try { await state.xid; } catch (error) { state.xid = undefined; throw error; } - if (context.scope.notify) notifyAtCommit(context, notification(context.scope, topic)); + if (context.scope.notify) notifyAtCommit(context, topic, hint ?? null); } export function primeDatabaseWorkTransactionId(connection: object, xid: string): void { @@ -97,16 +98,52 @@ export function primeDatabaseWorkTransactionId(connection: object, xid: string): if (state && !state.xid) state.xid = Promise.resolve(xid); } -function notifyAtCommit(context: Context, payload: string): void { +function notifyAtCommit(context: Context, topic: string, hint: string | null | undefined): void { + if (!validTopic(topic)) throw new Error("Invalid database work topic"); if (context.ended) throw new Error("Database work was signalled after its transaction or savepoint ended"); - const key = `${DATABASE_WORK_CHANNEL}\u0000${payload}`; - if (!context.pending!.has(key)) context.pending!.set(key, payload); + let pending = context.pending!.get(topic); + if (!pending) context.pending!.set(topic, pending = { hints: new Set(), unscoped: false }); + if (hint === undefined) pending.unscoped = true; + else if (hint !== null) pending.hints.add(hint); } -async function flushNotifications(connection: Connection, pending: Map): Promise { +function mergeNotification(pending: Map, topic: string, notification: PendingNotification) { + const existing = pending.get(topic); + if (!existing) { pending.set(topic, { hints: new Set(notification.hints), unscoped: notification.unscoped }); return; } + existing.unscoped ||= notification.unscoped; + for (const hint of notification.hints) existing.hints.add(hint); +} + +async function flushNotifications(connection: Connection, scope: Scope, pending: Map): Promise { if (pending.size === 0) return; + const payloads = [...pending].flatMap(([topic, { hints, unscoped }]) => + unscoped || hints.size === 0 ? [notification(scope, topic)] : hintedNotifications(scope, topic, hints)); await connection.execute(sql`select pg_notify(${DATABASE_WORK_CHANNEL}, payload) - from jsonb_array_elements_text(${JSON.stringify([...pending.values()])}::jsonb) as notification(payload)`); + from jsonb_array_elements_text(${JSON.stringify(payloads)}::jsonb) as notification(payload)`); +} + +export const DATABASE_WORK_NOTIFICATION_MAX_BYTES = 1024; +export const DATABASE_WORK_TOPIC_NOTIFICATIONS_MAX_BYTES = 7000; +const MAX_RECEIVED_NOTIFICATION_LENGTH = 8000; + +function hintedNotifications(scope: Scope, topic: string, hints: ReadonlySet): string[] { + const base = Buffer.byteLength(JSON.stringify({ version: 1, origin: scope.notificationOrigin, topic, hints: [] })); + const chunks: string[][] = []; + let chunk: string[] = [], chunkBytes = base, totalBytes = 0; + for (const hint of hints) { + const bytes = Buffer.byteLength(JSON.stringify(hint)); + if (chunk.length > 0 && chunkBytes + 1 + bytes > DATABASE_WORK_NOTIFICATION_MAX_BYTES) { + chunks.push(chunk); totalBytes += chunkBytes; + chunk = []; chunkBytes = base; + } + chunkBytes += chunk.length > 0 ? bytes + 1 : bytes; + chunk.push(hint); + if (chunkBytes > DATABASE_WORK_NOTIFICATION_MAX_BYTES || totalBytes + chunkBytes > DATABASE_WORK_TOPIC_NOTIFICATIONS_MAX_BYTES) { + return [notification(scope, topic)]; + } + } + chunks.push(chunk); + return chunks.map(list => JSON.stringify({ version: 1, origin: scope.notificationOrigin, topic, hints: list })); } export const DATABASE_WORK_CHANNEL = "paperclip_work_v1"; @@ -129,7 +166,7 @@ export async function notifyDatabaseWorkOnCommit(connection: object, topic: stri const context = contexts.get(connection); if (!context?.scope.notify) return; if (!context.transaction) throw new Error("Database work must be registered inside a transaction"); - notifyAtCommit(context, notification(context.scope, topic)); + notifyAtCommit(context, topic, undefined); } export function databaseWorkCommitNotification(owner: object, topic: string): string | null { @@ -147,7 +184,7 @@ export function publishDatabaseWorkSettled(owner: object, topic: string, hint?: } if (context.ended) throw new Error("Database work was signalled after its transaction or savepoint ended"); context.settled!.push([topic, hint]); - if (context.scope.notify) notifyAtCommit(context, notification(context.scope, topic)); + if (context.scope.notify) notifyAtCommit(context, topic, hint); } /** A post-commit invalidation hint, for already-persisted live UI state. */ @@ -157,13 +194,21 @@ export async function publishDatabaseWorkHint(owner: object, topic: string): Pro } export function receiveDatabaseWorkNotification(owner: object, payload: string): void { const scope = contexts.get(owner)?.scope; - if (!scope || payload.length > 1024) return; - let value: { version?: unknown; origin?: unknown; topic?: unknown }; + if (!scope || typeof payload !== "string" || payload.length > MAX_RECEIVED_NOTIFICATION_LENGTH) return; + let value: { version?: unknown; origin?: unknown; topic?: unknown; hints?: unknown }; try { value = JSON.parse(payload); } catch { return; } - if (!value || value.version !== 1 || typeof value.origin !== "string" || value.origin === scope.notificationOrigin + if (!value || typeof value !== "object" || typeof value.version !== "number" || !Number.isInteger(value.version) + || value.version < 1 || typeof value.origin !== "string" || value.origin === scope.notificationOrigin || typeof value.topic !== "string" || !validTopic(value.topic)) return; - if (workSignalDebug.enabled) workSignalDebug.event(value.topic, "remote_notification"); - publish(scope, value.topic); + const hints = value.version === 1 ? receivedHints(value.hints) : undefined; + if (workSignalDebug.enabled) { + workSignalDebug.event(value.topic, "remote_notification", { hints: hints ? JSON.stringify([...hints]) : null }); + } + publish(scope, value.topic, "settled", hints); +} +function receivedHints(value: unknown): ReadonlySet | undefined { + if (!Array.isArray(value) || value.length === 0 || !value.every(hint => typeof hint === "string")) return undefined; + return new Set(value); } /** Reconnect can lose hints. Reconstruct existing durable queues once; no poll. */ export function reconcileDatabaseWorkNotifications(owner: object): void { diff --git a/server/src/__tests__/scoped-deadline-work-notifications.test.ts b/server/src/__tests__/scoped-deadline-work-notifications.test.ts new file mode 100644 index 00000000000..c2b4bf7a64f --- /dev/null +++ b/server/src/__tests__/scoped-deadline-work-notifications.test.ts @@ -0,0 +1,111 @@ +import { setTimeout as delay } from "node:timers/promises"; +import { sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { createDb } from "@paperclipai/db"; +import { DATABASE_WORK_CHANNEL, enableDatabaseWorkNotifications, receiveDatabaseWorkNotification, + signalDatabaseWork } from "@paperclipai/db/work-signals"; +import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js"; +import { createDeliveryWorkCoordinator } from "../services/delivery-work-coordinator.js"; +import { DELIVERY_QUEUES } from "../services/delivery-work-notifications.js"; +import { registerScopedDeadlineWork } from "../services/scoped-deadline-work.js"; + +type Entry = { id: string; companyId: string; dueAt: number }; +const COMPANIES = 20; +const ROWS_PER_COMPANY = 40; +const queue = DELIVERY_QUEUES.secretProposalExpiry; + +const support = await getEmbeddedPostgresTestSupport(); +(support.supported ? describe : describe.skip)("scoped deadline work across server processes", () => { + let database: Awaited>; + let sender: ReturnType, receiver: ReturnType, listener: ReturnType; + let coordinator: ReturnType; + const loads: string[] = []; + let rowsLoaded = 0; + const onError = vi.fn(); + + beforeAll(async () => { + database = await startEmbeddedPostgresTestDatabase("scoped-deadline-notifications-"); + sender = createDb(database.connectionString); + receiver = createDb(database.connectionString); + listener = createDb(database.connectionString, { maxConnections: 1 }); + enableDatabaseWorkNotifications(sender); + enableDatabaseWorkNotifications(receiver); + await sender.execute(sql`create table scoped_deadline_fixture (id text primary key, company_id text not null)`); + await sender.execute(sql`insert into scoped_deadline_fixture + select 'company-' || c || '-' || r, 'company-' || c + from generate_series(1, ${COMPANIES}::int) as c, generate_series(1, ${ROWS_PER_COMPANY}::int) as r`); + await listener.$client.listen(DATABASE_WORK_CHANNEL, payload => receiveDatabaseWorkNotification(receiver, payload)); + coordinator = createDeliveryWorkCoordinator({ owner: receiver, canRun: () => true, onError }); + const worker = registerScopedDeadlineWork(coordinator, { + queue, + minimumPassIntervalMs: 0, + decodeScope: hint => hint === "*" ? undefined : hint, + matches: (entry, companyId) => companyId === undefined || entry.companyId === companyId, + scopeKey: { entry: entry => entry.companyId, scope: companyId => companyId }, + load: async companyId => { + const rows = companyId === undefined + ? await receiver.execute<{ id: string; company_id: string }>(sql`select id, company_id from scoped_deadline_fixture`) + : await receiver.execute<{ id: string; company_id: string }>(sql`select id, company_id from scoped_deadline_fixture + where company_id = ${companyId}`); + rowsLoaded += rows.length; + loads.push(companyId ?? "*"); + return rows.map(row => ({ id: row.id, companyId: row.company_id, dueAt: Date.now() + 3_600_000 })); + }, + check: async entry => entry, + }, () => true); + await worker.ready; + await eventually(() => loads.includes("*")); + }, 90_000); + + afterAll(async () => { + await coordinator?.stop(); + await listener?.$client.end({ timeout: 1 }); + await sender?.$client.end({ timeout: 1 }); + await receiver?.$client.end({ timeout: 1 }); + await database?.cleanup(); + }); + + async function eventually(check: () => boolean) { + const deadline = Date.now() + 10_000; + while (!check()) { + if (Date.now() > deadline) throw new Error("Scoped deadline work did not settle"); + await delay(10); + } + } + + async function commitInOtherProcess(write: (tx: Parameters[0]>[0]) => Promise) { + loads.length = 0; + rowsLoaded = 0; + await sender.transaction(write); + await eventually(() => loads.length > 0); + await delay(200); + return { loads: [...loads], rowsLoaded }; + } + + it("reloads only the hinted company when another process commits scoped work", async () => { + const result = await commitInOtherProcess(async tx => { + await signalDatabaseWork(tx, queue, "company-7"); + await tx.execute(sql`insert into scoped_deadline_fixture values ('company-7-new', 'company-7')`); + }); + expect(result).toEqual({ loads: ["company-7"], rowsLoaded: ROWS_PER_COMPANY + 1 }); + expect(onError).not.toHaveBeenCalled(); + }); + + it("reloads only the hinted companies when one commit touches several", async () => { + const result = await commitInOtherProcess(async tx => { + await signalDatabaseWork(tx, queue, "company-3"); + await signalDatabaseWork(tx, queue, "company-4"); + await signalDatabaseWork(tx, queue, "company-3"); + }); + expect(result.loads.sort()).toEqual(["company-3", "company-4"]); + expect(result.rowsLoaded).toBe(2 * ROWS_PER_COMPANY); + expect(onError).not.toHaveBeenCalled(); + }); + + it("still reloads every candidate when another process commits unscoped work", async () => { + const result = await commitInOtherProcess(async tx => { await signalDatabaseWork(tx, queue); }); + expect(result.loads).toEqual(["*"]); + expect(result.rowsLoaded).toBe(COMPANIES * ROWS_PER_COMPANY + 1); + expect(onError).not.toHaveBeenCalled(); + }); +});