From 2c77785245b27c4ee970804b36b1aa7fc1b2e1c0 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 4 Oct 2026 10:09:06 -0700 Subject: [PATCH] perf: keep accounting operator reads off the write lock Co-Authored-By: Paperclip --- doc/COST-ACCOUNTING.md | 2 +- .../cost-accounting-performance.test.ts | 21 +++++++++++++++++++ server/src/services/accounting-integrity.ts | 12 ++++------- server/src/services/accounting-transaction.ts | 11 ++++++++++ server/src/services/billing-reconciliation.ts | 4 ++-- 5 files changed, 39 insertions(+), 11 deletions(-) diff --git a/doc/COST-ACCOUNTING.md b/doc/COST-ACCOUNTING.md index 815cb07f830..cc6891325e4 100644 --- a/doc/COST-ACCOUNTING.md +++ b/doc/COST-ACCOUNTING.md @@ -130,7 +130,7 @@ Performance protections: - Cost writes aggregate the affected company, agent, and project policies in one ledger scan and one pending-run scan. Each policy retains its own scope and UTC/lifetime window. Exact ledger values remain authoritative, including unknown pricing and native recovery guards. - A budget operation reuses its observation for thresholds, incident amounts, and reasons. The observation also carries its UTC window across a midnight boundary. It does not cache observations across ledger mutations. PostgreSQL filters policies to the affected scopes before returning rows. - Overview policy and incident reads run in batches of four, so a large policy list does not enqueue one query per policy at once. -- Health uses a read-only, repeatable-read snapshot. Its counts and details agree without taking the company accounting write lock. Inspection and repair retain their existing locking requirements. +- Health, integrity inspection, and invoice comparison use read-only, repeatable-read snapshots. Their counts and details agree without taking the company accounting write lock. Repairs retain the write lock and revalidate the inspection fingerprint before making changes. - One heartbeat service shares a running receipt replay/recovery/policy sweep across overlapping scheduler ticks. Success or failure releases the guard; a later tick can retry. This is a per-process scheduler protection, not a distributed lease. Ledger locks and receipt idempotency still protect multiple processes. The regression suite checks SQL scan counts, mixed-scope/window totals, pending/unpriced/native recovery guards, snapshot consistency while a writer commits, and coalesced recovery through both success and failure. It also retains the concurrency, crash recovery, exact-money, and mutation tests. Wall-clock benchmarks stay separate from CI correctness gates. diff --git a/server/src/__tests__/cost-accounting-performance.test.ts b/server/src/__tests__/cost-accounting-performance.test.ts index 68821f54066..fb30c740f3d 100644 --- a/server/src/__tests__/cost-accounting-performance.test.ts +++ b/server/src/__tests__/cost-accounting-performance.test.ts @@ -4,6 +4,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest import { eq, sql } from "drizzle-orm"; import { drizzle } from "drizzle-orm/postgres-js"; import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, issues, nativeRunFinalizations, projects, type Db } from "@paperclipai/db"; +import { billingReconciliationService } from "../services/billing-reconciliation.js"; import { accountingIntegrityService } from "../services/accounting-integrity.js"; import { budgetService, budgetServiceInTransaction } from "../services/budgets.js"; import { withAccountingTransaction } from "../services/accounting-transaction.js"; @@ -202,4 +203,24 @@ describe("accounting performance invariants (PostgreSQL)", () => { expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull(); }); + + it.each(["inspection", "invoice comparison"])("does not take the company write lock during %s", async kind => { + const f = await fixture(); + const invoice = await billingReconciliationService(db).importInvoice(f.company.id, { biller: "fixture", externalId: "invoice", currency: "USD", + lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString() }] }, "board"); + const writer = await db.$client.reserve(); + await writer`begin`; + try { + await writer`select id from companies where id = ${f.company.id} for no key update`; + const original = db.transaction.bind(db); + const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => { + await tx.execute(sql`set local lock_timeout = '200ms'`); + return callback(tx); + }, config)); + if (kind === "inspection") expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]); + else expect((await billingReconciliationService(db).reconcile(f.company.id, invoice.id)).lines).toHaveLength(1); + expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" }); + } finally { await writer`rollback`; writer.release(); } + }); + }); diff --git a/server/src/services/accounting-integrity.ts b/server/src/services/accounting-integrity.ts index 206eb73d190..ad13779619a 100644 --- a/server/src/services/accounting-integrity.ts +++ b/server/src/services/accounting-integrity.ts @@ -3,7 +3,7 @@ import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@p import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared"; import { conflict, notFound } from "../errors.js"; import { receiptFingerprint } from "./receipt-fingerprint.js"; -import { withAccountingTransaction } from "./accounting-transaction.js"; +import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js"; import { logActivity } from "./activity-log.js"; import { accountRunCost } from "./run-cost-accounting.js"; import type { BudgetServiceHooks } from "./budgets.js"; @@ -70,7 +70,7 @@ async function inspect(tx: Db, companyId: string): Promise export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {}) { return { - inspect: (companyId: string) => withAccountingTransaction(db, companyId, tx => inspect(tx, companyId)), + inspect: (companyId: string) => withAccountingReadSnapshot(db, companyId, tx => inspect(tx, companyId)), repair: (companyId: string, fingerprint: string, reason: string, actorId: string) => withAccountingTransaction(db, companyId, async (tx, publications) => { if (!reason.trim()) throw conflict("A repair reason is required"); const before = await inspect(tx, companyId); @@ -98,11 +98,7 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = { throw error; } }, - // A consistent read snapshot must not queue behind (or delay) cost writes. - // Mutation paths still use the company accounting lock. - health: (companyId: string): Promise => db.transaction(async tx => { - const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId)); - if (!company) throw notFound("Company not found"); + health: (companyId: string): Promise => withAccountingReadSnapshot(db, companyId, async tx => { const [counts] = await tx.execute<{ pending: number; unpriced: number; oldest: string | null; cancellations: number; reserved: string }>(sql` select (select count(*)::int from heartbeat_runs where company_id = ${companyId} and cost_accounting_pending and status in ('succeeded','failed','timed_out','cancelled','interrupted')) as pending, (select count(*)::int from cost_events where company_id = ${companyId} and cost_status = 'unpriced' and billing_type <> 'subscription_included') as unpriced, @@ -120,6 +116,6 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = { where company_id = ${companyId} and cost_status = 'unpriced' and billing_type <> 'subscription_included' order by occurred_at,id limit 100`); return { companyId, pendingRunCount: counts.pending, unpricedEventCount: counts.unpriced, oldestPendingAt: counts.oldest, pendingCancellationCount: counts.cancellations, heldReservationCents: normalizeCents(counts.reserved), items: [...pending, ...unpriced.map(row => ({ ...row, state: "unpriced" as const, lastError: null, attempts: 0, lastAttemptAt: null }))] }; - }, { isolationLevel: "repeatable read", accessMode: "read only" }), + }), }; } diff --git a/server/src/services/accounting-transaction.ts b/server/src/services/accounting-transaction.ts index 3d4fec7b826..d7e206e78ca 100644 --- a/server/src/services/accounting-transaction.ts +++ b/server/src/services/accounting-transaction.ts @@ -20,6 +20,17 @@ export async function withAccountingTransaction(db: Db, companyId: string, wo return result; } +/** Consistent operator reports must not wait for or delay accounting writers. + * Reads see one committed snapshot; repairs still revalidate under the lock. */ +export function withAccountingReadSnapshot(db: Db, companyId: string, work: (tx: Db) => Promise): Promise { + return db.transaction(async transaction => { + const tx = transaction as unknown as Db; + const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId)); + if (!company) throw notFound("Company not found"); + return work(tx); + }, { isolationLevel: "repeatable read", accessMode: "read only" }); +} + /** Database history is durable even when the live subscriber channel fails. */ export function publishAccountingActivities(companyId: string, publications: ActivityPublication[]) { for (const publication of publications) { diff --git a/server/src/services/billing-reconciliation.ts b/server/src/services/billing-reconciliation.ts index 99e2af8689c..b1ca976977a 100644 --- a/server/src/services/billing-reconciliation.ts +++ b/server/src/services/billing-reconciliation.ts @@ -3,7 +3,7 @@ import { and, desc, eq, sql } from "drizzle-orm"; import { agentRuntimeState, costEvents, costAdjustments, billingInvoices, billingInvoiceLines, heartbeatRuns, type Db } from "@paperclipai/db"; import { adjustCostSchema, importBillingInvoiceSchema, normalizeCents, subtractCents, type AdjustCost, type ImportBillingInvoice } from "@paperclipai/shared"; import { conflict, notFound, unprocessable } from "../errors.js"; -import { withAccountingTransaction } from "./accounting-transaction.js"; +import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js"; import { receiptFingerprint } from "./receipt-fingerprint.js"; import { logActivity } from "./activity-log.js"; import { updateMonthlySpendProjections } from "./costs.js"; @@ -52,7 +52,7 @@ export function billingReconciliationService(db: Db, hooks: BudgetServiceHooks = return invoice; }), list: (companyId: string) => db.select().from(billingInvoices).where(eq(billingInvoices.companyId, companyId)).orderBy(desc(billingInvoices.createdAt)).limit(100), - reconcile: (companyId: string, invoiceId: string) => withAccountingTransaction(db, companyId, async tx => { + reconcile: (companyId: string, invoiceId: string) => withAccountingReadSnapshot(db, companyId, async tx => { const [invoice] = await tx.select().from(billingInvoices).where(and(eq(billingInvoices.companyId, companyId), eq(billingInvoices.id, invoiceId))); if (!invoice) throw notFound("Invoice not found"); const lines = await tx.select().from(billingInvoiceLines).where(and(eq(billingInvoiceLines.companyId, companyId), eq(billingInvoiceLines.invoiceId, invoiceId))).orderBy(billingInvoiceLines.externalId);