mirror of
https://github.com/paperclipai/paperclip.git
synced 2026-11-16 03:31:44 +01:00
perf: keep accounting operator reads off the write lock
Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
1 parent
24a4cff0d5
commit
2c77785245
5 files changed
+39
-11
No files matched your search
@@ -130,7 +130,7 @@ Performance protections:
|
||||
- Cost writes aggregate the affected company, agent, and project policies in one ledger scan and one pending-run scan. Each policy retains its own scope and UTC/lifetime window. Exact ledger values remain authoritative, including unknown pricing and native recovery guards.
|
||||
- A budget operation reuses its observation for thresholds, incident amounts, and reasons. The observation also carries its UTC window across a midnight boundary. It does not cache observations across ledger mutations. PostgreSQL filters policies to the affected scopes before returning rows.
|
||||
- Overview policy and incident reads run in batches of four, so a large policy list does not enqueue one query per policy at once.
|
||||
- Health uses a read-only, repeatable-read snapshot. Its counts and details agree without taking the company accounting write lock. Inspection and repair retain their existing locking requirements.
|
||||
- Health, integrity inspection, and invoice comparison use read-only, repeatable-read snapshots. Their counts and details agree without taking the company accounting write lock. Repairs retain the write lock and revalidate the inspection fingerprint before making changes.
|
||||
- One heartbeat service shares a running receipt replay/recovery/policy sweep across overlapping scheduler ticks. Success or failure releases the guard; a later tick can retry. This is a per-process scheduler protection, not a distributed lease. Ledger locks and receipt idempotency still protect multiple processes.
|
||||
|
||||
The regression suite checks SQL scan counts, mixed-scope/window totals, pending/unpriced/native recovery guards, snapshot consistency while a writer commits, and coalesced recovery through both success and failure. It also retains the concurrency, crash recovery, exact-money, and mutation tests. Wall-clock benchmarks stay separate from CI correctness gates.
|
||||
|
||||
@@ -4,6 +4,7 @@ import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { drizzle } from "drizzle-orm/postgres-js";
|
||||
import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, issues, nativeRunFinalizations, projects, type Db } from "@paperclipai/db";
|
||||
import { billingReconciliationService } from "../services/billing-reconciliation.js";
|
||||
import { accountingIntegrityService } from "../services/accounting-integrity.js";
|
||||
import { budgetService, budgetServiceInTransaction } from "../services/budgets.js";
|
||||
import { withAccountingTransaction } from "../services/accounting-transaction.js";
|
||||
@@ -202,4 +203,24 @@ describe("accounting performance invariants (PostgreSQL)", () => {
|
||||
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
|
||||
});
|
||||
|
||||
|
||||
it.each(["inspection", "invoice comparison"])("does not take the company write lock during %s", async kind => {
|
||||
const f = await fixture();
|
||||
const invoice = await billingReconciliationService(db).importInvoice(f.company.id, { biller: "fixture", externalId: "invoice", currency: "USD",
|
||||
lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString() }] }, "board");
|
||||
const writer = await db.$client.reserve();
|
||||
await writer`begin`;
|
||||
try {
|
||||
await writer`select id from companies where id = ${f.company.id} for no key update`;
|
||||
const original = db.transaction.bind(db);
|
||||
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
|
||||
await tx.execute(sql`set local lock_timeout = '200ms'`);
|
||||
return callback(tx);
|
||||
}, config));
|
||||
if (kind === "inspection") expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
|
||||
else expect((await billingReconciliationService(db).reconcile(f.company.id, invoice.id)).lines).toHaveLength(1);
|
||||
expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
|
||||
} finally { await writer`rollback`; writer.release(); }
|
||||
});
|
||||
|
||||
});
|
||||
@@ -3,7 +3,7 @@ import { agents, agentRuntimeState, companies, heartbeatRuns, type Db } from "@p
|
||||
import { normalizeCents, usdToCents, type AccountingFinding, type AccountingHealth, type AccountingInspection } from "@paperclipai/shared";
|
||||
import { conflict, notFound } from "../errors.js";
|
||||
import { receiptFingerprint } from "./receipt-fingerprint.js";
|
||||
import { withAccountingTransaction } from "./accounting-transaction.js";
|
||||
import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { accountRunCost } from "./run-cost-accounting.js";
|
||||
import type { BudgetServiceHooks } from "./budgets.js";
|
||||
@@ -70,7 +70,7 @@ async function inspect(tx: Db, companyId: string): Promise<AccountingInspection>
|
||||
|
||||
export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {}) {
|
||||
return {
|
||||
inspect: (companyId: string) => withAccountingTransaction(db, companyId, tx => inspect(tx, companyId)),
|
||||
inspect: (companyId: string) => withAccountingReadSnapshot(db, companyId, tx => inspect(tx, companyId)),
|
||||
repair: (companyId: string, fingerprint: string, reason: string, actorId: string) => withAccountingTransaction(db, companyId, async (tx, publications) => {
|
||||
if (!reason.trim()) throw conflict("A repair reason is required");
|
||||
const before = await inspect(tx, companyId);
|
||||
@@ -98,11 +98,7 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
// A consistent read snapshot must not queue behind (or delay) cost writes.
|
||||
// Mutation paths still use the company accounting lock.
|
||||
health: (companyId: string): Promise<AccountingHealth> => db.transaction(async tx => {
|
||||
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId));
|
||||
if (!company) throw notFound("Company not found");
|
||||
health: (companyId: string): Promise<AccountingHealth> => withAccountingReadSnapshot(db, companyId, async tx => {
|
||||
const [counts] = await tx.execute<{ pending: number; unpriced: number; oldest: string | null; cancellations: number; reserved: string }>(sql`
|
||||
select (select count(*)::int from heartbeat_runs where company_id = ${companyId} and cost_accounting_pending and status in ('succeeded','failed','timed_out','cancelled','interrupted')) as pending,
|
||||
(select count(*)::int from cost_events where company_id = ${companyId} and cost_status = 'unpriced' and billing_type <> 'subscription_included') as unpriced,
|
||||
@@ -120,6 +116,6 @@ export function accountingIntegrityService(db: Db, hooks: BudgetServiceHooks = {
|
||||
where company_id = ${companyId} and cost_status = 'unpriced' and billing_type <> 'subscription_included' order by occurred_at,id limit 100`);
|
||||
return { companyId, pendingRunCount: counts.pending, unpricedEventCount: counts.unpriced, oldestPendingAt: counts.oldest, pendingCancellationCount: counts.cancellations,
|
||||
heldReservationCents: normalizeCents(counts.reserved), items: [...pending, ...unpriced.map(row => ({ ...row, state: "unpriced" as const, lastError: null, attempts: 0, lastAttemptAt: null }))] };
|
||||
}, { isolationLevel: "repeatable read", accessMode: "read only" }),
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -20,6 +20,17 @@ export async function withAccountingTransaction<T>(db: Db, companyId: string, wo
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Consistent operator reports must not wait for or delay accounting writers.
|
||||
* Reads see one committed snapshot; repairs still revalidate under the lock. */
|
||||
export function withAccountingReadSnapshot<T>(db: Db, companyId: string, work: (tx: Db) => Promise<T>): Promise<T> {
|
||||
return db.transaction(async transaction => {
|
||||
const tx = transaction as unknown as Db;
|
||||
const [company] = await tx.select({ id: companies.id }).from(companies).where(eq(companies.id, companyId));
|
||||
if (!company) throw notFound("Company not found");
|
||||
return work(tx);
|
||||
}, { isolationLevel: "repeatable read", accessMode: "read only" });
|
||||
}
|
||||
|
||||
/** Database history is durable even when the live subscriber channel fails. */
|
||||
export function publishAccountingActivities(companyId: string, publications: ActivityPublication[]) {
|
||||
for (const publication of publications) {
|
||||
|
||||
@@ -3,7 +3,7 @@ import { and, desc, eq, sql } from "drizzle-orm";
|
||||
import { agentRuntimeState, costEvents, costAdjustments, billingInvoices, billingInvoiceLines, heartbeatRuns, type Db } from "@paperclipai/db";
|
||||
import { adjustCostSchema, importBillingInvoiceSchema, normalizeCents, subtractCents, type AdjustCost, type ImportBillingInvoice } from "@paperclipai/shared";
|
||||
import { conflict, notFound, unprocessable } from "../errors.js";
|
||||
import { withAccountingTransaction } from "./accounting-transaction.js";
|
||||
import { withAccountingReadSnapshot, withAccountingTransaction } from "./accounting-transaction.js";
|
||||
import { receiptFingerprint } from "./receipt-fingerprint.js";
|
||||
import { logActivity } from "./activity-log.js";
|
||||
import { updateMonthlySpendProjections } from "./costs.js";
|
||||
@@ -52,7 +52,7 @@ export function billingReconciliationService(db: Db, hooks: BudgetServiceHooks =
|
||||
return invoice;
|
||||
}),
|
||||
list: (companyId: string) => db.select().from(billingInvoices).where(eq(billingInvoices.companyId, companyId)).orderBy(desc(billingInvoices.createdAt)).limit(100),
|
||||
reconcile: (companyId: string, invoiceId: string) => withAccountingTransaction(db, companyId, async tx => {
|
||||
reconcile: (companyId: string, invoiceId: string) => withAccountingReadSnapshot(db, companyId, async tx => {
|
||||
const [invoice] = await tx.select().from(billingInvoices).where(and(eq(billingInvoices.companyId, companyId), eq(billingInvoices.id, invoiceId)));
|
||||
if (!invoice) throw notFound("Invoice not found");
|
||||
const lines = await tx.select().from(billingInvoiceLines).where(and(eq(billingInvoiceLines.companyId, companyId), eq(billingInvoiceLines.invoiceId, invoiceId))).orderBy(billingInvoiceLines.externalId);
|
||||
|
||||
Reference in new issue
Block a user