test: measure accounting writes with many budget policies

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed 2026-10-04 10:19:07 -07:00
1 parent 2c77785245
commit 463cb73433
2 files changed
+18 -3

No files matched your search

+1 -1
View File
@@ -110,7 +110,7 @@ A correction requires `idempotencyKey`, exact `expectedCents`, `correctedCents`,
`pnpm benchmark:accounting` creates its own disposable PostgreSQL database **and receipt-spool home**. It never uses the caller's database or pending receipts. `PAPERCLIP_ACCOUNTING_BENCH_ROWS` selects 1,000–10,000,000 events (one million by default), spread across twelve UTC months. Results go to `coverage/accounting/scale.json`; copy that file before running coverage, which cleans the same output directory. There are no machine-dependent latency assertions in CI.
The benchmark measures all-time reports, eight concurrent writers on one company, two active budgets, admission, eight concurrent ledger/budget/health report bundles competing with writers, durable checkpoints, 100,000 historical run rows, 102 policies, 1,000/10,000-file receipt backlogs, and recovery of 250 completed runs. An independent integrity check must find no discrepancies. Report bundles measure service/database work, not browser rendering, HTTP latency, provider quota calls, or financial-event queries. The fixture concentrates spend in one agent; it is not a production traffic model.
The benchmark measures all-time reports, eight concurrent writers on one company, two active budgets, admission, eight concurrent ledger/budget/health report bundles competing with writers, durable checkpoints, 100,000 historical run rows, overview/writes/admission with 102 policies (100 unrelated to the writing agent), 1,000/10,000-file receipt backlogs, and recovery of 250 completed runs. An independent integrity check must find no discrepancies. Report bundles measure service/database work, not browser rendering, HTTP latency, provider quota calls, or financial-event queries. The fixture concentrates spend in one agent; it is not a production traffic model.
The October 4 local comparison on macOS arm64, 10 CPUs, Node 25.6.1 measured the following at one million ledger events. PostgreSQL, API code, and storage shared one developer machine. These are observations, not service-level guarantees.
+17 -2
View File
@@ -128,7 +128,20 @@ try {
const extraAgents = await db.insert(agents).values(Array.from({ length: 100 }, (_, i) => ({ companyId: company.id, name: `Idle ${i}`, role: "engineer", adapterType: "process" }))).returning();
await db.insert(budgetPolicies).values(extraAgents.map(extra => ({ companyId: company.id, scopeType: "agent", scopeId: extra.id, windowKind: "calendar_month_utc", amount: 2_000_000_000 })));
const largeOverview: number[] = [];
for (let i = 0; i < 10; i++) largeOverview.push(await time(() => budgetService(db).overview(company.id)));
for (let i = 0; i < 10; i++) largeOverview.push(await time(async () => {
const overview = await budgetService(db).overview(company.id);
if (overview.policies.length !== 102) throw new Error("Benchmark policy fixture is incomplete");
}));
const largePolicyWrites: number[] = [], largePolicyAdmissions: number[] = [];
const largePolicyStart = performance.now();
for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `large-policy:${batch}:${index}`, occurredAt: new Date(),
})).then(ms => largePolicyWrites.push(ms))));
const largePolicyMs = performance.now() - largePolicyStart;
for (let i = 0; i < 20; i++) {
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
largePolicyAdmissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null)));
}
const integrity = await accountingIntegrityService(db).inspect(company.id);
if (integrity.findings.length) throw new Error(`Benchmark violated accounting integrity: ${JSON.stringify(integrity.findings)}`);
const plan = await db.$client`explain (analyze,buffers,format json) select sum(cost_cents) from cost_events where company_id = ${company.id} and project_id = ${project.id}
@@ -139,7 +152,9 @@ try {
budgetedCompanyWrites: { activePolicies: 2, ...distribution(budgetedWrites), writesPerSecond: budgetedWrites.length / (budgetedMs / 1000) },
admission: distribution(admissions), concurrentDashboard: distribution(dashboard), writesWhilePolling: distribution(writesWhilePolling),
durableCheckpoint: distribution(checkpoints), healthWith100kHistoricalRuns: distribution(health),
overviewWith102Policies: distribution(largeOverview), spoolBacklogs,
overviewWith102Policies: distribution(largeOverview),
writesWith102Policies: { activePolicies: 102, relevantPolicies: 2, ...distribution(largePolicyWrites), writesPerSecond: largePolicyWrites.length / (largePolicyMs / 1000) },
admissionWith102Policies: distribution(largePolicyAdmissions), spoolBacklogs,
eventLoopDelay: { p95Ms: eventLoop.percentile(95) / 1e6, maxMs: eventLoop.max / 1e6 },
recovery, projectBudgetQueryPlan: plan, integrityFindings: integrity.findings.length, totalMs: performance.now()-started };
const output = path.resolve("coverage/accounting/scale.json"); await mkdir(path.dirname(output), { recursive: true }); await writeFile(output, JSON.stringify(report,null,2));