From 463cb734334deb5c7fcd3ee8798347f92ab72807 Mon Sep 17 00:00:00 2001 From: Devin Foley Date: Sun, 4 Oct 2026 10:19:07 -0700 Subject: [PATCH] test: measure accounting writes with many budget policies Co-Authored-By: Paperclip --- doc/COST-ACCOUNTING.md | 2 +- server/scripts/benchmark-accounting.ts | 19 +++++++++++++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/doc/COST-ACCOUNTING.md b/doc/COST-ACCOUNTING.md index cc6891325e4..532f811d8e5 100644 --- a/doc/COST-ACCOUNTING.md +++ b/doc/COST-ACCOUNTING.md @@ -110,7 +110,7 @@ A correction requires `idempotencyKey`, exact `expectedCents`, `correctedCents`, `pnpm benchmark:accounting` creates its own disposable PostgreSQL database **and receipt-spool home**. It never uses the caller's database or pending receipts. `PAPERCLIP_ACCOUNTING_BENCH_ROWS` selects 1,000–10,000,000 events (one million by default), spread across twelve UTC months. Results go to `coverage/accounting/scale.json`; copy that file before running coverage, which cleans the same output directory. There are no machine-dependent latency assertions in CI. -The benchmark measures all-time reports, eight concurrent writers on one company, two active budgets, admission, eight concurrent ledger/budget/health report bundles competing with writers, durable checkpoints, 100,000 historical run rows, 102 policies, 1,000/10,000-file receipt backlogs, and recovery of 250 completed runs. An independent integrity check must find no discrepancies. Report bundles measure service/database work, not browser rendering, HTTP latency, provider quota calls, or financial-event queries. The fixture concentrates spend in one agent; it is not a production traffic model. +The benchmark measures all-time reports, eight concurrent writers on one company, two active budgets, admission, eight concurrent ledger/budget/health report bundles competing with writers, durable checkpoints, 100,000 historical run rows, overview/writes/admission with 102 policies (100 unrelated to the writing agent), 1,000/10,000-file receipt backlogs, and recovery of 250 completed runs. An independent integrity check must find no discrepancies. Report bundles measure service/database work, not browser rendering, HTTP latency, provider quota calls, or financial-event queries. The fixture concentrates spend in one agent; it is not a production traffic model. The October 4 local comparison on macOS arm64, 10 CPUs, Node 25.6.1 measured the following at one million ledger events. PostgreSQL, API code, and storage shared one developer machine. These are observations, not service-level guarantees. diff --git a/server/scripts/benchmark-accounting.ts b/server/scripts/benchmark-accounting.ts index dc3c9e5d1f3..05c26ffa3da 100644 --- a/server/scripts/benchmark-accounting.ts +++ b/server/scripts/benchmark-accounting.ts @@ -128,7 +128,20 @@ try { const extraAgents = await db.insert(agents).values(Array.from({ length: 100 }, (_, i) => ({ companyId: company.id, name: `Idle ${i}`, role: "engineer", adapterType: "process" }))).returning(); await db.insert(budgetPolicies).values(extraAgents.map(extra => ({ companyId: company.id, scopeType: "agent", scopeId: extra.id, windowKind: "calendar_month_utc", amount: 2_000_000_000 }))); const largeOverview: number[] = []; - for (let i = 0; i < 10; i++) largeOverview.push(await time(() => budgetService(db).overview(company.id))); + for (let i = 0; i < 10; i++) largeOverview.push(await time(async () => { + const overview = await budgetService(db).overview(company.id); + if (overview.policies.length !== 102) throw new Error("Benchmark policy fixture is incomplete"); + })); + const largePolicyWrites: number[] = [], largePolicyAdmissions: number[] = []; + const largePolicyStart = performance.now(); + for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, { + agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `large-policy:${batch}:${index}`, occurredAt: new Date(), + })).then(ms => largePolicyWrites.push(ms)))); + const largePolicyMs = performance.now() - largePolicyStart; + for (let i = 0; i < 20; i++) { + const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning(); + largePolicyAdmissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null))); + } const integrity = await accountingIntegrityService(db).inspect(company.id); if (integrity.findings.length) throw new Error(`Benchmark violated accounting integrity: ${JSON.stringify(integrity.findings)}`); const plan = await db.$client`explain (analyze,buffers,format json) select sum(cost_cents) from cost_events where company_id = ${company.id} and project_id = ${project.id} @@ -139,7 +152,9 @@ try { budgetedCompanyWrites: { activePolicies: 2, ...distribution(budgetedWrites), writesPerSecond: budgetedWrites.length / (budgetedMs / 1000) }, admission: distribution(admissions), concurrentDashboard: distribution(dashboard), writesWhilePolling: distribution(writesWhilePolling), durableCheckpoint: distribution(checkpoints), healthWith100kHistoricalRuns: distribution(health), - overviewWith102Policies: distribution(largeOverview), spoolBacklogs, + overviewWith102Policies: distribution(largeOverview), + writesWith102Policies: { activePolicies: 102, relevantPolicies: 2, ...distribution(largePolicyWrites), writesPerSecond: largePolicyWrites.length / (largePolicyMs / 1000) }, + admissionWith102Policies: distribution(largePolicyAdmissions), spoolBacklogs, eventLoopDelay: { p95Ms: eventLoop.percentile(95) / 1e6, maxMs: eventLoop.max / 1e6 }, recovery, projectBudgetQueryPlan: plan, integrityFindings: integrity.findings.length, totalMs: performance.now()-started }; const output = path.resolve("coverage/accounting/scale.json"); await mkdir(path.dirname(output), { recursive: true }); await writeFile(output, JSON.stringify(report,null,2));