PaperClipAI/server/scripts/benchmark-accounting.ts

import { budgetService } from "../src/services/budgets.js";
import { sql } from "drizzle-orm";
/** Reproducible synthetic PostgreSQL benchmark. Never reads DATABASE_URL or
 * provider credentials; startEmbeddedPostgresTestDatabase owns all data. */
import { performance } from "node:perf_hooks";
import { mkdir, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { agents, companies, projects, createDb, startEmbeddedPostgresTestDatabase } from "@paperclipai/db";
import { costService } from "../src/services/costs.js";
import { withAccountingTransaction } from "../src/services/accounting-transaction.js";
import { accountingIntegrityService } from "../src/services/accounting-integrity.js";
import { reconcileRunCosts } from "../src/services/run-cost-accounting.js";
const count = Number(process.env.PAPERCLIP_ACCOUNTING_BENCH_ROWS ?? 1_000_000);
if (!Number.isInteger(count) || count < 1000 || count > 10_000_000) throw new Error("Benchmark row count must be 1,000–10,000,000");
const database = await startEmbeddedPostgresTestDatabase("accounting-scale-");
const db = createDb(database.connectionString);
const started = performance.now();
function distribution(values: number[]) {
  const sorted = [...values].sort((a,b) => a-b);
  const pick = (p: number) => Number(sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)].toFixed(2));
  return { samples: sorted.length, p50Ms: pick(0.5), p95Ms: pick(0.95), p99Ms: pick(0.99), maxMs: pick(1) };
}
async function time(work: () => Promise<unknown>) { const start = performance.now(); await work(); return performance.now() - start; }
try {
  const [company] = await db.insert(companies).values({ name: "Scale fixture", issuePrefix: "SCALE" }).returning();
  const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
  const [project] = await db.insert(projects).values({ companyId: company.id, name: "Hot project" }).returning();
  const seedMs = await time(async () => {
    await db.$client`insert into cost_events (company_id,agent_id,project_id,provider,biller,model,cost_cents,input_tokens,cached_input_tokens,output_tokens,occurred_at)
      select ${company.id},${agent.id}, case when i % 100 = 0 then ${project.id}::uuid else null end, 'fixture','fixture','fixture', 0.1234567,7,11,3,
        date_trunc('month',now() at time zone 'UTC') at time zone 'UTC' - (i % 12) * interval '1 month'
      from generate_series(1,${count}) i`;
    await db.$client`analyze cost_events`;
    // Seed projections through an explicit reviewed repair, not benchmark-only SQL.
    const integrity = accountingIntegrityService(db); const review = await integrity.inspect(company.id);
    await integrity.repair(company.id, review.fingerprint, "Initialize synthetic benchmark projections", "benchmark");
  });
  const reports: number[] = [], projectReports: number[] = [];
  for (let i = 0; i < 20; i++) {
    reports.push(await time(() => costService(db).summary(company.id, { allTime: true })));
    projectReports.push(await time(() => costService(db).byProject(company.id, { allTime: true })));
  }
  const writes: number[] = []; const concurrency = 8; const writeStarted = performance.now();
  for (let batch = 0; batch < 8; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
    agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `bench:${batch}:${index}`, occurredAt: new Date(),
  })).then(ms => writes.push(ms))));
  const writeMs = performance.now() - writeStarted;
  await budgetService(db).upsertPolicy(company.id, { scopeType: "company", scopeId: company.id, amount: 2_000_000_000 }, "benchmark");
  await budgetService(db).upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 2_000_000_000 }, "benchmark");
  const budgetedWrites: number[] = [];
  const budgetedStart = performance.now();
  for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
    agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `budgeted:${batch}:${index}`, occurredAt: new Date(),
  })).then(ms => budgetedWrites.push(ms))));
  const budgetedMs = performance.now() - budgetedStart;
  const waits: number[] = [];
  await Promise.all(Array.from({ length: concurrency }, () => {
    const start = performance.now();
    return withAccountingTransaction(db, company.id, async tx => { waits.push(performance.now() - start); await tx.execute(sql`select pg_sleep(0.005)`); });
  }));
  await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,usage_json,finished_at)
    select ${company.id},${agent.id},'failed',true,'{"accountingReceiptReady":true,"costUsdExact":"0.000000001","inputTokens":1}'::jsonb, now() from generate_series(1,250)`;
  const recovery: Array<{ milliseconds: number; scanned: number; accounted: number }> = [];
  for (let i = 0; i < 3; i++) {
    const before = performance.now(); const result = await reconcileRunCosts(db); recovery.push({ milliseconds: performance.now()-before, ...result });
  }
  if (recovery.reduce((sum, r) => sum+r.accounted,0) !== 250) throw new Error("Benchmark recovery did not conserve receipt count");
  const integrity = await accountingIntegrityService(db).inspect(company.id);
  if (integrity.findings.length) throw new Error(`Benchmark violated accounting integrity: ${JSON.stringify(integrity.findings)}`);
  const plan = await db.$client`explain (analyze,buffers,format json) select sum(cost_cents) from cost_events where company_id = ${company.id} and project_id = ${project.id}
    and occurred_at >= date_trunc('month',now() at time zone 'UTC') at time zone 'UTC'`;
  const report = { generatedAt: new Date().toISOString(), platform: `${os.platform()} ${os.arch()}`, cpus: os.cpus().length, node: process.version,
    seedRows: count, seedMs, concurrency, allTimeSummary: distribution(reports), allTimeProjects: distribution(projectReports),
    hotCompanyWrites: { activePolicies: 0, ...distribution(writes), writesPerSecond: writes.length / (writeMs / 1000) }, companyLockAcquisition: distribution(waits),
    budgetedCompanyWrites: { activePolicies: 2, ...distribution(budgetedWrites), writesPerSecond: budgetedWrites.length / (budgetedMs / 1000) },
    recovery, projectBudgetQueryPlan: plan, integrityFindings: integrity.findings.length, totalMs: performance.now()-started };
  const output = path.resolve("coverage/accounting/scale.json"); await mkdir(path.dirname(output), { recursive: true }); await writeFile(output, JSON.stringify(report,null,2));
  process.stdout.write(JSON.stringify({ ...report, projectBudgetQueryPlan: "see coverage/accounting/scale.json" },null,2)+"\n");
} finally { await db.$client.end(); await database.cleanup(); }