fix: enforce accounting coverage and recovery regression gates

Co-Authored-By: Paperclip <noreply@paperclip.ing>
This commit is contained in:
Devin FoleyandPaperclip committed 2026-10-06 22:27:02 -07:00
1 parent c1102f05d4
commit ec91c9c709
45 files changed
+5400 -20

No files matched your search

+169
View File
@@ -0,0 +1,169 @@
import { randomUUID } from "node:crypto";
import { reserveRunBudget } from "../src/services/budget-reservations.js";
import { createRunUsageRecorder, indexPendingUsageReceipts, usageReceiptSpoolPath } from "../src/services/usage-receipts.js";
import { budgetService } from "../src/services/budgets.js";
import { sql } from "drizzle-orm";
/** Reproducible synthetic PostgreSQL benchmark. Never reads DATABASE_URL or
* provider credentials; startEmbeddedPostgresTestDatabase owns all data. */
import { performance, monitorEventLoopDelay } from "node:perf_hooks";
import { mkdir, writeFile, mkdtemp, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { agents, companies, projects, heartbeatRuns, budgetPolicies, createDb, startEmbeddedPostgresTestDatabase } from "@paperclipai/db";
import { costService } from "../src/services/costs.js";
import { withAccountingTransaction } from "../src/services/accounting-transaction.js";
import { accountingIntegrityService } from "../src/services/accounting-integrity.js";
import { reconcileRunCosts } from "../src/services/run-cost-accounting.js";
const count = Number(process.env.PAPERCLIP_ACCOUNTING_BENCH_ROWS ?? 1_000_000);
if (!Number.isInteger(count) || count < 1000 || count > 10_000_000) throw new Error("Benchmark row count must be 1,000–10,000,000");
// Recovery reads the instance receipt spool as well as the database. Always
// own both; a caller's real PAPERCLIP_HOME must never be scanned or replayed.
const previousHome = process.env.PAPERCLIP_HOME;
const benchmarkHome = await mkdtemp(path.join(os.tmpdir(), "accounting-benchmark-"));
process.env.PAPERCLIP_HOME = benchmarkHome;
const database = await startEmbeddedPostgresTestDatabase("accounting-scale-").catch(async error => {
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
await rm(benchmarkHome, { recursive: true, force: true });
throw error;
});
const db = createDb(database.connectionString);
const started = performance.now();
const eventLoop = monitorEventLoopDelay({ resolution: 10 }); eventLoop.enable();
function distribution(values: number[]) {
const sorted = [...values].sort((a,b) => a-b);
const pick = (p: number) => Number(sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)].toFixed(2));
return { samples: sorted.length, p50Ms: pick(0.5), p95Ms: pick(0.95), p99Ms: pick(0.99), maxMs: pick(1) };
}
async function time(work: () => Promise<unknown>) { const start = performance.now(); await work(); return performance.now() - start; }
try {
const [company] = await db.insert(companies).values({ name: "Scale fixture", issuePrefix: "SCALE" }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Hot project" }).returning();
const seedMs = await time(async () => {
await db.$client`insert into cost_events (company_id,agent_id,project_id,provider,biller,model,cost_cents,input_tokens,cached_input_tokens,output_tokens,occurred_at)
select ${company.id},${agent.id}, case when i % 100 = 0 then ${project.id}::uuid else null end, 'fixture','fixture','fixture', 0.1234567,7,11,3,
date_trunc('month',now() at time zone 'UTC') at time zone 'UTC' - (i % 12) * interval '1 month'
from generate_series(1,${count}) i`;
await db.$client`analyze cost_events`;
// Seed projections through an explicit reviewed repair, not benchmark-only SQL.
const integrity = accountingIntegrityService(db); const review = await integrity.inspect(company.id);
await integrity.repair(company.id, review.fingerprint, "Initialize synthetic benchmark projections", "benchmark");
});
const reports: number[] = [], projectReports: number[] = [];
for (let i = 0; i < 20; i++) {
reports.push(await time(() => costService(db).summary(company.id, { allTime: true })));
projectReports.push(await time(() => costService(db).byProject(company.id, { allTime: true })));
}
const writes: number[] = []; const concurrency = 8; const writeStarted = performance.now();
for (let batch = 0; batch < 8; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `bench:${batch}:${index}`, occurredAt: new Date(),
})).then(ms => writes.push(ms))));
const writeMs = performance.now() - writeStarted;
await budgetService(db).upsertPolicy(company.id, { scopeType: "company", scopeId: company.id, amount: 2_000_000_000 }, "benchmark");
await budgetService(db).upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 2_000_000_000 }, "benchmark");
const budgetedWrites: number[] = [];
const budgetedStart = performance.now();
for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `budgeted:${batch}:${index}`, occurredAt: new Date(),
})).then(ms => budgetedWrites.push(ms))));
const budgetedMs = performance.now() - budgetedStart;
const admissions: number[] = [];
for (let i = 0; i < 20; i++) {
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
admissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null)));
}
const dashboard: number[] = [], writesWhilePolling: number[] = [];
// Eight viewers each issue the ledger queries used by the Costs overview,
// together with budget and accounting health reads, while eight writes run.
for (let batch = 0; batch < 4; batch++) await Promise.all([
...Array.from({ length: 8 }, () => time(() => Promise.all([
costService(db).summary(company.id), costService(db).byAgent(company.id),
costService(db).byProject(company.id), costService(db).byAgentModel(company.id),
budgetService(db).overview(company.id), accountingIntegrityService(db).health(company.id),
])).then(ms => dashboard.push(ms))),
...Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `polling:${batch}:${index}`, occurredAt: new Date(),
})).then(ms => writesWhilePolling.push(ms))),
]);
const [checkpointRun] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
const recorder = await createRunUsageRecorder(db, { companyId: company.id, runId: checkpointRun.id, adapterType: "process" });
const checkpoints: number[] = [];
for (let i = 1; i <= 50; i++) checkpoints.push(await time(() => recorder.capture({ complete: false, usageBasis: "per_run", usage: { inputTokens: i, outputTokens: i }, costUsdExact: "0.000000001" })));
// Include historical run-table size, independently of the pending index.
await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,finished_at)
select ${company.id},${agent.id},'succeeded',false, now() from generate_series(1,100000)`;
await db.$client`analyze heartbeat_runs`;
const health: number[] = [];
for (let i = 0; i < 20; i++) health.push(await time(() => accountingIntegrityService(db).health(company.id)));
const waits: number[] = [];
await Promise.all(Array.from({ length: concurrency }, () => {
const start = performance.now();
return withAccountingTransaction(db, company.id, async tx => { waits.push(performance.now() - start); await tx.execute(sql`select pg_sleep(0.005)`); });
}));
await db.$client`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending,usage_json,finished_at)
select ${company.id},${agent.id},'failed',true,'{"accountingReceiptReady":true,"costUsdExact":"0.000000001","inputTokens":1}'::jsonb, now() from generate_series(1,250)`;
const recovery: Array<{ milliseconds: number; scanned: number; accounted: number }> = [];
for (let i = 0; i < 3; i++) {
const before = performance.now(); const result = await reconcileRunCosts(db); recovery.push({ milliseconds: performance.now()-before, ...result });
}
if (recovery.reduce((sum, r) => sum+r.accounted,0) !== 250) throw new Error("Benchmark recovery did not conserve receipt count");
const spool = usageReceiptSpoolPath(); await mkdir(spool, { recursive: true });
const spoolBacklogs: Array<{ files: number; indexMs: number; recorderStartMs: number }> = [];
const foreignRunId = randomUUID();
let published = 0;
for (const files of [1000, 10000]) {
// Publish synthetic immutable files directly; timed checkpoints above use
// the real fsync path. These belong to another run and must be retained.
for (; published < files; published += 50) await Promise.all(Array.from({ length: 50 }, (_, offset) => writeFile(path.join(spool, `${published + offset}.json`), JSON.stringify({
schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: company.id, runId: foreignRunId,
sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false },
}))));
const indexMs = await time(() => indexPendingUsageReceipts());
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
const recorderStartMs = await time(() => createRunUsageRecorder(db, { companyId: company.id, runId: run.id, adapterType: "process" }));
spoolBacklogs.push({ files, indexMs, recorderStartMs });
}
await rm(spool, { recursive: true, force: true });
// A large policy list must not make each write fetch every agent's policies.
const extraAgents = await db.insert(agents).values(Array.from({ length: 100 }, (_, i) => ({ companyId: company.id, name: `Idle ${i}`, role: "engineer", adapterType: "process" }))).returning();
await db.insert(budgetPolicies).values(extraAgents.map(extra => ({ companyId: company.id, scopeType: "agent", scopeId: extra.id, windowKind: "calendar_month_utc", amount: 2_000_000_000 })));
const largeOverview: number[] = [];
for (let i = 0; i < 10; i++) largeOverview.push(await time(async () => {
const overview = await budgetService(db).overview(company.id);
if (overview.policies.length !== 102) throw new Error("Benchmark policy fixture is incomplete");
}));
const largePolicyWrites: number[] = [], largePolicyAdmissions: number[] = [];
const largePolicyStart = performance.now();
for (let batch = 0; batch < 4; batch++) await Promise.all(Array.from({ length: concurrency }, (_, index) => time(() => costService(db).createEvent(company.id, {
agentId: agent.id, provider: "fixture", model: "fixture", costCents: "0.0000001", idempotencyKey: `large-policy:${batch}:${index}`, occurredAt: new Date(),
})).then(ms => largePolicyWrites.push(ms))));
const largePolicyMs = performance.now() - largePolicyStart;
for (let i = 0; i < 20; i++) {
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running" }).returning();
largePolicyAdmissions.push(await time(() => reserveRunBudget(db, company.id, run.id, null)));
}
const integrity = await accountingIntegrityService(db).inspect(company.id);
if (integrity.findings.length) throw new Error(`Benchmark violated accounting integrity: ${JSON.stringify(integrity.findings)}`);
const plan = await db.$client`explain (analyze,buffers,format json) select sum(cost_cents) from cost_events where company_id = ${company.id} and project_id = ${project.id}
and occurred_at >= date_trunc('month',now() at time zone 'UTC') at time zone 'UTC'`;
const report = { generatedAt: new Date().toISOString(), platform: `${os.platform()} ${os.arch()}`, cpus: os.cpus().length, node: process.version,
seedRows: count, seedMs, concurrency, allTimeSummary: distribution(reports), allTimeProjects: distribution(projectReports),
hotCompanyWrites: { activePolicies: 0, ...distribution(writes), writesPerSecond: writes.length / (writeMs / 1000) }, companyLockAcquisition: distribution(waits),
budgetedCompanyWrites: { activePolicies: 2, ...distribution(budgetedWrites), writesPerSecond: budgetedWrites.length / (budgetedMs / 1000) },
admission: distribution(admissions), concurrentDashboard: distribution(dashboard), writesWhilePolling: distribution(writesWhilePolling),
durableCheckpoint: distribution(checkpoints), healthWith100kHistoricalRuns: distribution(health),
overviewWith102Policies: distribution(largeOverview),
writesWith102Policies: { activePolicies: 102, relevantPolicies: 2, ...distribution(largePolicyWrites), writesPerSecond: largePolicyWrites.length / (largePolicyMs / 1000) },
admissionWith102Policies: distribution(largePolicyAdmissions), spoolBacklogs,
eventLoopDelay: { p95Ms: eventLoop.percentile(95) / 1e6, maxMs: eventLoop.max / 1e6 },
recovery, projectBudgetQueryPlan: plan, integrityFindings: integrity.findings.length, totalMs: performance.now()-started };
const output = path.resolve("coverage/accounting/scale.json"); await mkdir(path.dirname(output), { recursive: true }); await writeFile(output, JSON.stringify(report,null,2));
process.stdout.write(JSON.stringify({ ...report, projectBudgetQueryPlan: "see coverage/accounting/scale.json" },null,2)+"\n");
} finally {
eventLoop.disable();
try { await db.$client.end(); await database.cleanup(); }
finally {
if (previousHome === undefined) delete process.env.PAPERCLIP_HOME; else process.env.PAPERCLIP_HOME = previousHome;
await rm(benchmarkHome, { recursive: true, force: true });
}
}
@@ -1,3 +1,5 @@
import { costService, createCostEventInTransaction } from "../services/costs.js";
import { withAccountingTransaction } from "../services/accounting-transaction.js";
import { budgetService } from "../services/budgets.js";
import { buildPaperclipRuntimeMcpServers } from "../services/heartbeat.js";
import { resolveNativeRuntimeMcpSnapshot } from "../services/native-runtime/runtime-context.js";
@@ -480,6 +482,69 @@ const actor = { actorType: "user" as const, actorId: "browser-reviewer" };
f.access.refreshCatalog(f.connection.connectionId, actor),
).rejects.toThrow();
});
it("rolls back the Browser Use charge and Finance event if a spend projection fails", async () => {
const f = await fixture();
await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: 100, occurredAt: new Date() });
await f.service.execute(f.binding, f.grant, randomUUID(), "browser_start", { task: "Read example.com" });
f.complete();
await db.execute(sql`create function fail_browser_projection() returns trigger language plpgsql as $$ begin raise exception 'injected projection failure'; end $$`);
await db.execute(sql.raw(`create trigger fail_browser_projection before update of spent_monthly_cents on companies for each row when (new.id = '${f.company.id}'::uuid) execute function fail_browser_projection()`));
try {
await f.tick();
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(0);
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(100);
expect((await db.select().from(browserUseRuns).where(eq(browserUseRuns.companyId, f.company.id)))[0].accountedCents).toBe(0);
} finally {
await db.execute(sql`drop trigger fail_browser_projection on companies`);
await db.execute(sql`drop function fail_browser_projection()`);
}
await f.tick();
await f.tick();
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("115.0000000");
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(2);
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toMatchObject([{ amountCents: 15 }]);
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].spentMonthlyCents).toBe(115);
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(115);
});
it("settles Browser Use atomically behind another accounting writer without losing spend", async () => {
const f = await fixture();
const charge = (costCents: number) => ({ agentId: f.agent.id, provider: "fixture", model: "fixture", costCents, occurredAt: new Date() });
await costService(db).createEvent(f.company.id, charge(100));
await f.service.execute(f.binding, f.grant, randomUUID(), "browser_start", { task: "Read example.com" });
f.complete();
let release!: () => void;
let locked!: () => void;
const released = new Promise<void>(resolve => { release = resolve; });
const ready = new Promise<void>(resolve => { locked = resolve; });
const writer = withAccountingTransaction(db, f.company.id, async (tx, publications) => {
await createCostEventInTransaction(tx, f.company.id, charge(5), publications);
locked();
await released;
});
await ready;
const browser = f.tick();
try {
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database() and wait_event_type = 'Lock' and pid <> pg_backend_pid()`);
expect(waiting.length).toBeGreaterThan(0);
});
// Nothing from Browser Use may become visible before its projections.
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(0);
} finally {
release();
await Promise.all([writer, browser]);
}
await f.tick(); // Replaying the provider's cumulative total is a no-op.
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("120.0000000");
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].spentMonthlyCents).toBe(120);
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].spentMonthlyCents).toBe(120);
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toMatchObject([{ amountCents: 15 }]);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(3);
});
it("observes delayed browser.ready, hides viewer credentials, accounts once and cleans up", async () => {
const f = await fixture();
const invocation = randomUUID();
+120
View File
@@ -0,0 +1,120 @@
import { describe, expect, it } from "vitest";
import type { AdapterUsageCheckpoint } from "@paperclipai/adapter-utils";
import { priceCodexReceipt } from "../services/codex-pricing.js";
const receipt: AdapterUsageCheckpoint = {
provider: "openai",
biller: "openai",
billingType: "metered_api",
model: "gpt-6-astra",
complete: true,
usageBasis: "per_run",
costUsd: null,
usage: {
inputTokens: 123536,
cachedInputTokens: 1567209,
outputTokens: 6625,
},
};
describe("prospective Codex pricing", () => {
it.each([
["gpt-6-sol", "12.200000000"],
["gpt-6.1-sol", "12.100000000"],
["gpt-6-luna", "0.610000000"],
["gpt-5.6-sol", "24.400000000"],
])("uses the distinct published cache rate for %s", (model, expected) => {
expect(
priceCodexReceipt({
...receipt,
model,
usage: {
inputTokens: 1_000_000,
cachedInputTokens: 1_000_000,
outputTokens: 1_000_000,
},
}).costUsdExact,
).toBe(expected);
});
it("prices the staging token shape without counting cached input twice", () => {
const result = priceCodexReceipt(receipt);
expect(result.costUsdExact).toBe("3.133819000");
expect(result.costStatus).toBe("estimated");
expect(result.pricingProvenance).toMatchObject({
source: "rate_card",
version: "openai-standard-2026-09-30",
});
expect(result.pricingProvenance?.evidence).toContain(
"short per-request context assumed",
);
expect(receipt.costUsdExact).toBeUndefined();
});
it.each([
{ complete: false },
{ complete: false, usage: { inputTokens: 0, outputTokens: 0 } },
{ costUsd: 0 },
{ costUsd: 4.21 },
{ costUsdExact: "0.004" },
{ cacheAdjustedCostUsd: 0.01 },
{ provider: "other" },
{ biller: "proxy" },
{ model: "gpt-6-astra-custom" },
{ model: "unknown" },
{ model: "constructor" },
{ model: "__proto__" },
{ usageBasis: undefined },
{ usageBasis: null },
{ billingType: "unknown" },
{ billingType: "subscription_included" },
{ usageBasis: "session_cumulative" },
{ pricingContext: { serviceTier: "unsupported" } },
{ usage: undefined },
{ usage: { inputTokens: -1, outputTokens: 0 } },
{ usage: { inputTokens: 1, outputTokens: 0, cacheWriteTokens: 2 } },
])(
"preserves reported costs and refuses ambiguous pricing: %j",
(override) => {
const input = { ...receipt, ...override } as AdapterUsageCheckpoint;
expect(priceCodexReceipt(input)).toBe(input);
},
);
it.each([
["standard", "short", "61.000000000"],
["fast", "short", "122.000000000"],
["flex", "short", "30.500000000"],
["standard", "long", "97.000000000"],
["batch", "long", "48.500000000"],
] as const)("handles %s/%s rates", (serviceTier, contextTier, expected) => {
const result = priceCodexReceipt({
...receipt,
pricingContext: { serviceTier, contextTier },
usage: {
inputTokens: 1_000_000,
cachedInputTokens: 1_000_000,
outputTokens: 1_000_000,
cacheWriteTokens: 0,
},
});
expect(result.costUsdExact).toBe(expected);
});
it("charges cache writes once as a subset of input", () => {
expect(
priceCodexReceipt({
...receipt,
usage: {
inputTokens: 1_000_000,
cacheWriteTokens: 1_000_000,
outputTokens: 0,
},
}).costUsdExact,
).toBe("12.500000000");
});
it("retains nanodollar precision and does not reprice frozen receipts", () => {
const priced = priceCodexReceipt({
...receipt,
model: "gpt-6-luna",
usage: { inputTokens: 0, cachedInputTokens: 1, outputTokens: 0 },
});
expect(priced.costUsdExact).toBe("0.000000010");
expect(priceCodexReceipt(priced)).toBe(priced);
});
});
@@ -0,0 +1,162 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { agents, authUsers, companies, companyMemberships, costEvents, createDb, heartbeatRuns, issues, type Db } from "@paperclipai/db";
import { costService } from "../services/costs.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
describe("cost attribution by user (PostgreSQL)", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: Db;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-by-user-"); db = createDb(database.connectionString); }, 30_000);
afterAll(async () => { await database?.cleanup(); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "User costs", issuePrefix: `U${randomUUID().slice(0, 7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Shared worker", role: "engineer", adapterType: "process" }).returning();
return { companyId: company.id, agentId: agent.id };
}
async function member(companyId: string, name: string, status = "active") {
const userId = randomUUID();
await db.insert(authUsers).values({ id: userId, name, email: `${userId}@example.test`, createdAt: new Date(), updatedAt: new Date() });
await db.insert(companyMemberships).values({ companyId, principalType: "user", principalId: userId, status });
return userId;
}
async function run(f: Awaited<ReturnType<typeof fixture>>, responsibleUserId: string | null) {
return (await db.insert(heartbeatRuns).values({ ...f, responsibleUserId, status: "succeeded" }).returning())[0];
}
async function charge(f: Awaited<ReturnType<typeof fixture>>, overrides: Partial<typeof costEvents.$inferInsert> = {}) {
await db.insert(costEvents).values({ ...f, provider: "openai", model: "fixture", costCents: 1,
billingType: "metered_api", occurredAt: new Date(), ...overrides });
}
function trace() {
const queries: string[] = [];
const traced = drizzle(db.$client, { logger: { logQuery(query) { queries.push(query); } } }) as unknown as Db;
return { service: costService(traced), ledgerReads: () => queries.filter(query => query.startsWith("select") && query.includes('from "cost_events"')) };
}
it("reports spend for any number of users and only includes active humans with zero spend", async () => {
const f = await fixture(), foreign = await fixture();
const { service, ledgerReads } = trace();
expect(await service.byUser(f.companyId)).toEqual({ activeUserCount: 0, rows: [] });
await member(f.companyId, "Alice");
await member(f.companyId, "Invited", "pending");
await member(f.companyId, "Suspended", "suspended");
await member(f.companyId, "Former", "archived");
await member(foreign.companyId, "Foreign");
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "agent", principalId: f.agentId });
await charge(f);
expect(await service.byUser(f.companyId)).toMatchObject({ activeUserCount: 1, rows: [
{ userId: null, costCents: 1 }, { userName: "Alice", costCents: 0 },
] });
expect(ledgerReads()).toHaveLength(2);
await member(f.companyId, "Bob");
expect(await service.byUser(f.companyId)).toMatchObject({ activeUserCount: 2, rows: [
{ userId: null, costCents: 1 }, { userName: "Alice", costCents: 0 }, { userName: "Bob", costCents: 0 },
] });
expect(ledgerReads()).toHaveLength(3);
});
it("excludes the Board principal by ID and preserves its historical charges as unattributed", async () => {
const f = await fixture();
await db.insert(authUsers).values({ id: "local-board", name: "Board", email: "local@paperclip.local", createdAt: new Date(), updatedAt: new Date() }).onConflictDoNothing();
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: "local-board" });
const service = costService(db);
expect(await service.byUser(f.companyId)).toEqual({ activeUserCount: 0, rows: [] });
// A real person is never filtered by their display name.
const human = await member(f.companyId, "Board");
const humanRun = await run(f, human), boardRun = await run(f, "local-board");
await charge(f, { heartbeatRunId: humanRun.id, costCents: 0.25 });
await charge(f, { heartbeatRunId: boardRun.id, costCents: 1 });
const report = await service.byUser(f.companyId);
expect(report).toMatchObject({ activeUserCount: 1, rows: [
{ userId: null, userName: null, costCentsExact: "1.0000000", runCount: 1 },
{ userId: human, userName: "Board", costCentsExact: "0.2500000", runCount: 1 },
] });
expect(report.rows.some(row => row.userId === "local-board")).toBe(false);
expect((await service.summary(f.companyId)).spendCentsExact).toBe("1.2500000");
});
it("preserves exact spend, legacy cache semantics, zero-spend members and distinct runs", async () => {
const f = await fixture();
const alice = await member(f.companyId, "Alice"), bob = await member(f.companyId, "Bob");
const former = await member(f.companyId, "Former", "archived");
const a = await run(f, alice), b = await run(f, former);
const [otherAgent] = await db.insert(agents).values({ companyId: f.companyId, name: "Second worker", role: "engineer", adapterType: "process" }).returning();
const c = await run({ ...f, agentId: otherAgent.id }, alice);
await charge(f, { heartbeatRunId: a.id, costCents: 0.1000001, inputTokens: 100, cachedInputTokens: 40, outputTokens: 5 });
await charge(f, { heartbeatRunId: a.id, costCents: 0.2000002, receiptHash: "modern", costStatus: "estimated", inputTokens: 100, cachedInputTokens: 40, outputTokens: 10 });
await charge(f, { heartbeatRunId: b.id, costCents: 0.4 });
await charge(f, { heartbeatRunId: c.id, agentId: otherAgent.id, costCents: 0, costStatus: "unpriced", inputTokens: 9 });
await charge(f, { heartbeatRunId: a.id, costCents: 0, costStatus: "unpriced", billingType: "subscription_included" });
await charge(f, { costCents: 0.0000001 });
const { service, ledgerReads } = trace();
const report = await service.byUser(f.companyId, { allTime: true });
expect(ledgerReads()).toHaveLength(1);
expect(report.rows.find(row => row.userId === alice)).toMatchObject({
costCentsExact: "0.3000003", eventCount: 4, estimatedEventCount: 1, unpricedEventCount: 1,
inputTokens: 169, cachedInputTokens: 80, outputTokens: 15, runCount: 2,
});
expect(report.rows.find(row => row.userId === bob)).toMatchObject({ costCentsExact: "0.0000000", eventCount: 0, runCount: 0 });
expect(report.rows.find(row => row.userId === former)).toMatchObject({ userName: "Former", costCentsExact: "0.4000000", runCount: 1 });
expect(report.rows.find(row => row.userId === null)).toMatchObject({ costCentsExact: "0.0000001", runCount: 0 });
const sum = report.rows.reduce((total, row) => total + BigInt(row.costCentsExact.replace(".", "")), 0n);
expect(sum).toBe(7000004n);
expect((await service.summary(f.companyId, { allTime: true })).spendCentsExact).toBe("0.7000004");
});
it("retains charges and active memberships when user profiles are missing", async () => {
const f = await fixture();
const missing = [randomUUID(), randomUUID()].sort();
const former = randomUUID();
for (const userId of [...missing, former]) {
await db.insert(companyMemberships).values({ companyId: f.companyId, principalType: "user", principalId: userId,
status: userId === former ? "archived" : "active" });
}
const ownerRun = await run(f, former);
await charge(f, { heartbeatRunId: ownerRun.id, costCents: 0.125 });
const report = await costService(db).byUser(f.companyId);
expect(report).toMatchObject({ activeUserCount: 2, rows: [
{ userId: former, userName: "Former user", userImage: null, costCentsExact: "0.1250000", runCount: 1 },
...missing.map(userId => ({ userId, userName: "Unknown user", userImage: null, costCentsExact: "0.0000000", runCount: 0 })),
] });
expect((await costService(db).summary(f.companyId)).spendCentsExact).toBe("0.1250000");
});
it("keeps every company charge but never attributes through a foreign run, agent or user", async () => {
const f = await fixture(), foreign = await fixture();
const alice = await member(f.companyId, "Alice"); await member(f.companyId, "Bob");
const outsider = await member(foreign.companyId, "Secret foreign name");
const foreignRun = await run(foreign, alice);
const wrongUser = await run(f, outsider);
const unowned = await run(f, null);
const [otherAgent] = await db.insert(agents).values({ companyId: f.companyId, name: "Other", role: "engineer", adapterType: "process" }).returning();
const wrongAgent = await run({ ...f, agentId: otherAgent.id }, alice);
for (const r of [foreignRun, wrongUser, unowned, wrongAgent]) await charge(f, { heartbeatRunId: r.id });
await charge(foreign, { heartbeatRunId: foreignRun.id, costCents: 999 });
const report = await costService(db).byUser(f.companyId, { allTime: true });
expect(report.rows.find(row => row.userId === null)).toMatchObject({ costCentsExact: "4.0000000", eventCount: 4, runCount: 2 });
expect(report.rows.find(row => row.userId === alice)?.costCents).toBe(0);
expect(JSON.stringify(report)).not.toContain(outsider);
expect(JSON.stringify(report)).not.toContain("Secret foreign name");
});
it("filters receipt dates inclusively and never reassigns history when the issue owner changes", async () => {
const f = await fixture();
const alice = await member(f.companyId, "Alice"), bob = await member(f.companyId, "Bob");
const [issue] = await db.insert(issues).values({ companyId: f.companyId, title: "Shared work", responsibleUserId: alice }).returning();
const r = await run(f, alice);
const from = new Date("2001-01-01T00:00:00Z"), to = new Date("2001-01-02T00:00:00Z");
for (const occurredAt of [new Date(from.getTime() - 1), from, to, new Date(to.getTime() + 1)]) {
await charge(f, { heartbeatRunId: r.id, issueId: issue.id, occurredAt });
}
await db.update(issues).set({ responsibleUserId: bob }).where(eq(issues.id, issue.id));
const service = costService(db);
expect((await service.byUser(f.companyId, { from, to })).rows.find(row => row.userId === alice)).toMatchObject({ costCents: 2, runCount: 1 });
expect((await service.byUser(f.companyId, { allTime: true })).rows.find(row => row.userId === alice)?.costCents).toBe(4);
expect((await service.byUser(f.companyId)).rows.every(row => row.costCents === 0)).toBe(true);
await charge(f, { heartbeatRunId: r.id });
expect((await service.byUser(f.companyId)).rows.find(row => row.userId === alice)?.costCents).toBe(1);
await expect(service.byUser(randomUUID())).rejects.toThrow("Company not found");
});
});
@@ -0,0 +1,61 @@
import { describe, expect, it, vi } from "vitest";
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
import { parseClaudeStreamJson } from "../../../packages/adapters/claude-local/src/server/parse.js";
import { parseCodexJsonl } from "../../../packages/adapters/codex-local/src/server/parse.js";
import { parseCursorJsonl } from "../../../packages/adapters/cursor-local/src/server/parse.js";
import { parseGeminiJsonl } from "../../../packages/adapters/gemini-local/src/server/parse.js";
import { parseOpenCodeJsonl } from "../../../packages/adapters/opencode-local/src/server/parse.js";
import { parsePiJsonl } from "../../../packages/adapters/pi-local/src/server/parse.js";
const secret = "TRANSCRIPT_CONTENT_MUST_NOT_ENTER_ACCOUNTING_BUFFER";
const fixtures = [
{ name: "claude", event: { type: "result", total_cost_usd: 0.0123, result: secret, usage: { input_tokens: 7, cache_read_input_tokens: 11, output_tokens: 3 } }, parse: parseClaudeStreamJson, input: 7, cached: 11, output: 3 },
{ name: "codex", event: { type: "turn.completed", text: secret, usage: { input_tokens: 18, cached_input_tokens: 11, output_tokens: 3 } }, parse: parseCodexJsonl, input: 7, cached: 11, output: 3 },
{ name: "cursor", event: { type: "result", result: secret, usage: { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 } }, parse: parseCursorJsonl, input: 7, cached: 11, output: 3 },
{ name: "gemini", event: { type: "result", message: secret, stats: { input_tokens: 18, cached: 11, output_tokens: 1, total_tokens: 21 } }, parse: parseGeminiJsonl, input: 7, cached: 11, output: 3 },
{ name: "opencode", event: { type: "step_finish", part: { text: secret, cost: 0.0123, tokens: { input: 5, cache: { read: 11, write: 2 }, output: 1, reasoning: 2 } } }, parse: parseOpenCodeJsonl, input: 7, cached: 11, output: 3 },
{ name: "pi", event: { type: "turn_end", message: { role: "assistant", content: secret, usage: { input: 5, cacheWrite: 2, cacheRead: 11, output: 3, cost: { total: 0.0123 } } } }, parse: parsePiJsonl, input: 7, cached: 11, output: 3 },
];
describe("durable accounting stream projection", () => {
it.each(fixtures)("captures split $name usage without retaining transcript content", async f => {
const onUsage = vi.fn(async (..._args: any[]) => {}), onLog = vi.fn(async (..._args: any[]) => {});
const parse = vi.fn((source: string) => {
expect(source).not.toContain(secret);
const result = f.parse(source); return { usage: result.usage ?? undefined, complete: true };
});
const log = createUsageCheckpointLog(onLog, onUsage, parse);
const line = JSON.stringify(f.event) + "\n";
for (let index = 0; index < line.length; index += 7) await log("stdout", line.slice(index,index+7));
expect(onUsage).toHaveBeenCalledTimes(1);
expect(onUsage.mock.calls[0][0]).toMatchObject({ complete: true, attemptId: expect.any(String), usage: { inputTokens: f.input, cachedInputTokens: f.cached, outputTokens: f.output } });
expect(onLog.mock.calls.map(call => call[1]).join("")).toBe(line);
});
it("awaits durability before publication, suppresses identical snapshots and separates attempts", async () => {
const order: string[] = [];
const onUsage = vi.fn(async (..._args: any[]) => { order.push("durable"); });
const parse = () => ({ usage: { inputTokens: 1, outputTokens: 2 }, complete: false });
const log = createUsageCheckpointLog(async () => { order.push("published"); }, onUsage, parse);
const message = '{"type":"result","usage":{"input_tokens":1}}\n';
await log("stdout", message); await log("stdout", message);
expect(order).toEqual(["durable", "published", "published"]);
const other = createUsageCheckpointLog(async () => {}, onUsage, parse); await other("stdout", message);
expect(onUsage.mock.calls[0][0].attemptId).not.toBe(onUsage.mock.calls[1][0].attemptId);
});
it("propagates receipt failures and leaves ordinary logs and stderr alone when disabled", async () => {
const onLog = vi.fn(async (..._args: any[]) => {});
const disabled = createUsageCheckpointLog(onLog, undefined, () => null);
await disabled("stdout", "ordinary output"); await disabled.flush();
expect(onLog).toHaveBeenCalledWith("stdout", "ordinary output");
onLog.mockClear();
const onUsage = vi.fn(async () => { throw new Error("spool full"); });
const parse = vi.fn(() => ({ usage: { inputTokens: 1, outputTokens: 0 }, complete: false }));
const log = createUsageCheckpointLog(onLog, onUsage, parse);
await log("stderr", '{"usage":1}\n'); await log("stdout", 'invalid json\n{"text":"plain"}\n');
expect(onUsage).not.toHaveBeenCalled(); onLog.mockClear();
await log("stdout", '{"usage":{"input_tokens":1}}\n');
expect(onLog).toHaveBeenCalled();
await expect(log.flush()).rejects.toThrow("spool full");
const oversized = createUsageCheckpointLog(onLog, onUsage, parse);
await oversized("stdout", "x".repeat(8*1024*1024+1));
await expect(oversized.flush()).rejects.toThrow("exceeds");
});
});
@@ -0,0 +1,764 @@
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
it,
vi,
} from "vitest";
import { eq } from "drizzle-orm";
import {
agents,
companies,
companySecrets,
costEvents,
financeEvents,
heartbeatRuns,
providerBillingSnapshots,
createDb,
} from "@paperclipai/db";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { billingReconciliationService } from "../services/billing-reconciliation.js";
import { financeService } from "../services/finance.js";
import {
applyProviderDailyCosts,
importProviderDailyCosts,
} from "../services/provider-billing-import.js";
import * as secretsModule from "../services/secrets.js";
import {
createRunUsageRecorder,
persistUsageReceipt,
} from "../services/usage-receipts.js";
import { accountRunCost } from "../services/run-cost-accounting.js";
import { costService, getMonthlySpendTotal } from "../services/costs.js";
import { budgetService } from "../services/budgets.js";
import type { ImportProviderCosts } from "@paperclipai/shared";
describe("completed cost reporting paths (PostgreSQL)", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let spool: string;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase(
"accounting-completion-",
);
db = createDb(database.connectionString);
spool = await mkdtemp(join(tmpdir(), "completion-spool-"));
}, 90000);
afterAll(async () => {
await database?.cleanup();
if (spool) await rm(spool, { recursive: true, force: true });
});
afterEach(() => vi.restoreAllMocks());
const company = async () =>
(
await db
.insert(companies)
.values({
name: "Completion",
issuePrefix: `C${randomUUID().slice(0, 7)}`,
})
.returning()
)[0];
it("reads monthly spend from the ledger with company, agent and UTC month boundaries", async () => {
const c = await company();
const other = await company();
const [first, second] = await db.insert(agents).values([
{ companyId: c.id, name: "First", spentMonthlyCents: 9999 },
{ companyId: c.id, name: "Second", spentMonthlyCents: 9999 },
]).returning();
const [outsider] = await db.insert(agents).values({ companyId: other.id, name: "Other" }).returning();
const now = new Date();
const start = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 1));
const end = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + 1, 1));
const base = { companyId: c.id, agentId: first.id, provider: "test", model: "test", costCents: 1.125, occurredAt: start };
await db.insert(costEvents).values([
base,
{ ...base, agentId: second.id, costCents: 2.25, occurredAt: new Date(end.getTime() - 1) },
{ ...base, costCents: 500, occurredAt: new Date(start.getTime() - 1) },
{ ...base, costCents: 600, occurredAt: end },
{ ...base, companyId: other.id, agentId: outsider.id, costCents: 700 },
]);
expect(await getMonthlySpendTotal(db, { companyId: c.id })).toBe(3.375);
expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: first.id })).toBe(1.125);
expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: second.id })).toBe(2.25);
expect(await getMonthlySpendTotal(db, { companyId: c.id, agentId: outsider.id })).toBe(0);
expect(await getMonthlySpendTotal(db, { companyId: randomUUID() })).toBe(0);
});
it("scopes agent and model estimate counts to the company and selected period", async () => {
const c = await company();
const other = await company();
const [codie, bender] = await db.insert(agents).values([
{ companyId: c.id, name: "Codie", adapterType: "paperclip_runner" },
{ companyId: c.id, name: "Bender", adapterType: "claude_local" },
]).returning();
const [outsider] = await db.insert(agents).values({ companyId: other.id, name: "Other", adapterType: "process" }).returning();
const base = { companyId: c.id, agentId: codie.id, provider: "openai", biller: "openai", billingType: "metered_api", model: "gpt-6-astra", costCents: 100, occurredAt: new Date("2026-09-10T12:00:00Z") };
await db.insert(costEvents).values([
{ ...base, costStatus: "estimated" },
{ ...base, costStatus: "estimated" },
{ ...base, model: "gpt-6-sol", costStatus: "reported" },
{ ...base, costStatus: "reported", occurredAt: new Date("2026-08-31T23:59:59Z") },
{ ...base, costStatus: "estimated", occurredAt: new Date("2026-10-01T00:00:00Z") },
{ ...base, agentId: bender.id, costStatus: "reported" },
{ ...base, companyId: other.id, agentId: outsider.id, costStatus: "estimated" },
]);
const range = { from: new Date("2026-09-01T00:00:00Z"), to: new Date("2026-09-30T23:59:59Z") };
const service = costService(db);
const totals = await service.byAgent(c.id, range);
expect(totals).toHaveLength(2);
expect(totals.find(row => row.agentId === codie.id)).toMatchObject({ costCents: 300, eventCount: 3, estimatedEventCount: 2, apiRunCount: 0 });
expect(totals.find(row => row.agentId === bender.id)).toMatchObject({ eventCount: 1, estimatedEventCount: 0 });
const models = (await service.byAgentModel(c.id, range)).filter(row => row.agentId === codie.id);
expect(models).toHaveLength(2);
expect(models.find(row => row.model === "gpt-6-astra")).toMatchObject({ eventCount: 2, estimatedEventCount: 2 });
expect(models.find(row => row.model === "gpt-6-sol")).toMatchObject({ eventCount: 1, estimatedEventCount: 0 });
});
it("imports inference, fees and credits atomically into the timeline, without duplicating on retry", async () => {
const c = await company();
const service = billingReconciliationService(db);
const invoice = {
biller: "openai",
externalId: "invoice-1",
currency: "USD",
lines: [
{
externalId: "usage",
kind: "inference" as const,
amountCents: "10.0000001",
occurredAt: "2026-09-01T00:00:00Z",
},
{
externalId: "subscription",
kind: "fee" as const,
amountCents: "2000",
occurredAt: "2026-09-01T00:00:00Z",
},
{
externalId: "refund",
kind: "credit" as const,
amountCents: "100",
occurredAt: "2026-09-01T00:00:00Z",
},
],
};
const imports = await Promise.all([
service.importInvoice(c.id, invoice, "board"),
service.importInvoice(c.id, invoice, "board"),
]);
expect(imports[0].id).toBe(imports[1].id);
expect(
await financeService(db).list(c.id, {
from: new Date("2026-09-01"),
to: new Date("2026-09-30"),
}),
).toHaveLength(3);
expect(
(
await financeService(db).summary(c.id, {
from: new Date("2026-09-01"),
to: new Date("2026-09-30"),
})
).netCentsExact,
).toBe("1910.0000001");
expect(
await db.select().from(costEvents).where(eq(costEvents.companyId, c.id)),
).toHaveLength(0);
await expect(
service.importInvoice(
c.id,
{ ...invoice, lines: [{ ...invoice.lines[0], amountCents: "99" }] },
"board",
),
).rejects.toThrow("different contents");
expect(
await db
.select()
.from(financeEvents)
.where(eq(financeEvents.companyId, c.id)),
).toHaveLength(3);
});
it.each(["costEventId", "runId", "providerRequestId"] as const)("keeps overlapping invoice evidence out of Finance totals using %s", async (reference) => {
const c = await company();
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Browser worker" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: c.id, agentId: agent.id, invocationSource: "on_demand" }).returning();
const charge = await costService(db).createEvent(c.id, { agentId: agent.id, heartbeatRunId: run.id,
provider: "browser-use-cloud", model: "test", providerRequestId: "request-1", costCents: "15.0000001", occurredAt: new Date() });
const finance = financeService(db);
await finance.createEvent(c.id, { costEventId: charge.id, heartbeatRunId: run.id,
biller: "browser-use-cloud", eventKind: "inference_charge", amountCents: "15.0000001", occurredAt: new Date() });
const service = billingReconciliationService(db);
const linked = { [reference]: reference === "costEventId" ? charge.id : reference === "runId" ? run.id : "request-1" };
const invoice = { biller: "browser-use-cloud", externalId: "invoice-1", currency: "USD", lines: [
{ externalId: "usage", kind: "inference" as const, amountCents: "15.0000001", occurredAt: new Date().toISOString(), ...linked },
] };
const imported = await service.importInvoice(c.id, invoice, "board");
expect((await service.reconcile(c.id, imported.id)).lines[0]).toMatchObject({ status: "matched", matchedEventId: charge.id });
await service.importInvoice(c.id, invoice, "board");
// A second export, even with a revised price, is evidence for review and
// never a second debit for the same already-recorded purchase.
const revised = await service.importInvoice(c.id, { ...invoice, externalId: "invoice-2", lines: [
{ ...invoice.lines[0], amountCents: "16" },
] }, "board");
expect((await service.reconcile(c.id, revised.id)).lines[0].status).toBe("difference");
expect(await finance.summary(c.id)).toMatchObject({ netCentsExact: "15.0000001", eventCount: 1 });
expect(await finance.list(c.id)).toHaveLength(1);
});
it("links a uniquely matched invoice debit and serializes overlapping exports", async () => {
const c = await company();
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
const charge = await costService(db).createEvent(c.id, { agentId: agent.id,
provider: "test", model: "test", providerRequestId: "request", costCents: 15, occurredAt: new Date() });
const service = billingReconciliationService(db);
await Promise.all(["first-export", "second-export"].map(externalId => service.importInvoice(c.id, {
biller: "test", externalId, currency: "USD", lines: [{ externalId: "usage", amountCents: "15",
providerRequestId: "request", model: "test", occurredAt: new Date().toISOString() }],
}, "board")));
const rows = await financeService(db).list(c.id);
expect(rows).toHaveLength(1);
expect(rows[0].costEventId).toBe(charge.id);
expect((await financeService(db).summary(c.id)).netCentsExact).toBe("15.0000000");
});
it.each(["fee", "credit", "other currency", "other biller", "prior credit", "provider report"] as const)("preserves distinct invoice entries: %s", async (scenario) => {
const c = await company();
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
const charge = await costService(db).createEvent(c.id, { agentId: agent.id,
provider: "test", model: "test", costCents: 15, occurredAt: new Date() });
await financeService(db).createEvent(c.id, { costEventId: charge.id, biller: "test",
eventKind: "inference_charge", direction: scenario === "prior credit" ? "credit" : "debit",
amountCents: 15, occurredAt: new Date(),
metadataJson: scenario === "provider report" ? { source: "provider_cost_report" } : undefined });
const imported = await billingReconciliationService(db).importInvoice(c.id, {
biller: scenario === "other biller" ? "other" : "test", externalId: "invoice",
currency: scenario === "other currency" ? "EUR" : "USD",
lines: [{ externalId: "line", costEventId: charge.id, amountCents: "2",
kind: scenario === "fee" || scenario === "credit" ? scenario : "inference", occurredAt: new Date().toISOString() }],
}, "board");
const rows = await financeService(db).list(c.id);
expect(rows).toHaveLength(2);
expect(rows.find(row => row.metadataJson?.invoiceId === imported.id)).toMatchObject({
amountCentsExact: "2.0000000", currency: scenario === "other currency" ? "EUR" : "USD",
direction: scenario === "credit" ? "credit" : "debit",
});
});
it("retains ambiguous run evidence without guessing an additional debit", async () => {
const c = await company();
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: c.id, agentId: agent.id, invocationSource: "on_demand" }).returning();
for (const costCents of [5, 10]) await costService(db).createEvent(c.id, {
agentId: agent.id, heartbeatRunId: run.id, provider: "test", model: "test", costCents, occurredAt: new Date(),
});
const service = billingReconciliationService(db);
const imported = await service.importInvoice(c.id, { biller: "test", externalId: "invoice", currency: "USD",
lines: [{ externalId: "usage", runId: run.id, amountCents: "15", occurredAt: new Date().toISOString() }],
}, "board");
expect((await service.reconcile(c.id, imported.id)).lines[0].status).toBe("ambiguous");
expect(await financeService(db).list(c.id)).toHaveLength(0);
expect((await costService(db).summary(c.id)).spendCentsExact).toBe("15.0000000");
});
it("does not select a price when multiple invoice lines claim the same charge", async () => {
const c = await company();
const [agent] = await db.insert(agents).values({ companyId: c.id, name: "Worker" }).returning();
const charge = await costService(db).createEvent(c.id, { agentId: agent.id,
provider: "test", model: "test", costCents: 15, occurredAt: new Date() });
const service = billingReconciliationService(db);
const imported = await service.importInvoice(c.id, { biller: "test", externalId: "invoice", currency: "USD",
lines: [15, 16].map(amount => ({ externalId: `line-${amount}`, costEventId: charge.id,
amountCents: String(amount), occurredAt: new Date().toISOString() })),
}, "board");
expect((await service.reconcile(c.id, imported.id)).lines.map(line => line.status)).toEqual(["ambiguous", "ambiguous"]);
expect(await financeService(db).list(c.id)).toHaveLength(0);
});
it("records provider report corrections as deltas, including decreases and returns to an earlier total", async () => {
const c = await company();
const input: ImportProviderCosts = {
provider: "openai",
accountId: "org_test",
secretId: randomUUID(),
scopeIds: ["project"],
from: "2026-09-01",
to: "2026-09-02",
};
const update = (amountCents: string) =>
applyProviderDailyCosts(
db,
c.id,
input,
[{ day: "2026-09-01", scopeId: "project", amountCents }],
"board",
);
await update("100");
await update("100");
await update("120");
await update("90");
await update("100");
const rows = await financeService(db).list(c.id, {
from: new Date("2026-09-01"),
to: new Date("2026-09-30"),
});
expect(rows).toHaveLength(4);
expect(rows.filter((row) => row.direction === "credit")).toHaveLength(1);
const summary = await financeService(db).summary(c.id, { allTime: true });
expect(summary).toMatchObject({
netCentsExact: "0.0000000",
providerReportedCentsExact: "100.0000000",
});
expect(await financeService(db).byBiller(c.id, { allTime: true })).toEqual(
[],
);
expect(await financeService(db).byKind(c.id, { allTime: true })).toEqual(
[],
);
const [snapshot] = await db
.select()
.from(providerBillingSnapshots)
.where(eq(providerBillingSnapshots.companyId, c.id));
expect(snapshot.amountCents).toBe("100.0000000");
expect(snapshot.revision).toBe(4);
await expect(
applyProviderDailyCosts(
db,
c.id,
input,
[{ day: "2026-09-01", scopeId: "project", amountCents: "80" }],
"board",
new Map([["2026-09-01:project", 0]]),
),
).rejects.toThrow("newer provider report");
const other = await company();
await applyProviderDailyCosts(
db,
other.id,
input,
[{ day: "2026-09-01", scopeId: "project", amountCents: "1" }],
"board",
);
expect(
await db
.select()
.from(financeEvents)
.where(eq(financeEvents.companyId, c.id)),
).toHaveLength(4);
});
it.each([
null,
{},
{ providerBilling: "openai" },
{ providerBilling: { provider: "anthropic", accountId: "org_test" } },
{ providerBilling: { provider: "openai", accountId: "org_other" } },
{ providerBilling: { provider: "openai" } },
])("never reads or transmits a secret without the matching billing designation: %j", async (providerMetadata) => {
const c = await company();
const [secret] = await db.insert(companySecrets).values({
companyId: c.id, key: "unrelated", name: "Unrelated credential", providerMetadata,
}).returning();
const resolveSecretValue = vi.fn().mockResolvedValue("unrelated-private-key");
vi.spyOn(secretsModule, "secretService").mockReturnValue({ resolveSecretValue } as unknown as ReturnType<typeof secretsModule.secretService>);
const fetcher = vi.fn<typeof fetch>().mockResolvedValue(Response.json({ data: [], has_more: false }));
await expect(importProviderDailyCosts(db, c.id, {
provider: "openai", accountId: "org_test", secretId: secret.id,
scopeIds: ["project"], from: "2026-09-01", to: "2026-09-02",
}, "board", fetcher)).rejects.toThrow("designated for this billing provider and account");
expect(resolveSecretValue).not.toHaveBeenCalled();
expect(fetcher).not.toHaveBeenCalled();
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, c.id))).toEqual([]);
});
it("fetches with a company credential and keeps provider reports separate from invoice charges", async () => {
const c = await company();
const [secret] = await db
.insert(companySecrets)
.values({ companyId: c.id, key: "billing", name: "Billing admin", providerMetadata: { providerBilling: { provider: "openai", accountId: "org_test" } } })
.returning();
const resolveSecretValue = vi.fn().mockResolvedValue("fixture-admin-key");
vi.spyOn(secretsModule, "secretService").mockReturnValue({
resolveSecretValue,
} as unknown as ReturnType<typeof secretsModule.secretService>);
const fetcher = vi.fn<typeof fetch>().mockImplementation(async () =>
Response.json({
data: [
{
start_time: Date.parse("2026-09-01") / 1000,
end_time: Date.parse("2026-09-02") / 1000,
results: [
{
project_id: "project",
amount: { value: 0.06, currency: "usd" },
},
],
},
],
has_more: false,
}),
);
const input: ImportProviderCosts = {
provider: "openai",
accountId: "org_test",
secretId: secret.id,
scopeIds: ["project"],
from: "2026-09-01",
to: "2026-09-02",
};
expect(
await importProviderDailyCosts(db, c.id, input, "board", fetcher),
).toEqual({ daysRead: 1, eventsCreated: 1 });
expect(resolveSecretValue).toHaveBeenCalledWith(
c.id,
secret.id,
"latest",
expect.objectContaining({
accessContext: expect.objectContaining({
actorId: "board",
consumerId: "provider-billing-import",
}),
}),
);
expect(fetcher.mock.calls[0][1]?.headers).toMatchObject({
Authorization: "Bearer fixture-admin-key",
"OpenAI-Organization": "org_test",
});
expect(
await importProviderDailyCosts(db, c.id, input, "board", fetcher),
).toEqual({ daysRead: 1, eventsCreated: 0 });
await billingReconciliationService(db).importInvoice(
c.id,
{
biller: "openai",
externalId: "separate-invoice",
currency: "USD",
lines: [
{
externalId: "usage",
amountCents: "6",
kind: "inference",
occurredAt: "2026-09-01T00:00:00Z",
},
],
},
"board",
);
expect(
await financeService(db).summary(c.id, { allTime: true }),
).toMatchObject({
netCentsExact: "6.0000000",
providerReportedCentsExact: "6.0000000",
eventCount: 2,
});
expect(await financeService(db).byBiller(c.id, { allTime: true })).toEqual([
expect.objectContaining({
biller: "openai",
netCentsExact: "6.0000000",
eventCount: 1,
}),
]);
const other = await company();
await expect(
importProviderDailyCosts(db, other.id, input, "board", fetcher),
).rejects.toThrow("company billing credential");
expect(resolveSecretValue).toHaveBeenCalledTimes(2);
});
it("rejects a slow report when another importer changes the same provider snapshot", async () => {
const c = await company();
const [secret] = await db
.insert(companySecrets)
.values({ companyId: c.id, key: "billing-race", name: "Billing admin", providerMetadata: { providerBilling: { provider: "openai", accountId: "org_test" } } })
.returning();
vi.spyOn(secretsModule, "secretService").mockReturnValue({
resolveSecretValue: vi.fn().mockResolvedValue("fixture-admin-key"),
} as unknown as ReturnType<typeof secretsModule.secretService>);
const input: ImportProviderCosts = {
provider: "openai",
accountId: "org_test",
secretId: secret.id,
scopeIds: ["project"],
from: "2026-09-01",
to: "2026-09-02",
};
const report = (value: number) =>
Response.json({
data: [
{
start_time: Date.parse(input.from) / 1000,
end_time: Date.parse(input.to) / 1000,
results: [
{ project_id: "project", amount: { currency: "usd", value } },
],
},
],
has_more: false,
});
let release!: (response: Response) => void;
let markStarted!: () => void;
const slowResponse = new Promise<Response>((resolve) => {
release = resolve;
});
const started = new Promise<void>((resolve) => {
markStarted = resolve;
});
const slowImport = importProviderDailyCosts(
db,
c.id,
input,
"board",
async () => {
markStarted();
return slowResponse;
},
);
await started;
await importProviderDailyCosts(db, c.id, input, "board", async () =>
report(2),
);
release(report(1));
await expect(slowImport).rejects.toThrow("newer provider report");
expect(
await financeService(db).summary(c.id, { allTime: true }),
).toMatchObject({
providerReportedCentsExact: "200.0000000",
eventCount: 1,
});
});
it("rolls back an entire provider import when a later snapshot is invalid", async () => {
const c = await company();
const input: ImportProviderCosts = {
provider: "openai",
accountId: "org_test",
secretId: randomUUID(),
scopeIds: ["project"],
from: "2026-09-01",
to: "2026-09-03",
};
await expect(
applyProviderDailyCosts(
db,
c.id,
input,
[
{ day: "2026-09-01", scopeId: "project", amountCents: "5" },
{ day: "2026-09-02", scopeId: "project", amountCents: "-1" },
],
"board",
),
).rejects.toThrow();
expect(
await db
.select()
.from(providerBillingSnapshots)
.where(eq(providerBillingSnapshots.companyId, c.id)),
).toEqual([]);
expect(
await db
.select()
.from(financeEvents)
.where(eq(financeEvents.companyId, c.id)),
).toEqual([]);
});
it("stores an explicit zero report and credits an amended report down to zero only once", async () => {
const c = await company();
const input: ImportProviderCosts = {
provider: "anthropic",
accountId: "org_test",
secretId: randomUUID(),
scopeIds: ["default"],
from: "2026-09-01",
to: "2026-09-02",
};
const update = (amountCents: string) =>
applyProviderDailyCosts(
db,
c.id,
input,
[{ day: input.from, scopeId: "default", amountCents }],
"board",
);
expect((await update("0")).eventsCreated).toBe(0);
expect((await update("0")).eventsCreated).toBe(0);
expect((await update("1.0000001")).eventsCreated).toBe(1);
expect((await update("0")).eventsCreated).toBe(1);
expect((await update("0")).eventsCreated).toBe(0);
expect(
await financeService(db).summary(c.id, { allTime: true }),
).toMatchObject({
providerReportedCentsExact: "0.0000000",
netCentsExact: "0.0000000",
eventCount: 2,
});
});
it("freezes estimated dollars before finalization and persists them exactly once into costs and budgets", async () => {
const c = await company();
const budgets = budgetService(db);
await budgets.upsertPolicy(c.id, { scopeType: "company", scopeId: c.id, amount: 100 }, "board");
const [agent] = await db
.insert(agents)
.values({
companyId: c.id,
name: "Codie",
adapterType: "paperclip_runner",
})
.returning();
const [run] = await db
.insert(heartbeatRuns)
.values({
companyId: c.id,
agentId: agent.id,
invocationSource: "on_demand",
status: "running",
})
.returning();
const recorder = await createRunUsageRecorder(
db,
{ companyId: c.id, runId: run.id, adapterType: "paperclip_runner" },
spool,
);
const captured = await recorder.capture({
provider: "openai",
biller: "openai",
billingType: "metered_api",
model: "gpt-6-astra",
usageBasis: "per_run",
complete: true,
costUsd: null,
usage: {
inputTokens: 123536,
cachedInputTokens: 1567209,
outputTokens: 6625,
},
});
expect(captured.costUsdExact).toBe("3.133819000");
await db
.update(heartbeatRuns)
.set({ status: "failed", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, run.id));
await accountRunCost(db, run.id);
await accountRunCost(db, run.id);
const [event] = await db
.select()
.from(costEvents)
.where(eq(costEvents.heartbeatRunId, run.id));
expect(event.costStatus).toBe("estimated");
expect(event.pricingProvenance).toMatchObject({
source: "rate_card",
version: "openai-standard-2026-09-30",
});
const summary = await costService(db).summary(c.id);
expect(summary.spendCentsExact).toBe("313.3819000");
expect(summary.estimatedEventCount).toBe(1);
expect(await budgets.getInvocationBlock(c.id, agent.id)).not.toBeNull();
});
it("preserves cache-write evidence and pricing across multiple provider attempts", async () => {
const c = await company();
const [agent] = await db
.insert(agents)
.values({
companyId: c.id,
name: "Retry",
adapterType: "paperclip_runner",
})
.returning();
const [run] = await db
.insert(heartbeatRuns)
.values({ companyId: c.id, agentId: agent.id, status: "running" })
.returning();
const recorder = await createRunUsageRecorder(
db,
{ companyId: c.id, runId: run.id, adapterType: "paperclip_runner" },
spool,
);
await recorder.capture({
attemptId: randomUUID(),
complete: true,
provider: "openai",
biller: "openai",
billingType: "api",
model: "gpt-6-astra",
usageBasis: "per_run",
usage: { inputTokens: 10, cacheWriteTokens: 4, outputTokens: 2 },
});
const result = await recorder.capture({
attemptId: randomUUID(),
complete: true,
provider: "openai",
biller: "openai",
billingType: "api",
costUsd: 2,
});
expect(result).toMatchObject({
complete: true,
costUsdExact: "2.000210000",
costStatus: "estimated",
pricingProvenance: { source: "rate_card", version: "per-attempt/v1" },
usage: {
inputTokens: 10,
cacheWriteTokens: 4,
cachedInputTokens: 0,
outputTokens: 2,
},
});
await db
.update(heartbeatRuns)
.set({ status: "succeeded", finishedAt: new Date() })
.where(eq(heartbeatRuns.id, run.id));
await accountRunCost(db, run.id);
expect(await costService(db).summary(c.id)).toMatchObject({
spendCentsExact: "200.0210000",
estimatedEventCount: 1,
});
});
it("recovers older durable receipts without pricing metadata while preserving explicit zero", async () => {
const c = await company();
const sourceId = randomUUID();
const [agent] = await db
.insert(agents)
.values({
companyId: c.id,
name: "Legacy receipt",
adapterType: "process",
})
.returning();
const [run] = await db
.insert(heartbeatRuns)
.values({
companyId: c.id,
agentId: agent.id,
status: "running",
usageJson: { accountingReceiptSourceId: sourceId },
})
.returning();
const cases = [
{ costUsd: null },
{ costUsd: 0 },
{ costUsdExact: "0.000000001" },
];
for (const [index, receipt] of cases.entries()) {
await persistUsageReceipt(db, {
schema: "paperclip/accounting-receipt/v1",
id: randomUUID(),
companyId: c.id,
runId: run.id,
adapterType: "process",
sourceId,
sequence: index + 1,
receivedAt: new Date().toISOString(),
receipt: { ...receipt, complete: true },
});
const [stored] = await db
.select()
.from(heartbeatRuns)
.where(eq(heartbeatRuns.id, run.id));
expect(stored.usageJson?.pricingProvenance).toEqual({
source: index === 0 ? "unknown" : "provider_reported",
version: "accounting-receipt/v1",
});
}
});
});
@@ -0,0 +1,113 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { and, eq } from "drizzle-orm";
import { agents, agentRuntimeState, approvals, budgetIncidents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, projects } from "@paperclipai/db";
import { costService } from "../services/costs.js";
import { budgetService, type BudgetServiceHooks } from "../services/budgets.js";
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
function random(seed: number) {
return () => { seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0; return seed / 2 ** 32; };
}
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("seeded concurrent accounting invariants", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-interleavings-"); db = createDb(database.connectionString); }, 30_000);
afterAll(async () => { await database?.cleanup(); });
it.each([1, 7, 42, 97, 31337, 65537, 104729, 0xdeadbeef])("conserves the ledger, totals, incidents and delivery state for seed %s", async (seed) => {
const rng = random(seed);
const [company] = await db.insert(companies).values({ name: `Interleaving ${seed}`, issuePrefix: `R${randomUUID().slice(0,7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "idle", adapterType: "process" }).returning();
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project" }).returning();
const oracle = new Map<string, { ticks: bigint; input: number; cached: number; output: number; run: boolean }>();
const operations: Array<() => Promise<unknown>> = [];
let cancellationAttempts = 0;
let successfulDeliveries = 0;
const hooks: BudgetServiceHooks = { cancelWorkForScope: async (scope) => {
expect(scope.companyId).toBe(company.id);
expect(scope.createdBefore).toBeInstanceOf(Date);
cancellationAttempts++;
if (cancellationAttempts % 4 === 1) throw new Error(`Injected delivery interruption (seed ${seed})`);
successfulDeliveries++;
} };
const budgets = budgetService(db, hooks);
const costs = costService(db, hooks);
const now = new Date();
for (let index = 0; index < 12; index++) {
const ticks = Math.floor(rng() * 9_000_000) + 1;
const input = Math.floor(rng() * 300), cached = Math.floor(rng() * 500), output = Math.floor(rng() * 80);
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id,
status: ["succeeded", "failed", "cancelled", "timed_out"][index % 4], finishedAt: now, costAccountingPending: true,
usageJson: { accountingReceiptReady: true, provider: "fixture", model: `model-${index % 3}`, billingType: "metered_api", inputTokens: input, cachedInputTokens: cached, outputTokens: output, costUsd: ticks / 1e9, ledgerScope: { projectId: project.id } },
}).returning();
oracle.set(`heartbeat:${run.id}:final`, { ticks: BigInt(ticks), input, cached, output, run: true });
operations.push(() => accountRunCost(db, run.id, hooks), () => accountRunCost(db, run.id, hooks));
}
for (let index = 0; index < 10; index++) {
const ticks = Math.floor(rng() * 3_000_000) + 1;
const key = `manual-${index}`;
const receipt = { idempotencyKey: key, agentId: agent.id, projectId: project.id, provider: "fixture", model: "manual", billingType: "metered_api", costCents: ticks / 1e7, occurredAt: now };
oracle.set(key, { ticks: BigInt(ticks), input: 0, cached: 0, output: 0, run: false });
operations.push(() => costs.createEvent(company.id, receipt), () => costs.createEvent(company.id, receipt));
}
for (let index = 0; index < 12; index++) {
const amount = rng() < 0.5 ? 1 : 200;
const scopeType = index % 2 ? "agent" as const : "project" as const;
operations.push(() => budgets.upsertPolicy(company.id, { scopeType, scopeId: scopeType === "agent" ? agent.id : project.id, amount, notifyEnabled: false }, "board"));
operations.push(() => budgets.getInvocationBlock(company.id, agent.id, { projectId: project.id }));
operations.push(() => reconcileRunCosts(db, hooks));
}
// Reproducible submission order and jitter; the database is free to choose
// the actual interleaving. The oracle is independent of execution order.
for (let i = operations.length - 1; i > 0; i--) {
const j = Math.floor(rng() * (i + 1)); [operations[i], operations[j]] = [operations[j], operations[i]];
}
for (let offset = 0; offset < operations.length; offset += 8) {
await Promise.all(operations.slice(offset, offset + 8).map(async operation => {
await new Promise(resolve => setTimeout(resolve, Math.floor(rng() * 5)));
return operation();
}));
const incidents = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, company.id), eq(budgetIncidents.status, "open")));
expect(new Set(incidents.map(row => `${row.policyId}:${row.windowStart.toISOString()}:${row.thresholdType}`)).size).toBe(incidents.length);
}
await reconcileRunCosts(db, hooks);
const rows = await db.select().from(costEvents).where(eq(costEvents.companyId, company.id));
expect(rows).toHaveLength(oracle.size);
for (const row of rows) {
const expected = oracle.get(row.idempotencyKey!);
expect(expected, `unexpected receipt ${row.idempotencyKey}; seed ${seed}`).toBeDefined();
expect(BigInt(Math.round(row.costCents * 1e7))).toBe(expected!.ticks);
expect([row.inputTokens, row.cachedInputTokens, row.outputTokens]).toEqual([expected!.input, expected!.cached, expected!.output]);
}
const total = [...oracle.values()].reduce((sum, row) => sum + row.ticks, 0n);
expect(BigInt(Math.round((await costs.summary(company.id)).spendCents * 1e7))).toBe(total);
expect(BigInt(Math.round((await costs.byProject(company.id))[0].costCents * 1e7))).toBe(total);
const [companyProjection] = await db.select().from(companies).where(eq(companies.id, company.id));
const [agentProjection] = await db.select().from(agents).where(eq(agents.id, agent.id));
expect(BigInt(Math.round(companyProjection.spentMonthlyCents * 1e7))).toBe(total);
expect(BigInt(Math.round(agentProjection.spentMonthlyCents * 1e7))).toBe(total);
const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, agent.id));
const runs = [...oracle.values()].filter(row => row.run);
expect(BigInt(Math.round(runtime.totalCostCents * 1e7))).toBe(runs.reduce((sum, row) => sum + row.ticks, 0n));
expect([runtime.totalInputTokens,runtime.totalCachedInputTokens,runtime.totalOutputTokens])
.toEqual(["input","cached","output"].map(key => runs.reduce((sum,row) => sum + Number(row[key as "input" | "cached" | "output"]),0)));
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, company.id))).every(run => !run.costAccountingPending && run.costAccountedAt !== null)).toBe(true);
// Finish in a known hard-stop state, deliver failed effects, then grant more
// budget. This checks the final observable state, not only receipt counts.
await budgets.upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 1, notifyEnabled: false }, "board");
expect(await budgets.getInvocationBlock(company.id, agent.id)).toMatchObject({ scopeType: "agent" });
for (let pass = 0; pass < 4; pass++) await budgets.deliverPendingEnforcement(company.id);
expect(successfulDeliveries).toBeGreaterThan(0);
const policies = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, company.id));
expect(policies.every(row => row.enforcementVersion === row.enforcementDeliveredVersion)).toBe(true);
const allIncidents = await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, company.id));
const approvalRows = await db.select().from(approvals).where(eq(approvals.companyId, company.id));
expect(new Set(allIncidents.map(row => row.approvalId).filter(Boolean)).size).toBe(approvalRows.length);
await budgets.upsertPolicy(company.id, { scopeType: "agent", scopeId: agent.id, amount: 200 }, "board");
await budgets.upsertPolicy(company.id, { scopeType: "project", scopeId: project.id, amount: 200 }, "board");
expect(await budgets.getInvocationBlock(company.id, agent.id, { projectId: project.id })).toBeNull();
}, 60_000);
});
@@ -0,0 +1,126 @@
import { spawn, type ChildProcess } from "node:child_process";
import { randomUUID, randomInt } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { createServer } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import { agents, companies, costEvents, heartbeatRuns, budgetPolicies, createDb } from "@paperclipai/db";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
const root = fileURLToPath(new URL("../../..", import.meta.url));
async function until<T>(read: () => Promise<T>, accept: (value: T) => boolean, diagnostic: () => string = () => "", timeout = 40_000) {
const deadline = Date.now() + timeout;
while (Date.now() < deadline) {
const value = await read();
if (accept(value)) return value;
await new Promise(resolve => setTimeout(resolve, 30));
}
throw new Error(`Accounting process checkpoint timed out. ${diagnostic()}`);
}
(support.supported && process.platform !== "win32" ? describe : describe.skip)("accounting survives real server SIGKILL", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-crash-"); }, 30_000);
afterAll(async () => { await database?.cleanup(); });
it.each(["before_receipt", "before_runtime_totals", "before_commit", "after_commit_before_delivery_ack"] as const)("recovers exactly once after %s", async (phase) => {
const db = createDb(database.connectionString);
const observer = createDb(database.connectionString).$client;
const blocker = await createDb(database.connectionString).$client.reserve();
const home = await mkdtemp(path.join(tmpdir(), "pc-accounting-kill-"));
const key = randomInt(1, 2_000_000_000);
const [company] = await db.insert(companies).values({ name: "Crash accounting", issuePrefix: `K${randomUUID().slice(0,7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "idle", adapterType: "process", runtimeConfig: { heartbeat: { enabled: false } } }).returning();
const [policy] = await db.insert(budgetPolicies).values({ companyId: company.id, scopeType: "agent", scopeId: agent.id, metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1, notifyEnabled: false }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "failed", finishedAt: new Date(), costAccountingPending: true,
usageJson: { accountingReceiptReady: true, provider: "fixture", model: "fixture", billingType: "metered_api", inputTokens: 7, cachedInputTokens: 11, outputTokens: 3, costUsd: 0.0125 },
}).returning();
const table = phase === "before_receipt" ? "cost_events" : phase === "before_runtime_totals" ? "agent_runtime_state"
: phase === "before_commit" ? "heartbeat_runs" : "budget_policies";
const event = phase === "after_commit_before_delivery_ack" || phase === "before_commit" ? "UPDATE" : "INSERT";
const condition = phase === "after_commit_before_delivery_ack" ? "AND NEW.enforcement_delivered_version > OLD.enforcement_delivered_version"
: phase === "before_commit" ? "AND OLD.cost_accounting_pending = false AND NEW.cost_accounted_at IS NOT NULL" : "";
let child: ChildProcess | undefined;
let logs = "";
async function kill() {
if (!child || child.exitCode !== null || child.signalCode !== null) return;
const stopped = new Promise<void>((resolve) => child!.once("exit", () => resolve()));
process.kill(-child.pid!, "SIGKILL");
await stopped;
}
async function start() {
const port = await new Promise<number>((resolve, reject) => {
const listener = createServer(); listener.on("error", reject);
listener.listen(0, "127.0.0.1", () => { const address = listener.address() as { port: number }; listener.close(() => resolve(address.port)); });
});
logs = "";
child = spawn(process.execPath, ["cli/node_modules/tsx/dist/cli.mjs", "server/src/index.ts"], {
cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"],
env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR,
NODE_ENV: "test", DATABASE_URL: database.connectionString, PORT: String(port),
PAPERCLIP_HOME: home, PAPERCLIP_CONFIG: path.join(home, "config.json"), PAPERCLIP_INSTANCE_ID: "accounting-crash",
PAPERCLIP_DEPLOYMENT_MODE: "local_trusted", PAPERCLIP_BIND: "loopback", SERVE_UI: "false",
PAPERCLIP_OPEN_ON_LISTEN: "false", PAPERCLIP_TELEMETRY_DISABLED: "1", PAPERCLIP_ANNOUNCEMENTS_ENABLED: "false",
HEARTBEAT_SCHEDULER_ENABLED: "false", PAPERCLIP_DECISION_SIGNING_SECRET: "accounting-crash-test-decision-secret",
PAPERCLIP_AGENT_JWT_SECRET: "accounting-crash-test-agent-secret",
},
});
const capture = (data: Buffer) => { logs = (logs + data.toString()).slice(-16000); };
child.stdout!.on("data", capture); child.stderr!.on("data", capture);
return `http://127.0.0.1:${port}`;
}
try {
await blocker`SELECT pg_advisory_lock(${key})`;
await observer.unsafe(`CREATE FUNCTION accounting_crash_barrier() RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN IF NEW.company_id = '${company.id}'::uuid ${condition} THEN PERFORM pg_advisory_lock(${key}); END IF; RETURN NEW; END $$;
CREATE TRIGGER accounting_crash_barrier BEFORE ${event} ON ${table} FOR EACH ROW EXECUTE FUNCTION accounting_crash_barrier()`);
await start();
const waiting = await until(() => {
if (child!.exitCode !== null || child!.signalCode !== null) throw new Error(`Accounting server exited before the barrier: ${logs}`);
return observer`SELECT pid FROM pg_locks WHERE locktype='advisory' AND objid=${key} AND NOT granted`;
}, rows => rows.length > 0, () => logs);
const committed = phase === "after_commit_before_delivery_ack";
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toHaveLength(committed ? 1 : 0);
await kill();
expect(child?.signalCode).toBe("SIGKILL");
// A backend blocked inside our artificial barrier cannot observe its
// dead client until the query returns. Terminate that orphan connection
// before releasing the barrier, forcing PostgreSQL's disconnect rollback.
for (const row of waiting) await observer`SELECT pg_terminate_backend(${row.pid})`;
await until(() => observer`SELECT pid FROM pg_locks WHERE locktype='advisory' AND objid=${key} AND NOT granted`, rows => rows.length === 0);
await blocker`SELECT pg_advisory_unlock(${key})`;
await observer.unsafe(`DROP TRIGGER accounting_crash_barrier ON ${table}; DROP FUNCTION accounting_crash_barrier()`);
const baseUrl = await start();
await until(() => observer`SELECT cost_accounting_pending,cost_accounted_at FROM heartbeat_runs WHERE id=${run.id}`,
rows => rows[0].cost_accounting_pending === false && rows[0].cost_accounted_at !== null, () => logs);
await until(() => observer`SELECT enforcement_version,enforcement_delivered_version FROM budget_policies WHERE id=${policy.id}`,
rows => rows[0].enforcement_version > 0 && rows[0].enforcement_version === rows[0].enforcement_delivered_version, () => logs);
const assertLedger = async () => {
expect((await observer`SELECT count(*)::int AS count,sum(cost_cents)::text AS cents FROM cost_events WHERE heartbeat_run_id=${run.id}`)[0]).toEqual({ count: 1, cents: "1.2500000" });
expect((await observer`SELECT total_input_tokens,total_cached_input_tokens,total_output_tokens,total_cost_cents::text FROM agent_runtime_state WHERE agent_id=${agent.id}`)[0])
.toEqual({ total_input_tokens: "7", total_cached_input_tokens: "11", total_output_tokens: "3", total_cost_cents: "1.2500000" });
expect((await observer`SELECT spent_monthly_cents::text,status FROM agents WHERE id=${agent.id}`)[0]).toEqual({ spent_monthly_cents: "1.2500000", status: "paused" });
expect((await observer`SELECT count(*)::int AS count FROM budget_incidents WHERE policy_id=${policy.id}`)[0].count).toBe(1);
expect((await observer`SELECT count(*)::int AS count FROM approvals WHERE company_id=${company.id}`)[0].count).toBe(1);
};
await assertLedger();
await until(async () => fetch(`${baseUrl}/api/health`).then(r => r.ok).catch(() => false), Boolean, () => logs);
await kill();
const replayUrl = await start();
await until(async () => fetch(`${replayUrl}/api/health`).then(r => r.ok).catch(() => false), Boolean, () => logs);
await until(async () => logs.includes(`Server startup recovery complete on 127.0.0.1:${new URL(replayUrl).port}`), Boolean, () => logs);
await assertLedger();
} finally {
await kill();
await blocker`SELECT pg_advisory_unlock_all()`;
blocker.release();
await observer.unsafe(`DROP TRIGGER IF EXISTS accounting_crash_barrier ON ${table}; DROP FUNCTION IF EXISTS accounting_crash_barrier()`);
await observer.end();
await rm(home, { recursive: true, force: true });
}
}, 150_000);
});
@@ -0,0 +1,241 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { eq } from "drizzle-orm";
import { activityLog, agents, companies, costEvents, createDb, goals, heartbeatRuns, issues, projects } from "@paperclipai/db";
import { costService } from "../services/costs.js";
import { financeService } from "../services/finance.js";
import { budgetService } from "../services/budgets.js";
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
import { receiptFingerprint } from "../services/receipt-fingerprint.js";
import { subscribeCompanyLiveEvents } from "../services/live-events.js";
import { logger } from "../middleware/logger.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("accounting validation and recovery edges (PostgreSQL)", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-edges-"); db = createDb(database.connectionString); }, 30_000);
afterAll(async () => { await database?.cleanup(); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Accounting edges", issuePrefix: `E${randomUUID().slice(0, 7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project" }).returning();
const [issue] = await db.insert(issues).values({ companyId: company.id, title: "Work", projectId: project.id }).returning();
const [goal] = await db.insert(goals).values({ companyId: company.id, title: "Goal" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id }).returning();
const receipt = { agentId: agent.id, provider: "fixture", model: "fixture", billingType: "metered_api", costCents: 0.1234567, occurredAt: new Date() };
const cost = await costService(db).createEvent(company.id, receipt);
const finance = { eventKind: "inference_charge", biller: "fixture", amountCents: 123, occurredAt: new Date() };
return { company, agent, project, issue, goal, run, cost, receipt, finance,
links: { agentId: agent.id, projectId: project.id, issueId: issue.id, goalId: goal.id, heartbeatRunId: run.id, costEventId: cost.id } };
}
it.each(["agentId", "projectId", "issueId", "goalId", "heartbeatRunId", "costEventId"] as const)("rejects foreign and missing finance %s, accepts company-owned links", async field => {
const a = await fixture(), b = await fixture();
const service = financeService(db);
await expect(service.createEvent(a.company.id, { ...a.finance, [field]: b.links[field] })).rejects.toMatchObject({ status: 422 });
await expect(service.createEvent(a.company.id, { ...a.finance, [field]: randomUUID() })).rejects.toMatchObject({ status: 404 });
expect(await service.list(a.company.id)).toHaveLength(0);
const event = await service.createEvent(a.company.id, { ...a.finance, ...a.links }, { actorType: "user", actorId: "board", agentId: a.agent.id });
expect(event[field]).toBe(a.links[field]);
expect(await service.list(a.company.id)).toEqual([event]);
});
it("normalizes finance retries and rolls back changed keys and invalid amounts/currencies", async () => {
const f = await fixture(), service = financeService(db);
const receipt = { ...f.finance, idempotencyKey: "invoice", currency: "usd", metadataJson: { entries: [{ z: 2, a: 1 }, null] } };
const event = await service.createEvent(f.company.id, receipt);
expect((await service.createEvent(f.company.id, { ...receipt, currency: "USD", metadataJson: { entries: [{ a: 1, z: 2 }, null] } })).id).toBe(event.id);
await expect(service.createEvent(f.company.id, { ...receipt, amountCents: 124 })).rejects.toMatchObject({ status: 409 });
for (const invalid of [{ amountCents: -1 }, { amountCents: "1.2.3" }, { amountCents: Number.POSITIVE_INFINITY }, { currency: "US" }]) {
await expect(service.createEvent(f.company.id, { ...f.finance, ...invalid })).rejects.toMatchObject({ status: 422 });
}
expect(await service.list(f.company.id)).toHaveLength(1);
expect((await db.select().from(activityLog).where(eq(activityLog.companyId, f.company.id))).filter(row => row.action === "finance_event.reported")).toHaveLength(1);
});
it("keeps finance credits, estimates, currencies, dates, and companies separate in every report", async () => {
const f = await fixture(), foreign = await fixture(), service = financeService(db);
const date = new Date("2026-08-15T12:00:00Z");
await service.createEvent(f.company.id, { ...f.finance, occurredAt: date, amountCents: 150, estimated: true });
await service.createEvent(f.company.id, { ...f.finance, occurredAt: new Date(date.getTime() + 1), eventKind: "credit_refund", direction: "credit", amountCents: 40 });
await service.createEvent(f.company.id, { ...f.finance, occurredAt: date, amountCents: 900, currency: "EUR" });
await service.createEvent(f.company.id, { ...f.finance, occurredAt: new Date("2026-08-16T00:00:00Z"), amountCents: 999 });
await service.createEvent(foreign.company.id, { ...foreign.finance, occurredAt: date, amountCents: 999 });
const range = { from: date, to: new Date(date.getTime() + 1) };
expect(await service.summary(f.company.id, range)).toMatchObject({ debitCents: 150, creditCents: 40, estimatedDebitCents: 150, netCents: 110, eventCount: 2,
currencies: [expect.objectContaining({ currency: "EUR", netCents: 900, eventCount: 1 }), expect.objectContaining({ currency: "USD", netCents: 110, eventCount: 2 })] });
expect(await service.byBiller(f.company.id, range)).toEqual([
expect.objectContaining({ biller: "fixture", currency: "EUR", netCents: 900, eventCount: 1, kindCount: 1 }),
expect.objectContaining({ biller: "fixture", currency: "USD", netCents: 110, eventCount: 2, kindCount: 2 }),
]);
expect(await service.byKind(f.company.id, range)).toEqual(expect.arrayContaining([
expect.objectContaining({ eventKind: "inference_charge", currency: "EUR", netCents: 900 }),
expect.objectContaining({ eventKind: "inference_charge", currency: "USD", netCents: 150, estimatedDebitCents: 150 }),
expect.objectContaining({ eventKind: "credit_refund", currency: "USD", netCents: -40, billerCount: 1 }),
]));
expect(await service.list(f.company.id, range, 1)).toEqual([expect.objectContaining({ direction: "credit", amountCents: 40 })]);
expect(await service.summary(randomUUID())).toMatchObject({ currencies: [], debitCents: 0, creditCents: 0, netCents: 0, eventCount: 0 });
});
it("rejects missing companies, missing links, run/agent mismatches, and invalid cost inputs without side effects", async () => {
const f = await fixture(), costs = costService(db);
await expect(costs.createEvent(randomUUID(), f.receipt)).rejects.toMatchObject({ status: 404 });
await expect(costs.summary(randomUUID())).rejects.toMatchObject({ status: 404 });
for (const field of ["agentId", "issueId", "projectId", "goalId", "heartbeatRunId"]) {
await expect(costs.createEvent(f.company.id, { ...f.receipt, [field]: randomUUID() })).rejects.toMatchObject({ status: 404 });
}
const [otherAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning();
await expect(costs.createEvent(f.company.id, { ...f.receipt, agentId: otherAgent.id, heartbeatRunId: f.run.id })).rejects.toThrow("does not belong to agent");
for (const invalid of [{ costCents: -1 }, { costCents: NaN }, { inputTokens: 0.5 }, { cachedInputTokens: -1 }, { outputTokens: 2 ** 31 }]) {
await expect(costs.createEvent(f.company.id, { ...f.receipt, ...invalid })).rejects.toMatchObject({ status: 422 });
}
expect((await costs.summary(f.company.id)).spendCents).toBe(f.receipt.costCents);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
});
it("aggregates billers with fractional cents, cache tokens, and subscription runs without cross-company leakage", async () => {
const f = await fixture(), foreign = await fixture(), costs = costService(db);
const date = new Date("2026-08-15T12:00:00Z");
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: date, biller: "gateway", provider: "a", model: "a", heartbeatRunId: f.run.id, inputTokens: 2, cachedInputTokens: 3, outputTokens: 5 });
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: date, biller: "gateway", provider: "b", model: "b", heartbeatRunId: f.run.id, billingType: "subscription_included", costCents: 0, inputTokens: 7, cachedInputTokens: 11, outputTokens: 13 });
await costs.createEvent(foreign.company.id, { ...foreign.receipt, occurredAt: date, biller: "gateway", costCents: 999 });
expect(await costs.byBiller(f.company.id, { from: date, to: date })).toEqual([{
biller: "gateway", costCents: 0.1234567, costCentsExact: "0.1234567", inputTokens: 9, cachedInputTokens: 14, outputTokens: 18,
apiRunCount: 1, subscriptionRunCount: 1, subscriptionInputTokens: 7, subscriptionCachedInputTokens: 11, subscriptionOutputTokens: 13, providerCount: 2, modelCount: 2,
}]);
});
it("commits a receipt and its audit record even when a live subscriber throws", async () => {
const f = await fixture();
const listener = vi.fn(() => { throw new Error("Injected disconnected live subscriber"); });
const unsubscribe = subscribeCompanyLiveEvents(f.company.id, listener);
const warning = vi.spyOn(logger, "warn").mockImplementation(() => undefined);
try {
const receipt = { ...f.receipt, idempotencyKey: "committed-before-publication" };
const event = await costService(db).createEvent(f.company.id, receipt, { actorType: "user", actorId: "board" });
expect((await costService(db).createEvent(f.company.id, receipt)).id).toBe(event.id);
expect(listener).toHaveBeenCalledTimes(1);
expect(warning).toHaveBeenCalledOnce();
expect((await costService(db).summary(f.company.id)).spendCents).toBe(0.2469134);
expect(await db.select().from(activityLog).where(eq(activityLog.entityId, event.id))).toHaveLength(1);
} finally { unsubscribe(); warning.mockRestore(); }
});
it("reports lifetime history consistently across every cost grouping and finance view", async () => {
const f = await fixture(), costs = costService(db), finance = financeService(db);
const occurredAt = new Date("2000-01-01T00:00:00Z");
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt, costCents: 5, projectId: f.project.id, inputTokens: 2 });
await finance.createEvent(f.company.id, { ...f.finance, occurredAt });
const lifetime = { allTime: true };
for (const read of [costs.byAgent, costs.byAgentModel, costs.byProvider, costs.byBiller, costs.byProject]) {
const rows = await read(f.company.id, lifetime);
expect(rows.reduce((sum, row) => sum + row.costCents, 0)).toBe(5.1234567);
expect(rows.reduce((sum, row) => sum + row.inputTokens, 0)).toBe(2);
}
expect((await costs.summary(f.company.id)).spendCents).toBe(0.1234567);
expect((await costs.summary(f.company.id, lifetime)).spendCents).toBe(5.1234567);
expect((await finance.summary(f.company.id)).eventCount).toBe(0);
expect((await finance.summary(f.company.id, lifetime)).debitCents).toBe(123);
expect(await finance.list(f.company.id, lifetime)).toHaveLength(1);
expect((await finance.byBiller(f.company.id, lifetime))[0].netCents).toBe(123);
expect((await finance.byKind(f.company.id, lifetime))[0].netCents).toBe(123);
await db.update(companies).set({ budgetMonthlyCents: 1 }).where(eq(companies.id, f.company.id));
expect((await costs.summary(f.company.id)).utilizationPercent).toBe(12.35);
});
it.each(["company", "agent", "project"] as const)("rejects nonexistent and foreign %s budget scopes without creating policies", async scopeType => {
const f = await fixture(), foreign = await fixture(), budgets = budgetService(db);
await expect(budgets.upsertPolicy(f.company.id, { scopeType, scopeId: randomUUID(), amount: 10 }, "board")).rejects.toMatchObject({ status: 404 });
await expect(budgets.upsertPolicy(f.company.id, { scopeType, scopeId: foreign[scopeType].id, amount: 10 }, "board")).rejects.toMatchObject({ status: 422 });
expect(await budgets.listPolicies(f.company.id)).toHaveLength(0);
if (scopeType !== "company") {
await expect(budgets.getInvocationBlock(f.company.id, scopeType === "agent" ? foreign.agent.id : f.agent.id,
scopeType === "project" ? { projectId: foreign.project.id } : undefined)).rejects.toMatchObject({ status: 404 });
}
});
it.each(["unpriced", "pending"] as const)("refuses an incident budget raise while accounting is %s, without partial mutations", async kind => {
const f = await fixture(), budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 10 }, "board");
if (kind === "unpriced") {
await costService(db).createEvent(f.company.id, { ...f.receipt, costStatus: "unpriced", costCents: 0 });
} else {
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, finishedAt: new Date(), usageJson: { accountingReceiptReady: false } }).where(eq(heartbeatRuns.id, f.run.id));
await budgets.getInvocationBlock(f.company.id, f.agent.id);
}
const [incident] = (await budgets.overview(f.company.id)).activeIncidents;
const resolution = { action: "raise_budget_and_resume" as const, amount: 100 };
await expect(budgets.resolveIncident(f.company.id, randomUUID(), resolution, "board")).rejects.toMatchObject({ status: 404 });
const foreign = await fixture();
await expect(budgets.resolveIncident(foreign.company.id, incident.id, resolution, "board")).rejects.toMatchObject({ status: 404 });
await expect(budgets.resolveIncident(f.company.id, incident.id, resolution, "board")).rejects.toThrow(kind === "unpriced" ? /unpriced usage/ : /finish accounting/);
expect((await budgets.listPolicies(f.company.id))[0].amount).toBe(10);
expect((await budgets.overview(f.company.id)).activeIncidents).toEqual([expect.objectContaining({ id: incident.id, status: "open" })]);
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("paused");
});
it.each(["company", "project"] as const)("preserves an operator's %s pause even without a policy", async scopeType => {
const f = await fixture();
if (scopeType === "company") await db.update(companies).set({ status: "paused", pauseReason: "manual", pausedAt: new Date() }).where(eq(companies.id, f.company.id));
else await db.update(projects).set({ pauseReason: "manual", pausedAt: new Date() }).where(eq(projects.id, f.project.id));
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id }))
.toMatchObject({ scopeType, reason: expect.stringContaining("paused and cannot start") });
});
it("resolves a company incident and updates the legacy monthly budget together", async () => {
const f = await fixture(), budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2 });
const [incident] = (await budgets.overview(f.company.id)).activeIncidents;
await budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 10 }, "board");
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0]).toMatchObject({ status: "active", pauseReason: null, budgetMonthlyCents: 10 });
expect((await budgets.listPolicies(f.company.id))[0].amount).toBe(10);
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
});
it.each([false, true])("records subscription tokens at zero incremental charge (split=%s)", async split => {
const f = await fixture();
await db.update(heartbeatRuns).set({ status: "succeeded", costAccountingPending: true,
usageJson: { billingType: "subscription_included", costUsd: 1, inputTokens: 3, cachedInputTokens: 5, outputTokens: 7,
...(split ? { usageByModel: [{ model: "a", costUsd: 1, usage: { inputTokens: 3, cachedInputTokens: 5, outputTokens: 7 } }] } : {}),
},
}).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(true);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([
expect.objectContaining({ costCents: 0, costStatus: "reported", inputTokens: 3, cachedInputTokens: 5, outputTokens: 7, billingType: "subscription_included" }),
]);
});
it("rotates a full poisoned recovery batch so the next valid receipt cannot starve", async () => {
const f = await fixture(), foreign = await fixture();
const poison = await db.insert(heartbeatRuns).values(Array.from({ length: 100 }, () => ({
companyId: f.company.id, agentId: f.agent.id, status: "failed", costAccountingPending: true, updatedAt: new Date(0),
usageJson: { accountingReceiptReady: true, costUsd: 0.01, ledgerScope: { projectId: foreign.project.id } },
}))).returning();
const [valid] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "succeeded", costAccountingPending: true,
updatedAt: new Date(1), usageJson: { accountingReceiptReady: true, costUsd: 0.02 },
}).returning();
const error = vi.spyOn(logger, "error").mockImplementation(() => undefined);
try {
expect(await reconcileRunCosts(db)).toEqual({ scanned: 100, accounted: 0 });
expect(await reconcileRunCosts(db)).toEqual({ scanned: 100, accounted: 1 });
expect(error).toHaveBeenCalledTimes(199);
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, valid.id))).toEqual([expect.objectContaining({ costCents: 2 })]);
const rows = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.companyId, f.company.id));
expect(rows.filter(row => row.costAccountingPending)).toHaveLength(100);
expect(rows.filter(row => poison.some(bad => bad.id === row.id)).every(row => row.costAccountedAt === null)).toBe(true);
expect((await costService(db).summary(f.company.id)).spendCents).toBe(2.1234567);
expect(await accountRunCost(db, valid.id)).toBe(false);
expect(await accountRunCost(db, randomUUID())).toBe(false);
} finally { error.mockRestore(); }
}, 30_000);
});
it("fingerprints nested receipts independently of object order while preserving array order and values", () => {
const a = { items: [{ z: 2, a: 1 }, null], at: new Date("2026-01-01T00:00:00Z") };
expect(receiptFingerprint(a)).toBe(receiptFingerprint({ at: a.at, items: [{ a: 1, z: 2 }, null] }));
expect(receiptFingerprint(a)).not.toBe(receiptFingerprint({ ...a, items: [null, { z: 2, a: 1 }] }));
expect(receiptFingerprint(a)).not.toBe(receiptFingerprint({ ...a, items: [{ z: 3, a: 1 }, null] }));
});
@@ -0,0 +1,45 @@
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb } from "@paperclipai/db";
import { costService } from "../services/costs.js";
import { budgetService } from "../services/budgets.js";
import { accountingIntegrityService } from "../services/accounting-integrity.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("accounting mutation sentinels", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-mutation-"); db = createDb(database.connectionString); }, 30_000);
afterAll(async () => { await database?.cleanup(); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Mutation", issuePrefix: `M${randomUUID().slice(0,7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
return { company, agent, receipt: { agentId: agent.id, provider: "fixture", model: "fixture", costCents: "1.0000001", idempotencyKey: randomUUID(), occurredAt: new Date() } };
}
it("deduplicates a retried receipt", async () => {
const f = await fixture(), service = costService(db);
const first = await service.createEvent(f.company.id, f.receipt);
const second = await service.createEvent(f.company.id, f.receipt);
expect(second.id, "ACCOUNTING_ASSERTION deduplication").toBe(first.id);
expect((await service.summary(f.company.id)).eventCount).toBe(1);
});
it("isolates company reporting", async () => {
const f = await fixture(), other = await fixture(), service = costService(db);
await service.createEvent(f.company.id, f.receipt);
await service.createEvent(other.company.id, { ...other.receipt, costCents: 1000 });
expect((await service.summary(f.company.id)).spendCentsExact, "ACCOUNTING_ASSERTION company").toBe("1.0000001");
});
it("conserves agent projections", async () => {
const f = await fixture();
await costService(db).createEvent(f.company.id, f.receipt);
await costService(db).createEvent(f.company.id, { ...f.receipt, idempotencyKey: "second" });
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings, "ACCOUNTING_ASSERTION projection").toEqual([]);
});
it("stops at the exact budget boundary", async () => {
const f = await fixture();
const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1 }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 1 });
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id), "ACCOUNTING_ASSERTION threshold").not.toBeNull();
});
});
@@ -0,0 +1,551 @@
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { and, eq, sql } from "drizzle-orm";
import { accountingRuntimeBaselines, agents, agentRuntimeState, billingInvoiceLines, budgetPolicies, budgetReservations, companies, costEvents, createDb, heartbeatRuns, nativeRunFinalizations, issues, projects, runUsageReceipts } from "@paperclipai/db";
import { accountingIntegrityService } from "../services/accounting-integrity.js";
import { billingReconciliationService } from "../services/billing-reconciliation.js";
import { budgetService, type BudgetEnforcementScope } from "../services/budgets.js";
import { reserveRunBudget } from "../services/budget-reservations.js";
import { costService } from "../services/costs.js";
import { financeService } from "../services/finance.js";
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
import { createRunUsageRecorder, persistUsageReceipt, replayUsageReceipts, spoolUsageReceipt, usageReceiptSpoolPath, type UsageReceiptEnvelope } from "../services/usage-receipts.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("accounting operational edge cases", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: ReturnType<typeof createDb>, directory: string;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-operational-edges-"); db = createDb(database.connectionString); directory = await fs.mkdtemp(path.join(os.tmpdir(), "accounting-edges-")); },30_000);
afterAll(async () => { vi.restoreAllMocks(); await database?.cleanup(); await fs.rm(directory, { recursive: true, force: true }); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Edges", issuePrefix: `E${randomUUID().slice(0,7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Work" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, status: "running", usageJson: {} }).returning();
return { company, agent, project, run };
}
const usage = { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 };
async function event(f: Awaited<ReturnType<typeof fixture>>, extras: Record<string, unknown> = {}) {
return costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "model", costCents: "0.1234567", occurredAt: new Date(), ...extras });
}
it("initializes an old month once, excludes backdated events, and repairs current-month drift", async () => {
const f = await fixture();
await db.update(companies).set({ spendMonthUtc: "2000-01-01", spentMonthlyCents: 999 }).where(eq(companies.id,f.company.id));
await event(f); await event(f, { occurredAt: new Date("2000-01-01") }); await event(f);
const [row] = await db.select().from(companies).where(eq(companies.id,f.company.id));
expect(row.spentMonthlyCents).toBe(0.2469134);
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
await db.update(agents).set({ spentMonthlyCents: 4 }).where(eq(agents.id,f.agent.id));
const review = await accountingIntegrityService(db).inspect(f.company.id);
await accountingIntegrityService(db).repair(f.company.id,review.fingerprint,"Repair drift","board");
await event(f);
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
});
it("keeps fractional finance credits and net totals exact beyond number precision", async () => {
const f = await fixture(), finance = financeService(db);
const base = { biller: "fixture", eventKind: "inference_charge", occurredAt: new Date() };
await finance.createEvent(f.company.id,{ ...base, amountCents: "9007199254740992.0000001" });
await finance.createEvent(f.company.id,{ ...base, direction: "credit", amountCents: "9007199254740992" });
expect((await finance.summary(f.company.id)).netCentsExact).toBe("0.0000001");
expect((await finance.byBiller(f.company.id))[0].netCentsExact).toBe("0.0000001");
expect((await finance.byKind(f.company.id))[0].netCentsExact).toBe("0.0000001");
});
it("checks agent and project capacity, preserves reservations through policy edits, and releases proven bootstrap failures", async () => {
const f = await fixture(), budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id,{ scopeType:"agent",scopeId:f.agent.id,amount:20,reservationCents:"3" },"board");
await budgets.upsertPolicy(f.company.id,{ scopeType:"project",scopeId:f.project.id,windowKind:"lifetime",amount:10,reservationCents:"6" },"board");
await budgets.upsertPolicy(f.company.id,{ scopeType:"company",scopeId:f.company.id,amount:20,reservationCents:"1" },"board");
const reserved = await reserveRunBudget(db,f.company.id,f.run.id,f.project.id);
expect(reserved.amountCents).toBe("6.0000000");
await budgets.upsertPolicy(f.company.id,{ scopeType:"project",scopeId:f.project.id,windowKind:"lifetime",amount:10,reservationCents:"0" },"board");
await budgets.upsertPolicy(f.company.id,{ scopeType:"agent",scopeId:f.agent.id,amount:20,reservationCents:"0" },"board");
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId,f.run.id)))[0].amountCents).toBe("6.0000000");
await db.update(heartbeatRuns).set({ status:"failed",resultJson:{executionRecovery:{providerWorkStarted:false}} }).where(eq(heartbeatRuns.id,f.run.id));
await accountRunCost(db,f.run.id);
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId,f.run.id)))[0].state).toBe("released");
expect(await db.select().from(costEvents).where(eq(costEvents.companyId,f.company.id))).toEqual([]);
});
it.each(["retryable_failure", "observed"])("keeps native %s receipts open through same-run recovery", async (phase) => {
const f = await fixture();
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recoverable native work" }).returning();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
await reserveRunBudget(db, f.company.id, f.run.id, null);
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase });
const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };
const spool = path.join(directory, randomUUID());
const recorder = await createRunUsageRecorder(db, input, spool);
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(false);
await reconcileRunCosts(db);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
const resumed = await createRunUsageRecorder(db, input, spool);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptReady).toBe(false);
const empty = await resumed.complete({ exitCode: 0, signal: null, timedOut: false });
expect(empty).toMatchObject({ complete: false, usage: { inputTokens: 100, outputTokens: 10 } });
// The provider's native delta is cumulative for this run across recovery.
await resumed.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id));
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(true);
expect(await accountRunCost(db, f.run.id)).toBe(false);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, outputTokens: 15, costCents: 15 });
});
it.each([false, true])("recovers pending native receipts beyond the startup batch before replacement (save failure: %s)", async (failSave) => {
const f = await fixture(), other = await fixture();
await db.update(heartbeatRuns).set({ runtimeMode: "native" }).where(eq(heartbeatRuns.id, f.run.id));
await reserveRunBudget(db, f.company.id, f.run.id, null);
const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };
const spool = path.join(directory, randomUUID());
const recorder = await createRunUsageRecorder(db, input, spool);
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
const before = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0];
const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable"));
try {
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 });
} finally { unavailable.mockRestore(); }
const pending = (await fs.readdir(spool)).filter(name => name.endsWith(".json"));
expect(pending).toHaveLength(2);
for (const name of pending) await fs.rename(path.join(spool, name), path.join(spool, `zzz-${name}`));
const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(),
companyId: other.company.id, runId: other.run.id, sourceId: randomUUID(), sequence: 1,
receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
await Promise.all(Array.from({ length: 101 }, async (_, index) => {
await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 }));
}));
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 100, failed: 0 });
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100);
if (failSave) {
await db.execute(sql`create function reject_recovered_receipt() returns trigger language plpgsql as $$ begin raise exception 'receipt save failed'; end $$`);
await db.execute(sql`create trigger reject_recovered_receipt before insert on run_usage_receipts for each row execute function reject_recovered_receipt()`);
try {
await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow();
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId)
.toBe(before.usageJson?.accountingReceiptSourceId);
expect((await fs.readdir(spool)).filter(name => name.startsWith("zzz-"))).toHaveLength(2);
} finally {
await db.execute(sql`drop trigger reject_recovered_receipt on run_usage_receipts`);
await db.execute(sql`drop function reject_recovered_receipt()`);
}
}
const resumed = await createRunUsageRecorder(db, input, spool);
expect((await fs.readdir(spool)).filter(name => name.startsWith("zzz-"))).toHaveLength(0);
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(3);
expect(await resumed.complete({ exitCode: 0, signal: null, timedOut: false }))
.toMatchObject({ complete: false, usage: { inputTokens: 200, outputTokens: 20 } });
await resumed.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 220, outputTokens: 22 }, costUsd: 0.22 });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(true);
expect(await accountRunCost(db, f.run.id)).toBe(false);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0])
.toMatchObject({ inputTokens: 220, outputTokens: 22, costCents: 22 });
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 1, failed: 0 });
expect((await costService(db).summary(f.company.id)).spendCents).toBe(22);
});
it.each(["company", "agent", "project"] as const)("blocks fresh %s work without pausing native recovery", async (scopeType) => {
const f = await fixture(), cancelWorkForScope = vi.fn(async (_scope: BudgetEnforcementScope) => {});
const budgets = budgetService(db, { cancelWorkForScope });
const scopeId = f[scopeType].id;
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board");
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover under budget" }).returning();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure" });
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" }, path.join(directory, randomUUID()));
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
// Also recover a budget-owned pause left by the previous implementation.
const table = { company: companies, agent: agents, project: projects }[scopeType];
await db.update(table).set({ pauseReason: "budget", ...(scopeType === "project" ? {} : { status: "paused" }) }).where(eq(table.id, scopeId));
await reconcileRunCosts(db, { cancelWorkForScope });
expect((await db.select().from(table).where(eq(table.id, scopeId)))[0].pauseReason).toBeNull();
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle");
expect(cancelWorkForScope.mock.calls.filter(([scope]) => scope.companyId === f.company.id)).toEqual([]);
expect((await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id }))?.reason).toContain("native run recovers");
const [fresh] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning();
await expect(reserveRunBudget(db, f.company.id, fresh.id, f.project.id)).rejects.toThrow("native run recovers");
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
await db.update(nativeRunFinalizations).set({ leaseOwner: "recovery", leaseExpiresAt: new Date(Date.now() + 60_000) }).where(eq(nativeRunFinalizations.runId, f.run.id));
expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "recovery"))
.toMatchObject({ id: original.id, reused: true, amountCents: "20.0000000", state: "held" });
expect(await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id))).toHaveLength(1);
await recorder.capture({ complete: true, usageBasis: "per_run", usage: { inputTokens: 150, outputTokens: 15 }, costUsd: 0.15 });
await db.update(nativeRunFinalizations).set({ phase: "terminal_failure" }).where(eq(nativeRunFinalizations.runId, f.run.id));
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(true);
expect(await accountRunCost(db, f.run.id)).toBe(false);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ inputTokens: 150, costCents: 15 });
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id, { projectId: f.project.id })).toBeNull();
});
it.each(["company", "agent", "project"] as const)("rechecks %s stops before native reservation reuse", async scopeType => {
const f = await fixture(), budgets = budgetService(db), scopeId = f[scopeType].id;
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100, reservationCents: "20" }, "board");
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Retry after budget edit" }).returning();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
const original = await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: "retryable_failure", leaseOwner: "retry", leaseExpiresAt: new Date(Date.now() + 60_000) });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
await event(f, { costCents: 50, projectId: f.project.id });
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 40 }, "board");
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
await expect(reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).rejects.toThrow(/budget/);
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 100 }, "board");
expect(await reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).toMatchObject({ id: original.id, reused: true });
const table = { company: companies, agent: agents, project: projects }[scopeType];
await db.update(table).set({ pauseReason: "manual", ...(scopeType === "project" ? { pausedAt: new Date() } : { status: "paused" }) }).where(eq(table.id, scopeId));
await expect(reserveRunBudget(db, f.company.id, f.run.id, f.project.id, {}, "retry")).rejects.toThrow(/paused/);
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0]).toMatchObject({ id: original.id, state: "held", amountCents: "20.0000000" });
});
it.each(["closed", "over_budget", "unpriced", "manual"])("preserves %s stops while native accounting is unfinished", async (stop) => {
const f = await fixture(), budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Pending native run" }).returning();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, f.run.id));
await reserveRunBudget(db, f.company.id, f.run.id, null);
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: f.run.id, issueId: issue.id, phase: stop === "closed" ? "terminal_failure" : "retryable_failure" });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
if (stop === "over_budget") await event(f, { costCents: 100 });
if (stop === "unpriced") await event(f, { costCents: 0, costStatus: "unpriced" });
if (stop === "manual") await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id));
await budgets.reconcilePolicies();
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0])
.toMatchObject({ status: "paused", pauseReason: stop === "manual" ? "manual" : "budget" });
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
});
it.each(["complete", "partial", "receipt_failure", "ledger_failure", "cleanup_failure", "moved", "unstable_spool"])("settles the newest run receipt beyond a full replay batch (%s)", async (scenario) => {
const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath();
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" });
await reserveRunBudget(db, f.company.id, f.run.id, null);
await recorder.capture({ complete: true, usage: { inputTokens: 100, outputTokens: 10 }, costUsd: 0.1 });
const unavailable = vi.spyOn(db, "transaction").mockRejectedValue(new Error("Database unavailable"));
try { await recorder.capture({ complete: scenario !== "partial", usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 }); }
finally { unavailable.mockRestore(); }
const [pending] = (await fs.readdir(spool)).filter(name => name.endsWith(".json"));
const file = path.join(spool, `zzz-${pending}`);
await fs.rename(path.join(spool, pending), file);
const unrelated: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id,
runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
for (let index = 0; index < 101; index++) await fs.writeFile(path.join(spool, `000-${index}.json`), JSON.stringify({ ...unrelated, id: randomUUID(), sequence: index + 1 }));
expect(await replayUsageReceipts(db)).toEqual({ replayed: 100, failed: 0 });
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.inputTokens).toBe(100);
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
if (scenario === "unstable_spool") {
const stat = fs.lstat.bind(fs);
const unstable = vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => {
if (target === file) throw Object.assign(new Error("Concurrent rename"), { code: "ENOENT" });
return stat(target, ...args);
});
try { await expect(accountRunCost(db, f.run.id)).rejects.toThrow("retry required"); }
finally { unstable.mockRestore(); }
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountedAt).toBeNull();
expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200);
}
if (scenario.endsWith("_failure") && scenario !== "cleanup_failure") {
const table = scenario === "receipt_failure" ? "run_usage_receipts" : "cost_events";
await db.execute(sql`create function reject_settlement() returns trigger language plpgsql as $$ begin raise exception 'settlement save failed'; end $$`);
await db.execute(sql.raw(`create trigger reject_settlement before insert on ${table} for each row execute function reject_settlement()`));
try {
await expect(accountRunCost(db, f.run.id)).rejects.toThrow();
expect(JSON.parse(await fs.readFile(file, "utf8")).receipt.usage.inputTokens).toBe(200);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0])
.toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 100 } });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
expect(await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, f.run.id))).toHaveLength(1);
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("held");
} finally {
await db.execute(sql.raw(`drop trigger reject_settlement on ${table}`));
await db.execute(sql`drop function reject_settlement()`);
}
}
const remove = fs.rm.bind(fs), stat = fs.lstat.bind(fs);
let moved = false;
const rm = scenario === "cleanup_failure" ? vi.spyOn(fs, "rm").mockImplementation(async (target, options) => {
if (target === file) throw new Error("Temporary cleanup failure");
return remove(target, options);
}) : null;
const lstat = scenario === "moved" ? vi.spyOn(fs, "lstat").mockImplementation(async (target, ...args) => {
if (target === file && !moved) { moved = true; await fs.rename(file, `${file.slice(0, -5)}-moved.json`); }
return stat(target, ...args);
}) : null;
try { await reconcileRunCosts(db); } finally { rm?.mockRestore(); lstat?.mockRestore(); }
if (scenario === "partial") {
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0])
.toMatchObject({ costAccountedAt: null, usageJson: { inputTokens: 200, accountingReceiptReady: false } });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
await recorder.capture({ complete: true, usage: { inputTokens: 300, outputTokens: 30 }, costUsd: 0.3 });
expect(await accountRunCost(db, f.run.id)).toBe(true);
}
expect(await accountRunCost(db, f.run.id)).toBe(false);
const expected = scenario === "partial" ? 300 : 200;
const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
expect(charges).toHaveLength(1);
expect(charges[0]).toMatchObject({ inputTokens: expected, outputTokens: expected / 10, costCents: expected / 10 });
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("settled");
expect(await replayUsageReceipts(db)).toEqual({ replayed: scenario === "cleanup_failure" ? 2 : 1, failed: 0 });
expect((await costService(db).summary(f.company.id)).spendCents).toBe(expected / 10);
expect(await fs.readdir(spool)).toEqual([]);
});
it("leaves unrelated corrupt spool evidence alone and aborts handoff on unreadable or invalid matching receipts", async () => {
const f = await fixture(), spool = path.join(directory, randomUUID());
const input = { companyId: f.company.id, runId: f.run.id, adapterType: "paperclip_runner" };
await createRunUsageRecorder(db, input, spool);
const source = (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId;
const matching = path.join(spool, "matching.json");
await fs.writeFile(matching, JSON.stringify({ ...input, receipt: "invalid" }));
await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow();
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId).toBe(source);
const read = vi.spyOn(fs, "readFile").mockRejectedValueOnce(Object.assign(new Error("Unreadable receipt"), { code: "EACCES" }));
try { await expect(createRunUsageRecorder(db, input, spool)).rejects.toThrow("Unreadable receipt"); } finally { read.mockRestore(); }
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].usageJson?.accountingReceiptSourceId).toBe(source);
await fs.rm(matching);
const unrelated = ["invalid", "null", "{}", JSON.stringify({ companyId: f.company.id }),
JSON.stringify({ companyId: f.company.id, runId: randomUUID() }), "x".repeat(1024 * 1024 + 1)];
for (const [index, value] of unrelated.entries()) await fs.writeFile(path.join(spool, `${index}.json`), value);
await fs.mkdir(path.join(spool, "directory.json"));
const stat = vi.spyOn(fs, "lstat").mockRejectedValueOnce(Object.assign(new Error("Concurrent replay removed file"), { code: "ENOENT" }));
try { await createRunUsageRecorder(db, input, spool); } finally { stat.mockRestore(); }
await createRunUsageRecorder(db, input, spool);
expect(await fs.readdir(spool)).toHaveLength(unrelated.length + 1);
});
it("indexes a recovery backlog once outside company locks and still sees newly published receipts", async () => {
const f = await fixture(), other = await fixture(), spool = usageReceiptSpoolPath();
const runs = [f.run];
for (let i = 0; i < 7; i++) {
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running" }).returning();
runs.push(run);
}
for (const run of runs) {
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" });
await recorder.capture({ complete: true, usage: { inputTokens: 50, outputTokens: 5 }, costUsd: 0.05 });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
}
const [first] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: other.company.id,
runId: other.run.id, sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: false } };
const foreignFiles = new Set<string>();
for (let i = 0; i < 256; i++) foreignFiles.add(await spoolUsageReceipt({ ...envelope, id: randomUUID(), sequence: i + 1 }));
const reads = new Map<string, number>(), read = fs.readFile.bind(fs);
let checkedLock = false, lockWasAvailable = false;
const spy = vi.spyOn(fs, "readFile").mockImplementation(async (file, options) => {
if (typeof file === "string" && foreignFiles.has(file)) {
reads.set(file, (reads.get(file) ?? 0) + 1);
if (!checkedLock) {
checkedLock = true;
await db.transaction(async tx => {
await tx.execute(sql`set local lock_timeout = '100ms'`);
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
});
lockWasAvailable = true;
// Published after the index's directory snapshot: the locked drain
// must discover it instead of trusting a stale batch inventory.
await spoolUsageReceipt({ ...envelope, id: randomUUID(), companyId: f.company.id, runId: f.run.id,
sourceId: String(first.usageJson?.accountingReceiptSourceId), sequence: 2,
receipt: { complete: true, usage: { inputTokens: 200, outputTokens: 20 }, costUsd: 0.2 } });
}
}
return read(file, options);
});
try { await reconcileRunCosts(db); } finally { spy.mockRestore(); }
expect(lockWasAvailable).toBe(true);
expect(reads.size).toBe(256);
expect([...reads.values()]).toEqual(Array(256).fill(1));
const charges = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
expect(charges).toHaveLength(8);
expect(charges.find(charge => charge.heartbeatRunId === f.run.id)).toMatchObject({ inputTokens: 200, costCents: 20 });
expect((await costService(db).summary(f.company.id)).spendCents).toBe(55);
const listing = vi.spyOn(fs, "readdir");
try {
for (const run of runs) expect(await accountRunCost(db, run.id)).toBe(false);
// An acknowledged run with a stale pending flag still needs its locked
// cleanup, but neither form of duplicate finalization needs spool I/O.
await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(false);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountingPending).toBe(false);
expect(listing).not.toHaveBeenCalled();
} finally { listing.mockRestore(); }
await Promise.all([...foreignFiles].map(file => fs.rm(file)));
});
it("reports over-limit advisory budgets without blocking, and tolerates legacy active zero limits", async () => {
const f = await fixture(), budgets = budgetService(db);
const policy = await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1, hardStopEnabled: false }, "board");
await event(f, { costCents: "2.0000001" });
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ status: "hard_stop", observedAmountExact: "2.0000001", paused: false });
// Older databases can contain an active zero policy; it must never become
// an accidental hard stop during an upgrade or a projection inspection.
await db.update(budgetPolicies).set({ amount: 0, isActive: true }).where(eq(budgetPolicies.id, policy.policyId));
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ status: "ok", amount: 0 });
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
});
it("recovers late bootstrap proof without replacing the stop owner's terminal metadata", async () => {
const f = await fixture();
await reserveRunBudget(db, f.company.id, f.run.id, null);
const stopped = { executionCancellation: { state: "acknowledged" } };
await db.update(heartbeatRuns).set({ status: "cancelled", resultJson: stopped,
usageJson: { accountingReceiptReady: false, accountingProviderWorkStarted: false },
}).where(eq(heartbeatRuns.id, f.run.id));
const integrity = accountingIntegrityService(db);
expect((await integrity.health(f.company.id)).items[0].state).toBe("retryable");
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
await reconcileRunCosts(db);
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
expect(run).toMatchObject({ status: "cancelled", resultJson: stopped, costAccountingPending: false });
expect(run.costAccountedAt).not.toBeNull();
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id)))[0].state).toBe("released");
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toEqual([]);
});
it("rejects absent, foreign, finished and blocked dispatch without changing the ledger", async () => {
const f = await fixture(), other = await fixture();
await expect(reserveRunBudget(db,f.company.id,randomUUID(),null)).rejects.toThrow("not found");
await expect(reserveRunBudget(db,other.company.id,f.run.id,null)).rejects.toThrow("not found");
await expect(reserveRunBudget(db,f.company.id,f.run.id,other.project.id)).rejects.toThrow("not found");
await db.update(heartbeatRuns).set({status:"failed"}).where(eq(heartbeatRuns.id,f.run.id));
await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("no longer");
await db.update(heartbeatRuns).set({status:"running",costAccountedAt:new Date()}).where(eq(heartbeatRuns.id,f.run.id));
await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("no longer");
await db.update(heartbeatRuns).set({costAccountedAt:null}).where(eq(heartbeatRuns.id,f.run.id));
await budgetService(db).upsertPolicy(f.company.id,{scopeType:"agent",scopeId:f.agent.id,amount:1},"board");
await event(f,{costCents:1});
await expect(reserveRunBudget(db,f.company.id,f.run.id,null)).rejects.toThrow("paused");
await expect(budgetService(db).upsertPolicy(f.company.id,{scopeType:"agent",scopeId:f.agent.id,amount:1,reservationCents:-1},"board")).rejects.toThrow("negative");
});
it("distinguishes legacy totals, missing acknowledgements, missing receipts, and corrupted receipt amounts", async () => {
const f = await fixture(); const integrity = accountingIntegrityService(db);
await db.insert(agentRuntimeState).values({companyId:f.company.id,agentId:f.agent.id,adapterType:"process",totalCostCents:4});
expect((await integrity.inspect(f.company.id)).findings[0].kind).toBe("legacy_runtime");
await db.update(heartbeatRuns).set({status:"failed",costAccountingPending:true,usageJson:{...usage,costUsdExact:"0.01",accountingReceiptReady:true}}).where(eq(heartbeatRuns.id,f.run.id));
expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_acknowledgement")).toBe(true);
expect(await integrity.retry(f.company.id,f.run.id,"board")).toEqual({accounted:true});
await db.update(heartbeatRuns).set({usageJson:{...usage,costUsdExact:"0.02"}}).where(eq(heartbeatRuns.id,f.run.id));
expect((await integrity.inspect(f.company.id)).findings.map(f=>f.kind)).toContain("receipt_mismatch");
await db.update(heartbeatRuns).set({usageJson:{...usage,costUsdExact:"invalid"}}).where(eq(heartbeatRuns.id,f.run.id));
expect((await integrity.inspect(f.company.id)).findings.find(f=>f.kind==="receipt_mismatch")?.expected.cents).toBe("invalid receipt");
await db.delete(costEvents).where(eq(costEvents.heartbeatRunId,f.run.id));
expect((await integrity.inspect(f.company.id)).findings.map(f=>f.kind)).toContain("missing_receipt");
await expect(integrity.repair(f.company.id,"a".repeat(64)," ","board")).rejects.toThrow("reason");
await expect(integrity.retry(f.company.id,randomUUID(),"board")).rejects.toThrow("not found");
});
it("records a failed operator retry and keeps it actionable", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({status:"failed",costAccountingPending:true,usageJson:{inputTokens:0.1,accountingReceiptReady:true}}).where(eq(heartbeatRuns.id,f.run.id));
const integrity = accountingIntegrityService(db);
await expect(integrity.retry(f.company.id,f.run.id,"board")).rejects.toThrow("Invalid cost");
const health = await integrity.health(f.company.id);
expect(health.items[0]).toMatchObject({state:"retryable",attempts:1,lastError:"Invalid cost receipt"});
expect(health.items[0].lastAttemptAt).not.toBeNull();
});
it("matches provider request IDs and run/model pairs, leaves ambiguous and non-inference lines unresolved", async () => {
const f = await fixture(), service = billingReconciliationService(db);
await event(f,{heartbeatRunId:f.run.id,providerRequestId:"request"}); await event(f,{heartbeatRunId:f.run.id,model:"second"});
const line = {amountCents:"1",occurredAt:new Date().toISOString()};
const invoice = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"requests",currency:"USD",lines:[
{...line,externalId:"provider",providerRequestId:"request"}, {...line,externalId:"model",runId:f.run.id,model:"second"},
{...line,externalId:"ambiguous",runId:f.run.id}, {...line,externalId:"unknown",providerRequestId:"absent"},
{...line,externalId:"unlinked"}, {...line,externalId:"fee",kind:"fee"}, {...line,externalId:"credit",kind:"credit"},
]},"board");
expect((await service.list(f.company.id))[0].id).toBe(invoice.id);
const report = await service.reconcile(f.company.id,invoice.id);
expect(Object.fromEntries(report.lines.map(l=>[l.externalId,l.status]))).toEqual({provider:"difference",model:"difference",ambiguous:"ambiguous",unknown:"unmatched",unlinked:"unmatched",fee:"non_inference",credit:"non_inference"});
await expect(service.importInvoice(f.company.id,{biller:"fixture",externalId:"missing-run",currency:"USD",lines:[{...line,externalId:"run",runId:randomUUID()}]},"board")).rejects.toThrow("run not found");
});
it("rejects contradictory invoice evidence and reused correction keys, and preserves legacy originals", async () => {
const f = await fixture(), service = billingReconciliationService(db); const charge = await event(f);
await db.update(costEvents).set({reportedCostCents:null}).where(eq(costEvents.id,charge.id));
const input = {idempotencyKey:"correction",expectedCents:"0.1234567",correctedCents:"1",reason:"Reviewed",pricing:{source:"operator" as const}};
await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:randomUUID()},"board")).rejects.toThrow("not found");
const invoice = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"bad-evidence",currency:"USD",lines:[{externalId:"line",amountCents:"2",occurredAt:new Date().toISOString(),costEventId:charge.id}]},"board");
const [line] = (await service.reconcile(f.company.id,invoice.id)).lines;
await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:line.id},"board")).rejects.toThrow("does not support");
const noMatch = await service.importInvoice(f.company.id,{biller:"fixture",externalId:"unverified",currency:"USD",lines:[{externalId:"line",amountCents:"1",occurredAt:new Date().toISOString()}]},"board");
await expect(service.adjust(f.company.id,charge.id,{...input,invoiceLineId:(await service.reconcile(f.company.id,noMatch.id)).lines[0].id},"board")).rejects.toThrow("unverified");
await service.adjust(f.company.id,charge.id,input,"board");
await expect(service.adjust(f.company.id,charge.id,{...input,correctedCents:"2"},"board")).rejects.toThrow("different contents");
expect((await db.select().from(costEvents).where(eq(costEvents.id,charge.id)))[0].reportedCostCents).toBe("0.1234567");
});
it("aggregates attempts exactly, rejects incomplete attempts, and never lets a late old attempt replace the current attempt", async () => {
const f = await fixture(), spool = path.join(directory,randomUUID());
const recorder = await createRunUsageRecorder(db,{companyId:f.company.id,runId:f.run.id,adapterType:"process"},spool);
const first=randomUUID(), second=randomUUID();
await recorder.capture({attemptId:first,usage,costUsdExact:"0.010000001",complete:true,billingType:"api"});
await recorder.capture({attemptId:second,usage,costUsd:0.02,complete:false,billingType:"api"});
await recorder.capture({attemptId:first,usage,costUsdExact:"0.010000001",complete:true,billingType:"api"});
const result = await recorder.complete({exitCode:0,signal:null,timedOut:false,usage,costUsd:0.03,billingType:"api"});
expect(result).toMatchObject({costUsdExact:"0.040000001",complete:true,usage:{inputTokens:14,cachedInputTokens:22,outputTokens:6}});
await recorder.capture({attemptId:randomUUID(),usage,complete:false,billingType:"subscription"});
const uncertain = await recorder.complete({exitCode:1,signal:null,timedOut:false,usage});
expect(uncertain).toMatchObject({costUsd:0.040000001,costUsdExact:"0.040000001",costStatus:"unpriced",billingType:"unknown"});
const partial = await recorder.complete({exitCode:1,signal:null,timedOut:false});
expect(partial.complete).toBe(false);
expect(partial.usage?.inputTokens).toBe(21);
});
it.each(["block", "allow"] as const)("keeps known attempted-run spend when unknown prices %s new work", async (unpricedUsagePolicy) => {
const f = await fixture(), budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, unpricedUsagePolicy }, "board");
await reserveRunBudget(db, f.company.id, f.run.id, null);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: f.run.id, adapterType: "process" }, path.join(directory, randomUUID()));
await recorder.capture({ attemptId: randomUUID(), complete: true, billingType: "metered_api", usage, costUsdExact: "0.010000001" });
const aggregate = await recorder.capture({ attemptId: randomUUID(), complete: true, billingType: "metered_api", usage, costUsd: null, costStatus: "unpriced" });
expect(aggregate).toMatchObject({ costUsdExact: "0.010000001", costStatus: "unpriced", complete: true });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(true);
expect(await accountRunCost(db, f.run.id)).toBe(false);
expect(await costService(db).summary(f.company.id)).toMatchObject({ spendCentsExact: "1.0000001", unpricedEventCount: 1, pricingComplete: false, pendingRunCount: 0 });
const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
expect(runtime.totalCostCents).toBe(1.0000001);
const [reservation] = await db.select().from(budgetReservations).where(eq(budgetReservations.runId, f.run.id));
expect(reservation.state).toBe("settled");
expect((await budgets.getInvocationBlock(f.company.id, f.agent.id)) !== null).toBe(unpricedUsagePolicy === "block");
});
it("refuses invalid checkpoints and does not certify a run after capture failure", async () => {
const f = await fixture();
const recorder = await createRunUsageRecorder(db,{companyId:f.company.id,runId:f.run.id,adapterType:"process"},path.join(directory,randomUUID()));
await expect(recorder.capture({complete:true,costUsdExact:"NaN"})).rejects.toThrow();
const result = await recorder.complete({exitCode:0,signal:null,timedOut:false,usage,costUsd:0.01});
expect(result.complete).toBe(false);
await expect(createRunUsageRecorder(db,{companyId:f.company.id,runId:randomUUID(),adapterType:"process"},path.join(directory,randomUUID()))).rejects.toThrow("cannot accept");
});
it("preserves disk errors, rejects missing runs, and handles invalid spool entries without deleting evidence", async () => {
const f=await fixture(), spool=path.join(directory,randomUUID());
const envelope:UsageReceiptEnvelope={schema:"paperclip/accounting-receipt/v1",id:randomUUID(),companyId:f.company.id,runId:randomUUID(),sourceId:randomUUID(),sequence:1,receivedAt:new Date().toISOString(),adapterType:"process",receipt:{complete:true,costUsd:1}};
await expect(persistUsageReceipt(db,envelope)).rejects.toThrow("not found");
const rename=vi.spyOn(fs,"rename").mockRejectedValueOnce(new Error("Disk rename unavailable"));
try { await expect(spoolUsageReceipt(envelope,spool)).rejects.toThrow("Disk rename"); } finally { rename.mockRestore(); }
expect(await fs.readdir(spool)).toEqual([]);
await fs.writeFile(path.join(spool,"large.json"),"x".repeat(1024*1024+1));
expect((await replayUsageReceipts(db,spool)).failed).toBe(1);
const read=vi.spyOn(fs,"readdir").mockRejectedValueOnce(Object.assign(new Error("denied"),{code:"EACCES"}));
try { await expect(replayUsageReceipts(db,spool)).rejects.toThrow("denied"); } finally { read.mockRestore(); }
});
it("uses the isolated instance spool and retains evidence when retry rotation itself fails", async () => {
const f = await fixture();
const envelope: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.company.id, runId: f.run.id,
sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: true, costUsd: 0 } };
const file = await spoolUsageReceipt(envelope);
expect(path.dirname(file)).toBe(usageReceiptSpoolPath());
expect((await replayUsageReceipts(db)).replayed).toBe(1);
const spool = path.join(directory, randomUUID()); await fs.mkdir(spool);
const bad = path.join(spool, "bad.json"); await fs.writeFile(bad, "invalid");
const rename = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("read-only directory"));
try { expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 0, failed: 1 }); } finally { rename.mockRestore(); }
expect(await fs.readFile(bad, "utf8")).toBe("invalid");
});
});
@@ -0,0 +1,502 @@
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { and, eq, sql } from "drizzle-orm";
import { agents, agentRuntimeState, budgetReservations, companies, costEvents, costAdjustments, createDb, heartbeatRuns, runUsageReceipts, issues, nativeRunFinalizations } from "@paperclipai/db";
import { accountingIntegrityService } from "../services/accounting-integrity.js";
import { billingReconciliationService } from "../services/billing-reconciliation.js";
import { budgetService } from "../services/budgets.js";
import { reserveRunBudget } from "../services/budget-reservations.js";
import { costService } from "../services/costs.js";
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
import { createRunUsageRecorder, persistUsageReceipt, replayUsageReceipts, spoolUsageReceipt, type UsageReceiptEnvelope } from "../services/usage-receipts.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { createUsageCheckpointLog } from "@paperclipai/adapter-utils/usage-checkpoint";
import { runChildProcess } from "@paperclipai/adapter-utils/server-utils";
import { createClaudeStreamParser, parseClaudeStreamJson } from "../../../packages/adapters/claude-local/src/server/parse.js";
import { parseCodexJsonl } from "../../../packages/adapters/codex-local/src/server/parse.js";
import { parseOpenCodeJsonl } from "../../../packages/adapters/opencode-local/src/server/parse.js";
import { parseCursorJsonl } from "../../../packages/adapters/cursor-local/src/server/parse.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("accounting operations and durable receipts (PostgreSQL)", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let directory: string;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("paperclip-accounting-operators-"); db = createDb(database.connectionString); directory = await fs.mkdtemp(path.join(os.tmpdir(), "accounting-spool-test-")); }, 30_000);
afterAll(async () => { await database?.cleanup(); await fs.rm(directory, { recursive: true, force: true }); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Accounting", issuePrefix: `A${randomUUID().slice(0,7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
return { company, agent };
}
async function runFor(f: Awaited<ReturnType<typeof fixture>>, status = "running") {
return (await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, invocationSource: "on_demand", status }).returning())[0];
}
async function receipt(f: Awaited<ReturnType<typeof fixture>>, costCents: number | string = "1.1234567") {
return costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test-model", costCents, occurredAt: new Date(), idempotencyKey: randomUUID() });
}
it("keeps the last nanocent exact in storage, reports, projections and admission", async () => {
const f = await fixture();
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 2_000_000_000 }, "board");
await receipt(f, "1999999999.9999999");
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1999999999.9999999");
await receipt(f, "0.0000001");
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
const huge = await fixture();
await receipt(huge, "99999999999999999.0000001");
expect((await accountingIntegrityService(db).inspect(huge.company.id)).findings).toEqual([]);
});
it("admits only affordable concurrent reservations and settles atomically with the receipt", async () => {
const f = await fixture(); const runs = await Promise.all(Array.from({ length: 4 }, () => runFor(f)));
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 10, reservationCents: "6" }, "board");
const results = await Promise.allSettled(runs.map(run => reserveRunBudget(db, f.company.id, run.id, null)));
expect(results.filter(r => r.status === "fulfilled")).toHaveLength(1);
const index = results.findIndex(r => r.status === "fulfilled"); const run = runs[index];
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("6.0000000");
await expect(reserveRunBudget(db, f.company.id, run.id, null)).rejects.toThrow("already reserved");
// Reservations never disappear merely because wall-clock time passed.
await db.update(budgetReservations).set({ createdAt: new Date("2000-01-01") }).where(eq(budgetReservations.runId, run.id));
const denied = runs[(index + 1) % runs.length];
await expect(reserveRunBudget(db, f.company.id, denied.id, null)).rejects.toThrow("reserved");
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), usageJson: { costUsdExact: "0.04", billingType: "metered_api", accountingReceiptReady: true } }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(true);
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000");
await reserveRunBudget(db, f.company.id, denied.id, null);
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
});
it("reuses a held native reservation only for its current recovery lease", async () => {
const f = await fixture(); const run = await runFor(f);
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover" }).returning();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: issue.id }).where(eq(heartbeatRuns.id, run.id));
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: run.id, issueId: issue.id,
phase: "running", leaseOwner: "successor", leaseExpiresAt: new Date(Date.now() + 60_000) });
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 10, reservationCents: "6" }, "board");
const original = await reserveRunBudget(db, f.company.id, run.id, null);
await expect(reserveRunBudget(db, f.company.id, run.id, null)).rejects.toThrow("already reserved");
await expect(reserveRunBudget(db, f.company.id, run.id, null, {}, "stale-owner")).rejects.toThrow("already reserved");
expect(await reserveRunBudget(db, f.company.id, run.id, null, {}, "successor")).toEqual({ ...original, reused: true });
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("6.0000000");
await db.update(nativeRunFinalizations).set({ leaseExpiresAt: new Date(0) }).where(eq(nativeRunFinalizations.runId, run.id));
await expect(reserveRunBudget(db, f.company.id, run.id, null, {}, "successor")).rejects.toThrow("already reserved");
});
it.each([false, true])("settles absent usage without hiding explicit unknown pricing (%s)", async (unpriced) => {
const f = await fixture(); const run = await runFor(f);
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
await reserveRunBudget(db, f.company.id, run.id, null);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory);
await recorder.complete({ exitCode: 0, signal: null, timedOut: false, ...(unpriced ? { provider: "moonshot", costStatus: "unpriced" as const, costUsd: null } : {}) });
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(true);
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000");
const summary = await costService(db).summary(f.company.id);
expect(summary.unpricedEventCount).toBe(unpriced ? 1 : 0);
const block = await budgetService(db).getInvocationBlock(f.company.id, f.agent.id);
if (unpriced) expect(block).not.toBeNull(); else expect(block).toBeNull();
});
it.each([
...["turn.failed", "error", "turn.completed"].map(type => ({
label: `Codex ${type}`, adapterType: "codex_local", parse: () => parseCodexJsonl(JSON.stringify({ type })),
})),
...["result", "step_finish"].map(type => ({
label: `Cursor ${type}`, adapterType: "cursor", parse: () => parseCursorJsonl(JSON.stringify({ type })),
})),
{ label: "OpenCode step_finish", adapterType: "opencode_local", parse: () => parseOpenCodeJsonl(JSON.stringify({ type: "step_finish" })) },
])("keeps $label without usage unpriced and its reservation held", async ({ adapterType, parse }) => {
const f = await fixture(); const run = await runFor(f);
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
await reserveRunBudget(db, f.company.id, run.id, null);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType }, directory);
const parsed = parse();
const accounting = { provider: "openai", biller: "openai", billingType: "api" as const, model: "gpt-6-astra", usageBasis: "per_run" as const,
usage: parsed.usageReported ? parsed.usage : undefined, costUsd: null, costStatus: "unpriced" as const };
await recorder.capture({ ...accounting, complete: parsed.usageComplete });
await recorder.complete({ ...accounting, exitCode: 1, signal: null, timedOut: false, usageComplete: parsed.usageComplete });
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
expect(stored.usageJson).toMatchObject({ costUsd: null, costUsdExact: null, costStatus: "unpriced", accountingReceiptReady: false });
expect(await accountRunCost(db, run.id)).toBe(false);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]);
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("10.0000000");
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
});
it("recovers a flushed zero-usage OpenCode receipt after stopping before the adapter result is saved", async () => {
const f = await fixture(); const run = await runFor(f);
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
await reserveRunBudget(db, f.company.id, run.id, null);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "opencode_local" }, directory);
const log = createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => ({
provider: "openai", biller: "openai", billingType: "unknown", model: "openai/test", usageBasis: "per_run",
usage: parseOpenCodeJsonl(records).usage, costUsd: null, complete: false,
}));
await log("stdout", JSON.stringify({ type: "step_finish", part: { tokens: { input: 0, output: 0 } } }));
await log.flush({ complete: true });
// No recorder.complete(adapterResult): the process stopped during cleanup.
await db.update(heartbeatRuns).set({ status: "interrupted", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(true);
const events = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
expect(events).toHaveLength(1);
expect(events[0]).toMatchObject({ inputTokens: 0, outputTokens: 0, costCents: 0, costStatus: "unpriced" });
expect((await accountingIntegrityService(db).health(f.company.id))).toMatchObject({ heldReservationCents: "0.0000000", pendingRunCount: 0 });
expect(await accountRunCost(db, run.id)).toBe(false);
});
it("keeps an interrupted partial receipt pending and settles its reservation when evidence becomes complete", async () => {
const f = await fixture(); const run = await runFor(f);
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, reservationCents: "10" }, "board");
await reserveRunBudget(db, f.company.id, run.id, null);
const usage = { accountingReceiptReady: false, costUsdExact: "0.01", inputTokens: 5, provider: "test", billingType: "metered_api" };
await db.update(heartbeatRuns).set({ status: "interrupted", usageJson: usage }).where(eq(heartbeatRuns.id, run.id));
const integrity = accountingIntegrityService(db);
expect((await integrity.health(f.company.id)).pendingRunCount).toBe(1);
expect((await integrity.health(f.company.id)).items[0]).toMatchObject({ runId: run.id, state: "waiting_for_receipt" });
expect((await costService(db).summary(f.company.id)).pendingRunCount).toBe(1);
expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_acknowledgement")).toBe(true);
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
expect(await accountRunCost(db, run.id)).toBe(false);
expect((await integrity.health(f.company.id)).heldReservationCents).toBe("10.0000000");
await db.update(heartbeatRuns).set({ usageJson: { ...usage, accountingReceiptReady: true } }).where(eq(heartbeatRuns.id, run.id));
await reconcileRunCosts(db);
expect((await costService(db).summary(f.company.id))).toMatchObject({ pendingRunCount: 0, spendCents: 1 });
expect((await integrity.health(f.company.id)).heldReservationCents).toBe("0.0000000");
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
expect(await accountRunCost(db, run.id)).toBe(false);
});
it("independently detects and repairs corrupted projections with a stale-review guard and audit", async () => {
const f = await fixture(); const run = await runFor(f, "succeeded");
await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { accountingReceiptReady: true, costUsdExact: "0.012345678", inputTokens: 7, cachedInputTokens: 3, outputTokens: 11 } }).where(eq(heartbeatRuns.id, run.id));
await accountRunCost(db, run.id);
const service = accountingIntegrityService(db);
expect((await service.inspect(f.company.id)).findings).toEqual([]);
await db.update(companies).set({ spentMonthlyCents: 999 }).where(eq(companies.id, f.company.id));
await db.update(agents).set({ spentMonthlyCents: 888 }).where(eq(agents.id, f.agent.id));
await db.update(agentRuntimeState).set({ totalCostCents: 777, totalInputTokens: 600 }).where(eq(agentRuntimeState.agentId, f.agent.id));
const review = await service.inspect(f.company.id);
expect(review.findings.map(f => f.kind)).toEqual(["agent_projection", "company_projection", "runtime_projection"]);
await db.update(companies).set({ spentMonthlyCents: 998 }).where(eq(companies.id, f.company.id));
await expect(service.repair(f.company.id, review.fingerprint, "Fix drift", "board")).rejects.toThrow("changed since inspection");
const fresh = await service.inspect(f.company.id);
expect((await service.repair(f.company.id, fresh.fingerprint, "Fix drift", "board")).findings).toEqual([]);
const audit = await db.execute(sql`select details from activity_log where company_id = ${f.company.id} and action = 'accounting.projections_repaired'`);
expect(audit).toHaveLength(1);
expect(await accountRunCost(db, run.id)).toBe(false);
expect((await service.inspect(f.company.id)).findings).toEqual([]);
});
it.each(["before", "after", "concurrent"])("accounts separate run charges %s the provider receipt without false integrity findings", async (timing) => {
const f = await fixture(); const run = await runFor(f, "succeeded");
await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { costUsdExact: "0.01", inputTokens: 7, cachedInputTokens: 3, outputTokens: 11 } }).where(eq(heartbeatRuns.id, run.id));
const extra = { agentId: f.agent.id, heartbeatRunId: run.id, provider: "test", model: "tool", costCents: "0.1234567", inputTokens: 2, cachedInputTokens: 4, outputTokens: 6, occurredAt: new Date(), idempotencyKey: "extra-charge" };
const costs = costService(db);
if (timing === "before") await costs.createEvent(f.company.id, extra);
if (timing === "concurrent") await Promise.all([costs.createEvent(f.company.id, extra), accountRunCost(db, run.id)]);
else await accountRunCost(db, run.id);
const event = await costs.createEvent(f.company.id, extra);
// Replaying either source cannot count it twice.
await costs.createEvent(f.company.id, extra);
expect(await accountRunCost(db, run.id)).toBe(false);
const [runtime] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
expect(runtime).toMatchObject({ totalCostCents: 1.1234567, totalInputTokens: 9, totalCachedInputTokens: 7, totalOutputTokens: 17 });
const integrity = accountingIntegrityService(db);
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
await billingReconciliationService(db).adjust(f.company.id, event.id, { idempotencyKey: "extra-correction", expectedCents: "0.1234567", correctedCents: "0.2234567", reason: "Provider correction", pricing: { source: "provider_invoice", evidence: "external charge" } }, "board");
expect((await costs.summary(f.company.id)).spendCentsExact).toBe("1.2234567");
expect((await integrity.inspect(f.company.id)).findings).toEqual([]);
await db.update(agentRuntimeState).set({ totalCostCents: 999 }).where(eq(agentRuntimeState.agentId, f.agent.id));
const review = await integrity.inspect(f.company.id);
expect(review.findings.map(f => f.kind)).toEqual(["runtime_projection"]);
expect((await integrity.repair(f.company.id, review.fingerprint, "Fix drift", "board")).findings).toEqual([]);
// An extra charge cannot conceal a missing original provider receipt.
await db.delete(costEvents).where(and(eq(costEvents.heartbeatRunId, run.id), eq(costEvents.idempotencyKey, `heartbeat:${run.id}:final`)));
expect((await integrity.inspect(f.company.id)).findings.some(f => f.kind === "missing_receipt")).toBe(true);
});
it("rejects public charges that impersonate the provider receipt namespace", async () => {
const f = await fixture();
await expect(costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test", costCents: 1, occurredAt: new Date(), idempotencyKey: "heartbeat:fake:final" })).rejects.toThrow("reserved");
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]);
});
it.each(["zero", "paid", "tokens"] as const)("preserves model attribution after a retry with %s earlier usage", async kind => {
const f = await fixture(); const run = await runFor(f);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory);
await recorder.capture({ attemptId: randomUUID(), provider: "anthropic", billingType: "metered_api",
usage: { inputTokens: kind === "tokens" ? 1 : 0, outputTokens: 0, cacheWriteTokens: undefined },
costUsd: kind === "paid" ? 0.001 : 0, complete: true });
const success = { attemptId: randomUUID(), provider: "anthropic", biller: "anthropic", billingType: "metered_api" as const, model: "mixed",
usage: { inputTokens: 30, outputTokens: 4, cachedInputTokens: 100 }, costUsd: 0.007, complete: true,
usageByModel: [
{ model: "large", costUsd: 0.005, usage: { inputTokens: 20, outputTokens: 3, cachedInputTokens: 100 } },
{ model: "small", costUsd: 0.002, usage: { inputTokens: 10, outputTokens: 1 } },
] };
await recorder.capture(success);
const final = await recorder.complete({ ...success, exitCode: 0, signal: null, timedOut: false, usageComplete: true });
expect(final.usageByModel).toEqual(kind === "zero" ? success.usageByModel : undefined);
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(true);
const rows = await costService(db).byProvider(f.company.id);
expect(rows.map(row => row.model).sort()).toEqual(kind === "zero" ? ["large", "small"] : ["mixed"]);
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe(kind === "paid" ? "0.8000000" : "0.7000000");
});
it("imports invoices idempotently and appends reviewed corrections without losing the provider receipt", async () => {
const f = await fixture();
const originalPricing = { source: "rate_card" as const, version: "2026-09-01", inputCentsPerMillion: "12.3456789" };
const event = await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "test", model: "test-model",
costCents: "1.1234567", occurredAt: new Date(), pricingProvenance: originalPricing });
const service = billingReconciliationService(db);
const invoice = { biller: "test", externalId: "invoice-1", currency: "usd", lines: [{ externalId: "request-1", amountCents: "2.0000001", occurredAt: new Date().toISOString(), costEventId: event.id }] };
const imported = await service.importInvoice(f.company.id, invoice, "board");
expect((await service.importInvoice(f.company.id, invoice, "board")).id).toBe(imported.id);
await expect(service.importInvoice(f.company.id, { ...invoice, currency: "EUR" }, "board")).rejects.toThrow("different contents");
const review = await service.reconcile(f.company.id, imported.id);
expect(review.lines[0]).toMatchObject({ status: "difference", recordedCents: "1.1234567", differenceCents: "0.8765434" });
const input = { idempotencyKey: "fix-1", expectedCents: "1.1234567", correctedCents: "2.0000001", invoiceLineId: review.lines[0].id, reason: "Provider invoice", pricing: { source: "provider_invoice" as const, evidence: "invoice-1" } };
const corrected = await service.adjust(f.company.id, event.id, input, "board");
expect(corrected.previousPricing).toEqual(originalPricing);
expect(corrected.pricing).toEqual(input.pricing);
expect((await service.adjust(f.company.id, event.id, input, "board")).id).toBe(corrected.id);
expect((await service.reconcile(f.company.id, imported.id)).lines[0].status).toBe("matched");
const [row] = await db.select().from(costEvents).where(eq(costEvents.id, event.id));
expect(row.reportedCostCents).toBe("1.1234567"); expect(row.receiptHash).toBe(event.receiptHash);
expect((await service.adjustments(f.company.id, event.id))).toHaveLength(1);
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
await expect(service.adjust(f.company.id, event.id, { ...input, idempotencyKey: "stale" }, "board")).rejects.toThrow("changed since review");
const second = await service.adjust(f.company.id, event.id, { ...input, invoiceLineId: undefined, idempotencyKey: "fix-2",
expectedCents: "2.0000001", correctedCents: "3", pricing: { source: "operator" }, reason: "Reviewed updated evidence" }, "board");
expect(second.previousPricing).toEqual(input.pricing);
expect((await service.adjustments(f.company.id, event.id))).toHaveLength(2);
});
it("refuses cross-company references, guesses, duplicate invoice matches, and currency conversion", async () => {
const f = await fixture(), foreign = await fixture(); const event = await receipt(f);
const service = billingReconciliationService(db);
const base = { biller: "test", externalId: "invoice", currency: "USD", lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString(), costEventId: event.id }] };
await expect(service.importInvoice(foreign.company.id, base, "board")).rejects.toThrow("not found");
await expect(service.adjust(foreign.company.id, event.id, { idempotencyKey: "bad", expectedCents: 0, correctedCents: 1, reason: "test", pricing: { source: "operator" } }, "board")).rejects.toThrow("not found");
const duplicate = await service.importInvoice(f.company.id, { ...base, lines: [...base.lines, { ...base.lines[0], externalId: "second" }] }, "board");
expect((await service.reconcile(f.company.id, duplicate.id)).lines.map(l => l.status)).toEqual(["ambiguous", "ambiguous"]);
const eur = await service.importInvoice(f.company.id, { ...base, externalId: "eur", currency: "EUR" }, "board");
expect((await service.reconcile(f.company.id, eur.id)).lines[0].status).toBe("unsupported_currency");
await expect(service.reconcile(foreign.company.id, eur.id)).rejects.toThrow("not found");
expect(await db.select().from(costAdjustments).where(eq(costAdjustments.companyId, f.company.id))).toEqual([]);
});
it.each(["provider_invoice", "rate_card"] as const)("prices unknown usage from %s and preserves its estimate status", async (source) => {
const f = await fixture(); const run = await runFor(f, "failed");
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 10 }, "board");
await db.update(heartbeatRuns).set({ costAccountingPending: true, usageJson: { accountingReceiptReady: true, inputTokens: 9, provider: "test" } }).where(eq(heartbeatRuns.id, run.id));
await accountRunCost(db, run.id);
expect((await accountingIntegrityService(db).health(f.company.id)).unpricedEventCount).toBe(1);
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).not.toBeNull();
const [event] = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id));
await billingReconciliationService(db).adjust(f.company.id, event.id, { idempotencyKey: "pricing", expectedCents: 0, correctedCents: "1.0000001", reason: "Verified provider invoice", pricing: { source, evidence: "billing-export" } }, "board");
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await accountingIntegrityService(db).health(f.company.id)).unpricedEventCount).toBe(0);
expect((await costService(db).summary(f.company.id)).estimatedEventCount).toBe(source === "rate_card" ? 1 : 0);
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
});
it("spools a receipt during database failure, replays once, fences old sources, and preserves partial uncertainty", async () => {
const f = await fixture(); const run = await runFor(f); const spool = path.join(directory, randomUUID());
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, spool);
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Database unavailable"));
await recorder.capture({ costUsdExact: "0.010000001", usage: { inputTokens: 7, outputTokens: 2 }, complete: false });
transaction.mockRestore();
expect((await fs.readdir(spool)).filter(n => n.endsWith(".json"))).toHaveLength(1);
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 1, failed: 0 });
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 0, failed: 0 });
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(false);
expect((await accountingIntegrityService(db).health(f.company.id)).items[0].state).toBe("waiting_for_receipt");
await recorder.complete({ exitCode: 0, signal: null, timedOut: false, costUsdExact: "0.010000001", usage: { inputTokens: 7, outputTokens: 2 } });
expect(await accountRunCost(db, run.id)).toBe(true);
const receipts = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, run.id));
expect(receipts).toHaveLength(2);
await persistUsageReceipt(db, receipts[0].receiptJson as UsageReceiptEnvelope);
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.0000001");
const tampered = { ...(receipts[0].receiptJson as UsageReceiptEnvelope), receipt: { complete: true, costUsd: 123 } };
await expect(persistUsageReceipt(db, tampered)).rejects.toThrow("Conflicting usage");
});
it("settles a complete streamed Claude receipt when the final stdout tail clips its result", async () => {
const f = await fixture(); const run = await runFor(f);
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, reservationCents: 10 }, "board");
await reserveRunBudget(db, f.company.id, run.id, null);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory);
const consume = createClaudeStreamParser();
const log = createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => {
const parsed = consume(records);
return { provider: "anthropic", biller: "anthropic", model: "claude-fable-5-1", billingType: "metered_api",
usage: parsed.usage ?? undefined, costUsd: parsed.costUsd, complete: parsed.resultJson !== null };
});
const proc = await runChildProcess(run.id, process.execPath, ["-e", `console.log(JSON.stringify({
type: "result", subtype: "success", result: "x".repeat(5 * 1024 * 1024),
total_cost_usd: 0.012345678, usage: { input_tokens: 7, cache_read_input_tokens: 3, output_tokens: 2 }
}))`], { cwd: directory, env: {}, timeoutSec: 10, graceSec: 1, onLog: log });
await log.flush();
const tail = parseClaudeStreamJson(proc.stdout);
expect(tail.resultJson).toBeNull();
const final = await recorder.complete({ exitCode: proc.exitCode, signal: proc.signal, timedOut: proc.timedOut,
provider: "anthropic", billingType: "metered_api", usage: tail.usage ?? undefined, costUsd: tail.costUsd,
usageComplete: tail.resultJson !== null });
expect(final).toMatchObject({ complete: true, costUsd: 0.012345678, usage: { inputTokens: 7, cachedInputTokens: 3, outputTokens: 2 } });
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(true);
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.2345678");
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("0.0000000");
});
it("holds failed retry accounting when flush throws before final completion", async () => {
const f = await fixture(); const run = await runFor(f); const spool = path.join(directory, randomUUID());
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100, reservationCents: 10 }, "board");
await reserveRunBudget(db, f.company.id, run.id, null);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool);
const attempt = () => {
const consume = createClaudeStreamParser();
return createUsageCheckpointLog(async () => {}, async receipt => { await recorder.capture(receipt); }, records => {
const parsed = consume(records);
return { provider: "anthropic", billingType: "metered_api", usage: parsed.usage ?? undefined,
costUsd: parsed.costUsd, complete: parsed.resultJson !== null };
});
};
const rejectedResume = attempt();
await rejectedResume("stdout", JSON.stringify({ type: "result", subtype: "error_during_execution", total_cost_usd: 0,
usage: { input_tokens: 0, output_tokens: 0 } }) + "\n");
await rejectedResume.flush();
const [prior] = await db.select().from(runUsageReceipts).where(eq(runUsageReceipts.runId, run.id));
expect((prior.receiptJson as UsageReceiptEnvelope).receipt.complete).toBe(true);
const paidRetry = attempt();
const write = vi.spyOn(fs, "rename").mockRejectedValueOnce(new Error("Receipt disk full"));
try {
await paidRetry("stdout", JSON.stringify({ type: "result", subtype: "success", total_cost_usd: 0.25,
usage: { input_tokens: 10, output_tokens: 2 } }) + "\n");
await expect(paidRetry.flush()).rejects.toThrow("Receipt disk full");
} finally { write.mockRestore(); }
// This is the adapter's throw path: complete() is never called.
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
const read = async () => (await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0];
expect(await read()).toMatchObject({ costAccountingPending: true, costAccountedAt: null,
accountingLastError: "usage_capture_failed", usageJson: { accountingReceiptReady: false, accountingCaptureFailed: true } });
expect(await accountRunCost(db, run.id)).toBe(false);
// Old and later complete envelopes must not clear lost-capture evidence.
await spoolUsageReceipt(prior.receiptJson as UsageReceiptEnvelope, spool);
await spoolUsageReceipt({ ...prior.receiptJson as UsageReceiptEnvelope, id: randomUUID(), sequence: 100 }, spool);
expect(await replayUsageReceipts(db, spool)).toEqual({ replayed: 2, failed: 0 });
expect((await read()).usageJson?.accountingReceiptReady).toBe(false);
const replacement = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool);
expect((await replacement.complete({ exitCode: 0, signal: null, timedOut: false, costUsd: 0.25 })).complete).toBe(false);
// Even a finalizer overwriting readiness or claiming pre-provider failure cannot erase the fence.
await db.update(heartbeatRuns).set({ usageJson: { ...(await read()).usageJson, accountingReceiptReady: true },
resultJson: { executionRecovery: { providerWorkStarted: false } } }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(false);
expect((await read()).costAccountedAt).toBeNull();
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, run.id))).toEqual([]);
expect(await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id))).toEqual([]);
expect((await accountingIntegrityService(db).health(f.company.id)).heldReservationCents).toBe("10.0000000");
});
it.each(["capture", "complete", "invalid"] as const)("persists the failure fence before %s rejects", async kind => {
const f = await fixture(); const run = await runFor(f);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory);
await recorder.persistFailure(); // Healthy failure finalization has no fence to save.
await recorder.capture({ costUsd: 0, complete: true });
const rename = vi.spyOn(fs, "rename");
if (kind !== "invalid") rename.mockRejectedValueOnce(new Error("Receipt disk full"));
try {
const result = kind === "complete"
? recorder.complete({ exitCode: 0, signal: null, timedOut: false, costUsd: 1 })
: recorder.capture({ costUsd: kind === "invalid" ? -1 : 1, complete: true });
await expect(result).rejects.toThrow();
const [stored] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id));
expect(stored.usageJson).toMatchObject({ accountingCaptureFailed: true, accountingReceiptReady: false });
} finally { rename.mockRestore(); }
});
it("retries a failed durable fence before allowing failure finalization", async () => {
const f = await fixture(); const run = await runFor(f);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "process" }, directory);
await recorder.capture({ costUsd: 0, complete: true });
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Database unavailable"));
try {
await expect(recorder.capture({ costUsd: -1, complete: true })).rejects.toThrow();
expect(transaction).toHaveBeenCalledTimes(1);
transaction.mockRejectedValueOnce(new Error("Database still unavailable"));
await expect(recorder.persistFailure()).rejects.toThrow("Database still unavailable");
await recorder.persistFailure();
} finally { transaction.mockRestore(); }
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(false);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0]).toMatchObject({
costAccountedAt: null, accountingLastError: "usage_capture_failed",
usageJson: { accountingCaptureFailed: true, accountingReceiptReady: false },
});
});
it("does not let a replaced recorder's failed capture poison its successor", async () => {
const f = await fixture(); const run = await runFor(f);
const input = { companyId: f.company.id, runId: run.id, adapterType: "process" };
const old = await createRunUsageRecorder(db, input, directory);
const current = await createRunUsageRecorder(db, input, directory);
await current.capture({ costUsd: 0.25, complete: true });
await expect(old.capture({ costUsd: -1, complete: true })).rejects.toThrow();
await old.persistFailure();
await db.update(heartbeatRuns).set({ status: "failed" }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(true);
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("25.0000000");
});
it.each(["more_tokens", "more_cost", "new_attempt"])("keeps additional incomplete provider work pending (%s)", async (kind) => {
const f = await fixture(); const run = await runFor(f);
const recorder = await createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, directory);
await recorder.capture({ provider: "anthropic", usage: { inputTokens: 7, outputTokens: 2 }, costUsd: 0.01, complete: true });
if (kind === "new_attempt") await recorder.capture({ attemptId: randomUUID(), provider: "anthropic", usage: { inputTokens: 1, outputTokens: 0 }, complete: false });
const final = await recorder.complete({ exitCode: 1, signal: null, timedOut: false, usageComplete: false,
usage: { inputTokens: kind === "more_tokens" ? 8 : 7, outputTokens: 2 }, costUsd: kind === "more_cost" ? 0.02 : 0.01 });
expect(final.complete).toBe(false);
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
expect(await accountRunCost(db, run.id)).toBe(false);
});
it.skipIf(process.platform === "win32")("rejects recorder setup before dispatch when its new directory parent cannot be flushed", async () => {
const f = await fixture(); const run = await runFor(f);
const spool = path.join(directory, randomUUID(), "receipts");
const realParent = await fs.realpath(directory);
const open = fs.open.bind(fs);
const probe = vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
if (args[0] === realParent) throw Object.assign(new Error("unflushed parent"), { code: "EACCES" });
return open(...args);
});
try {
await expect(createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool)).rejects.toMatchObject({ code: "EACCES" });
expect(probe.mock.calls.some(([file]) => String(file).endsWith(".probe"))).toBe(false);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].usageJson?.accountingReceiptSourceId).toBeUndefined();
} finally { probe.mockRestore(); }
await expect(createRunUsageRecorder(db, { companyId: f.company.id, runId: run.id, adapterType: "claude_local" }, spool)).resolves.toBeDefined();
});
it("retains invalid spool evidence and bounds recovery work", async () => {
const spool = path.join(directory, randomUUID()); await fs.mkdir(spool);
await fs.writeFile(path.join(spool, "bad.json"), "not json");
expect(await replayUsageReceipts(db, spool, 1)).toEqual({ replayed: 0, failed: 1 });
expect((await fs.readdir(spool))[0]).toMatch(/^retry-/);
expect(await replayUsageReceipts(db, path.join(spool, "missing"))).toEqual({ replayed: 0, failed: 0 });
const f = await fixture(); const run = await runFor(f);
const unknown: UsageReceiptEnvelope = { schema: "paperclip/accounting-receipt/v1", id: randomUUID(), companyId: f.company.id, runId: run.id,
sourceId: randomUUID(), sequence: 1, receivedAt: new Date().toISOString(), adapterType: "process", receipt: { complete: true, costUsd: 10 } };
await spoolUsageReceipt(unknown, spool);
expect((await replayUsageReceipts(db, spool, 1)).replayed).toBe(1);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, run.id)))[0].costAccountingPending).toBe(false);
});
});
@@ -0,0 +1,266 @@
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import { eq, sql } from "drizzle-orm";
import { drizzle } from "drizzle-orm/postgres-js";
import { agents, budgetPolicies, companies, costEvents, createDb, heartbeatRuns, issues, nativeRunFinalizations, projects, type Db } from "@paperclipai/db";
import { billingReconciliationService } from "../services/billing-reconciliation.js";
import { accountingIntegrityService } from "../services/accounting-integrity.js";
import { budgetService, budgetServiceInTransaction } from "../services/budgets.js";
import { withAccountingTransaction } from "../services/accounting-transaction.js";
import { costService } from "../services/costs.js";
import { createCostAccountingReconciler } from "../services/run-cost-accounting.js";
import { startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
function deferred() {
let resolve!: () => void;
const promise = new Promise<void>(done => { resolve = done; });
return { promise, resolve };
}
describe("accounting performance invariants (PostgreSQL)", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>, db: Db;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-performance-"); db = createDb(database.connectionString); }, 30_000);
afterEach(() => { vi.restoreAllMocks(); vi.useRealTimers(); });
afterAll(async () => { await database?.cleanup(); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Performance", issuePrefix: `P${randomUUID().slice(0, 7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process" }).returning();
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Work" }).returning();
return { company, agent, project };
}
function trace() {
const queries: string[] = [];
const traced = drizzle(db.$client, { logger: { logQuery(query) { queries.push(query); } } }) as unknown as Db;
return { traced, ledgerReads: () => queries.filter(query => query.startsWith("select") && query.includes('from "cost_events"')) };
}
it("reads health while a writer holds the company lock and keeps counts/items in one snapshot", async () => {
const f = await fixture();
const writer = await db.$client.reserve();
await writer`begin`;
try {
await writer`select id from companies where id = ${f.company.id} for no key update`;
const original = db.transaction.bind(db);
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
// A lock attempt fails promptly. There is no elapsed-time pass/fail assertion.
await tx.execute(sql`set local lock_timeout = '200ms'`);
const execute = tx.execute.bind(tx);
let counted = false;
vi.spyOn(tx, "execute").mockImplementation(query => {
const result = execute(query);
const executeRaw = result.execute.bind(result);
vi.spyOn(result, "execute").mockImplementation(async () => {
const rows = await executeRaw();
if (!counted) {
counted = true;
// Commit a pending row between health's counts and item list.
await writer`insert into heartbeat_runs (company_id,agent_id,status,cost_accounting_pending)
values (${f.company.id},${f.agent.id},'failed',true)`;
await writer`commit`;
}
return rows;
});
return result;
});
return callback(tx);
}, config));
expect(await accountingIntegrityService(db).health(f.company.id)).toMatchObject({ pendingRunCount: 0, items: [] });
expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
transaction.mockRestore();
expect(await accountingIntegrityService(db).health(f.company.id)).toMatchObject({ pendingRunCount: 1, items: [expect.objectContaining({ state: "retryable" })] });
} finally { await writer`rollback`; writer.release(); }
// Remove the fixture's pending debt before the global recovery tests.
await db.execute(sql`delete from heartbeat_runs where company_id = ${f.company.id}`);
await expect(accountingIntegrityService(db).health(randomUUID())).rejects.toThrow("Company not found");
});
it("keeps summary incompleteness consistent when an unpriced run settles between reads", async () => {
const f = await fixture();
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
status: "failed", costAccountingPending: true, finishedAt: new Date() }).returning();
const writer = await db.$client.reserve();
await writer`begin`;
try {
await writer`select id from companies where id = ${f.company.id} for no key update`;
const original = db.transaction.bind(db);
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
await tx.execute(sql`set local lock_timeout = '200ms'`);
const select = tx.select.bind(tx);
vi.spyOn(tx, "select").mockImplementation((...args: Parameters<typeof tx.select>) => {
const query = select(...args);
const from = query.from.bind(query);
vi.spyOn(query, "from").mockImplementation(table => {
const result = from(table);
if (table === costEvents) {
const execute = result.execute.bind(result);
vi.spyOn(result, "execute").mockImplementation(async () => {
const rows = await execute();
await writer`insert into cost_events (company_id,agent_id,provider,model,cost_cents,cost_status,billing_type,occurred_at)
values (${f.company.id},${f.agent.id},'fixture','fixture',0,'unpriced','metered_api',now())`;
await writer`update heartbeat_runs set cost_accounting_pending=false where id=${run.id}`;
await writer`commit`;
return rows;
});
}
return result;
});
return query;
});
return callback(tx);
}, config));
expect(await costService(db).summary(f.company.id)).toMatchObject({ pendingRunCount: 1, unpricedEventCount: 0, pricingComplete: false });
transaction.mockRestore();
expect(await costService(db).summary(f.company.id)).toMatchObject({ pendingRunCount: 0, unpricedEventCount: 1, pricingComplete: false });
} finally { await writer`rollback`; writer.release(); }
});
it("shares receipt replay across overlapping ticks and releases the guard after replay failure", async () => {
const reconcile = createCostAccountingReconciler(db);
const gate = deferred();
const read = vi.spyOn(fs, "readdir").mockImplementationOnce(async () => { await gate.promise; throw new Error("Spool unavailable"); });
const first = reconcile(), second = reconcile();
expect(second).toBe(first);
expect(read).toHaveBeenCalledTimes(1);
const settled = Promise.allSettled([first, second]);
gate.resolve();
expect(await settled).toEqual([expect.objectContaining({ status: "rejected" }), expect.objectContaining({ status: "rejected" })]);
read.mockRestore();
expect(await reconcile()).toMatchObject({ scanned: 0, accounted: 0 });
});
it("keeps the guard through budget delivery and retries after reconciliation failure", async () => {
const f = await fixture();
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board");
await costService(db).createEvent(f.company.id, { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: 1, occurredAt: new Date() });
const entered = deferred(), gate = deferred();
const cancelWorkForScope = vi.fn(async () => { entered.resolve(); await gate.promise; });
const reconcile = createCostAccountingReconciler(db, { cancelWorkForScope });
const first = reconcile();
try {
await entered.promise;
expect(reconcile()).toBe(first);
expect(cancelWorkForScope).toHaveBeenCalledTimes(1);
} finally { gate.resolve(); }
await first;
const select = vi.spyOn(db, "select").mockImplementationOnce(() => { throw new Error("Database unavailable"); });
await expect(reconcile()).rejects.toThrow("Database unavailable");
select.mockRestore();
const next = reconcile(); expect(next).not.toBe(first);
await next;
});
it("uses one ledger observation per overview policy and per admission policy", async () => {
const f = await fixture();
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100 }, "board");
const { traced, ledgerReads } = trace();
expect((await budgetService(traced).overview(f.company.id)).policies[0]).toMatchObject({ observedAmountExact: "0.0000000", status: "ok" });
expect(ledgerReads()).toHaveLength(1);
await budgetService(traced).getInvocationBlock(f.company.id, f.agent.id);
expect(ledgerReads()).toHaveLength(2);
});
it("shares a scan across mixed policy scopes/windows without trusting monthly projections", async () => {
const f = await fixture(), foreign = await fixture();
const [other] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning();
const values = [
{ scopeType: "company", scopeId: f.company.id, windowKind: "calendar_month_utc", amount: 4 },
{ scopeType: "agent", scopeId: f.agent.id, windowKind: "lifetime", amount: 10 },
{ scopeType: "project", scopeId: f.project.id, windowKind: "lifetime", amount: 9 },
{ scopeType: "agent", scopeId: other.id, windowKind: "calendar_month_utc", amount: 1 },
];
await db.insert(budgetPolicies).values(values.map(value => ({ ...value, companyId: f.company.id })));
const base = { provider: "fixture", model: "fixture", costCents: 3, occurredAt: new Date() };
const [event] = await db.insert(costEvents).values([
{ ...base, companyId: f.company.id, agentId: f.agent.id, projectId: f.project.id, costCents: 7, occurredAt: new Date("2000-01-01") },
{ ...base, companyId: f.company.id, agentId: f.agent.id, projectId: f.project.id },
{ ...base, companyId: f.company.id, agentId: other.id },
{ ...base, companyId: foreign.company.id, agentId: foreign.agent.id, costCents: 999 },
]).returning();
await db.update(companies).set({ spentMonthlyCents: 0 }).where(eq(companies.id, f.company.id));
const { traced, ledgerReads } = trace();
await withAccountingTransaction(traced, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
expect(ledgerReads()).toHaveLength(1);
const incidents = await db.execute<{ scope_type: string; amount_observed: string }>(sql`select scope_type,amount_observed::text from budget_incidents where company_id = ${f.company.id} order by scope_type`);
expect(incidents).toEqual([
{ scope_type: "agent", amount_observed: "10.0000000" },
{ scope_type: "company", amount_observed: "6.0000000" },
{ scope_type: "project", amount_observed: "10.0000000" },
]);
});
it("keeps unpriced and pending-run guards scoped when batching native recovery observations", async () => {
const f = await fixture();
const [other] = await db.insert(agents).values({ companyId: f.company.id, name: "Other", role: "engineer", adapterType: "process" }).returning();
await db.insert(budgetPolicies).values([
{ companyId: f.company.id, scopeType: "company", scopeId: f.company.id, windowKind: "calendar_month_utc", amount: 100 },
{ companyId: f.company.id, scopeType: "agent", scopeId: f.agent.id, windowKind: "lifetime", amount: 100 },
{ companyId: f.company.id, scopeType: "project", scopeId: f.project.id, windowKind: "lifetime", amount: 100 },
]);
const [issue] = await db.insert(issues).values({ companyId: f.company.id, title: "Recover" }).returning();
const [native] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id,
status: "failed", runtimeMode: "native", nativeIssueId: issue.id, costAccountingPending: true, finishedAt: new Date() }).returning();
await db.insert(nativeRunFinalizations).values({ companyId: f.company.id, runId: native.id, issueId: issue.id, phase: "retryable_failure" });
const base = { companyId: f.company.id, provider: "fixture", model: "fixture", costCents: 0, occurredAt: new Date() };
const [event] = await db.insert(costEvents).values([
{ ...base, agentId: f.agent.id, projectId: f.project.id },
{ ...base, agentId: other.id, projectId: f.project.id, occurredAt: new Date("2000-01-01"), costStatus: "unpriced" },
]).returning();
await withAccountingTransaction(db, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
const overview = await budgetService(db).overview(f.company.id);
expect(overview.activeIncidents.map(row => row.scopeType)).toEqual(["project"]);
expect(overview.policies).toEqual(expect.arrayContaining([
expect.objectContaining({ scopeType: "company", status: "ok", pendingRunCount: 1, unpricedEventCount: 0 }),
expect.objectContaining({ scopeType: "agent", status: "ok", pendingRunCount: 1, unpricedEventCount: 0 }),
expect.objectContaining({ scopeType: "project", status: "hard_stop", pendingRunCount: 0, unpricedEventCount: 1 }),
]));
await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "failed",
costAccountingPending: true, finishedAt: new Date(), usageJson: { ledgerScope: { projectId: f.project.id } } });
await withAccountingTransaction(db, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
expect((await budgetService(db).overview(f.company.id)).activeIncidents.map(row => row.scopeType).sort()).toEqual(["agent", "company", "project"]);
});
it("keeps incident amounts and overview windows tied to the observed UTC month across midnight", async () => {
const f = await fixture();
await db.insert(budgetPolicies).values({ companyId: f.company.id, scopeType: "company", scopeId: f.company.id,
windowKind: "calendar_month_utc", amount: 1 });
const [event] = await db.insert(costEvents).values({ companyId: f.company.id, agentId: f.agent.id,
provider: "fixture", model: "fixture", costCents: 1, occurredAt: new Date("2026-01-31T12:00:00Z") }).returning();
const traced = drizzle(db.$client, { logger: { logQuery(query) {
if (query.startsWith("select") && query.includes('from "cost_events"')) vi.setSystemTime(new Date("2026-02-01T00:00:00Z"));
} } }) as unknown as Db;
vi.useFakeTimers({ toFake: ["Date"] });
vi.setSystemTime(new Date("2026-01-31T23:59:59.999Z"));
await withAccountingTransaction(traced, f.company.id, tx => budgetServiceInTransaction(tx).evaluateCostEvent(event));
const incident = (await budgetService(db).overview(f.company.id)).activeIncidents[0];
expect(incident).toMatchObject({ amountObserved: 1, windowStart: new Date("2026-01-01T00:00:00Z"), windowEnd: new Date("2026-02-01T00:00:00Z") });
vi.setSystemTime(new Date("2026-01-31T23:59:59.999Z"));
expect((await budgetService(traced).overview(f.company.id)).policies[0]).toMatchObject({
observedAmountExact: "1.0000000", windowStart: new Date("2026-01-01T00:00:00Z"), windowEnd: new Date("2026-02-01T00:00:00Z"),
});
// The next operation uses February and releases the prior month's pause.
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
});
it.each(["inspection", "invoice comparison"])("does not take the company write lock during %s", async kind => {
const f = await fixture();
const invoice = await billingReconciliationService(db).importInvoice(f.company.id, { biller: "fixture", externalId: "invoice", currency: "USD",
lines: [{ externalId: "line", amountCents: "1", occurredAt: new Date().toISOString() }] }, "board");
const writer = await db.$client.reserve();
await writer`begin`;
try {
await writer`select id from companies where id = ${f.company.id} for no key update`;
const original = db.transaction.bind(db);
const transaction = vi.spyOn(db, "transaction").mockImplementation((callback, config) => original(async tx => {
await tx.execute(sql`set local lock_timeout = '200ms'`);
return callback(tx);
}, config));
if (kind === "inspection") expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
else expect((await billingReconciliationService(db).reconcile(f.company.id, invoice.id)).lines).toHaveLength(1);
expect(transaction).toHaveBeenCalledWith(expect.any(Function), { isolationLevel: "repeatable read", accessMode: "read only" });
} finally { await writer`rollback`; writer.release(); }
});
});
@@ -0,0 +1,110 @@
import { spawn } from "node:child_process";
import { randomUUID } from "node:crypto";
import { mkdtemp, rm } from "node:fs/promises";
import { createConnection, createServer, type Socket } from "node:net";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { eq } from "drizzle-orm";
import { agents, companies, costEvents, createDb, heartbeatRuns } from "@paperclipai/db";
import { costService } from "../services/costs.js";
import { accountingIntegrityService } from "../services/accounting-integrity.js";
import { replayUsageReceipts } from "../services/usage-receipts.js";
import { accountRunCost } from "../services/run-cost-accounting.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
const root = fileURLToPath(new URL("../../..", import.meta.url));
(support.supported && process.platform !== "win32" ? describe : describe.skip)("accounting across process and connection failures", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
beforeAll(async () => { database = await startEmbeddedPostgresTestDatabase("accounting-processes-"); db = createDb(database.connectionString); }, 30_000);
afterAll(async () => { await database?.cleanup(); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Process", issuePrefix: `P${randomUUID().slice(0,7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", adapterType: "process", status: "idle" }).returning();
return { company, agent };
}
function worker(mode: string, input: unknown) {
return spawn(process.execPath, ["cli/node_modules/tsx/dist/cli.mjs", "server/src/__tests__/fixtures/accounting-writer.ts", mode, JSON.stringify(input)], {
cwd: root, detached: true, stdio: ["ignore", "pipe", "pipe"],
env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, PAPERCLIP_ACCOUNTING_TEST_DATABASE: database.connectionString, PAPERCLIP_TELEMETRY_DISABLED: "1" },
});
}
it("deduplicates the same receipt from independent server processes", async () => {
const f = await fixture();
const receipt = { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: "0.1234567", idempotencyKey: "shared-key", occurredAt: new Date().toISOString() };
const children = Array.from({ length: 4 }, () => worker("write", { companyId: f.company.id, receipt }));
try {
await Promise.all(children.map(child => new Promise<void>((resolve, reject) => {
let log = ""; child.stdout.on("data", d => log += d); child.stderr.on("data", d => log += d);
child.once("error", reject); child.once("exit", code => code === 0 && log.includes("WRITES_COMMITTED") ? resolve() : reject(new Error(log)));
})));
expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))).toHaveLength(1);
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
} finally { for (const child of children) if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid!, "SIGKILL"); }
}, 45_000);
it("recovers a receipt after SIGKILL before its first database persistence", async () => {
const f = await fixture(); const directory = await mkdtemp(path.join(tmpdir(), "accounting-early-kill-"));
const [run] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, status: "running", costAccountingPending: true, usageJson: { accountingReceiptReady: false } }).returning();
const child = worker("receipt", { companyId: f.company.id, runId: run.id, directory });
try {
await new Promise<void>((resolve, reject) => {
let log = ""; child.stdout.on("data", data => { log += data; if (log.includes("RECEIPT_DURABLE")) resolve(); });
child.stderr.on("data", data => log += data); child.once("error", reject); child.once("exit", () => reject(new Error(`Exited before checkpoint: ${log}`)));
});
const stopped = new Promise(resolve => child.once("exit", resolve)); process.kill(-child.pid!, "SIGKILL"); await stopped;
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toEqual([]);
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date() }).where(eq(heartbeatRuns.id, run.id));
expect(await replayUsageReceipts(db, directory)).toEqual({ replayed: 1, failed: 0 });
expect(await accountRunCost(db, run.id)).toBe(true);
expect(await accountRunCost(db, run.id)).toBe(false);
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("1.2345678");
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
} finally {
if (child.exitCode === null && child.signalCode === null) process.kill(-child.pid!, "SIGKILL");
await rm(directory, { recursive: true, force: true });
}
}, 45_000);
it("retries safely when PostgreSQL commits but its acknowledgement is lost", async () => {
const f = await fixture(); const target = new URL(database.connectionString); const sockets = new Set<Socket>(); let dropped = false;
// Parse PostgreSQL server frames, suppress precisely CommandComplete(COMMIT),
// then break TCP. The database has committed while the writer sees failure.
const proxy = createServer(client => {
const upstream = createConnection({ host: target.hostname, port: Number(target.port) }); sockets.add(client); sockets.add(upstream);
client.on("error", () => {}); upstream.on("error", () => client.destroy()); client.pipe(upstream);
let pending = Buffer.alloc(0);
upstream.on("data", chunk => {
pending = Buffer.concat([pending, chunk]);
while (pending.length >= 5) {
const size = pending.readInt32BE(1) + 1;
if (size < 5 || size > 16 * 1024 * 1024) { client.destroy(); upstream.destroy(); return; }
if (pending.length < size) return;
const message = pending.subarray(0, size); pending = pending.subarray(size);
if (!dropped && message[0] === 67 && message.subarray(5).toString() === "COMMIT\0") {
dropped = true; client.destroy(); upstream.destroy(); return;
}
client.write(message);
}
});
client.on("close", () => { sockets.delete(client); upstream.destroy(); });
upstream.on("close", () => { sockets.delete(upstream); client.destroy(); });
});
await new Promise<void>(resolve => proxy.listen(0, "127.0.0.1", resolve));
const forwarded = new URL(database.connectionString); forwarded.hostname = "127.0.0.1"; forwarded.port = String((proxy.address() as { port: number }).port); forwarded.searchParams.set("sslmode", "disable");
const writer = createDb(forwarded.toString());
const receipt = { agentId: f.agent.id, provider: "fixture", model: "fixture", costCents: "0.9999999", idempotencyKey: "ambiguous-commit", occurredAt: new Date() };
try {
await expect(costService(writer).createEvent(f.company.id, receipt)).rejects.toThrow();
expect(dropped).toBe(true);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(1);
await costService(writer).createEvent(f.company.id, receipt);
expect((await costService(db).summary(f.company.id)).spendCentsExact).toBe("0.9999999");
expect((await accountingIntegrityService(db).inspect(f.company.id)).findings).toEqual([]);
} finally {
await writer.$client.end({ timeout: 1 }); for (const socket of sockets) socket.destroy(); await new Promise<void>(resolve => proxy.close(() => resolve()));
}
}, 30_000);
});
@@ -0,0 +1,741 @@
import { upsertBudgetPolicySchema } from "@paperclipai/shared";
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
import { and, eq, sql } from "drizzle-orm";
import { activityLog, agentRuntimeState, agentWakeupRequests, agents, approvals, budgetIncidents, budgetPolicies, companies, completionContracts, costEvents, createDb, financeEvents, goals, heartbeatRuns, issues, nativeRunResults, projects } from "@paperclipai/db";
import { reserveRunBudget } from "../services/budget-reservations.js";
import { createRunDispatch } from "../modules/run-dispatch/index.js";
import { costService, createCostEventInTransaction } from "../services/costs.js";
import { budgetService, withCurrentBudgetEnforcement, type BudgetEnforcementScope } from "../services/budgets.js";
import { accountRunCost, reconcileRunCosts } from "../services/run-cost-accounting.js";
import { financeService } from "../services/finance.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
import { hoistModuleGraph } from "./helpers/hoist-module-graph.js";
import { NativeRunCoordinatorStore } from "../services/native-runtime/native-run-coordinator-store.js";
import { CONTROL_PLANE_CONFORMANCE_RESULT, CONTROL_PLANE_CONFORMANCE_TERMINAL } from "../vendor/paperclip-runner/testing.js";
const support = await getEmbeddedPostgresTestSupport();
const databaseDescribe = support.supported ? describe : describe.skip;
databaseDescribe("cost accounting reliability (PostgreSQL)", () => {
const services = hoistModuleGraph(() => {}, async () => {
const [agentModule, companyModule, approvalModule] = await Promise.all([
import("../services/agents.js"), import("../services/companies.js"), import("../services/approvals.js"),
]);
return { agentService: agentModule.agentService, companyService: companyModule.companyService, approvalService: approvalModule.approvalService };
});
let db: ReturnType<typeof createDb>;
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("paperclip-cost-reliability-");
db = createDb(database.connectionString);
}, 30_000);
afterAll(async () => { await database?.cleanup(); });
async function fixture() {
const [company] = await db.insert(companies).values({ name: "Accounting", issuePrefix: `T${randomUUID().slice(0, 7)}` }).returning();
const [agent] = await db.insert(agents).values({ companyId: company.id, name: "Worker", role: "engineer", status: "active", adapterType: "codex_local" }).returning();
const [project] = await db.insert(projects).values({ companyId: company.id, name: "Project", status: "in_progress" }).returning();
const [issue] = await db.insert(issues).values({ companyId: company.id, projectId: project.id, title: "Work" }).returning();
const [goal] = await db.insert(goals).values({ companyId: company.id, title: "Goal" }).returning();
const [run] = await db.insert(heartbeatRuns).values({ companyId: company.id, agentId: agent.id, invocationSource: "on_demand" }).returning();
const receipt = { agentId: agent.id, provider: "test", model: "test", billingType: "metered_api", costCents: 100, occurredAt: new Date() };
return { company, agent, project, issue, goal, run, receipt };
}
it.each([false, true])("deletes linked accounting records before company runs (concurrent writer: %s)", async (concurrentWriter) => {
const f = await fixture();
const other = await fixture();
await costService(db).createEvent(other.company.id, other.receipt);
const charge = await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id, issueId: f.issue.id });
await financeService(db).createEvent(f.company.id, {
eventKind: "platform_fee", direction: "debit", biller: "paperclip", amountCents: 5, currency: "USD", occurredAt: new Date(),
heartbeatRunId: f.run.id, costEventId: charge.id,
});
const [policy] = await db.insert(budgetPolicies).values({ companyId: f.company.id, scopeType: "company", scopeId: f.company.id,
metric: "billed_cents", windowKind: "calendar_month_utc", amount: 1000 }).returning();
await db.insert(budgetIncidents).values({ companyId: f.company.id, policyId: policy.id, scopeType: "company", scopeId: f.company.id,
metric: policy.metric, windowKind: policy.windowKind, windowStart: new Date(), windowEnd: new Date(),
thresholdType: "warning", amountLimit: 1000, amountObserved: 100 });
let removing: Promise<{ value?: unknown; error?: unknown }> | undefined;
try {
if (concurrentWriter) {
await db.transaction(async tx => {
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
removing = services.value.companyService(db).remove(f.company.id).then(value => ({ value }), error => ({ error }));
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and wait_event_type = 'Lock' and query like '%companies%for no key update%'`);
expect(waiting.length).toBeGreaterThan(0);
}, { timeout: 3000 });
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id, { ...f.receipt, heartbeatRunId: f.run.id }, []);
});
expect(await removing!).toMatchObject({ value: { id: f.company.id } });
} else expect(await services.value.companyService(db).remove(f.company.id)).toMatchObject({ id: f.company.id });
expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(0);
for (const table of [heartbeatRuns, costEvents, financeEvents, budgetPolicies, budgetIncidents]) {
expect(await db.select().from(table).where(eq(table.companyId, f.company.id))).toHaveLength(0);
}
expect((await costService(db).summary(other.company.id)).spendCents).toBe(100);
expect(await services.value.companyService(db).remove(f.company.id)).toBeNull();
} finally { await removing; }
});
it("serializes agent deletion before child locks and preserves a concurrent settled charge", async () => {
const f = await fixture();
let removing: Promise<{ value?: unknown; error?: unknown }> | undefined;
try {
await db.transaction(async tx => {
await tx.execute(sql`set local lock_timeout = '1s'`);
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update");
removing = services.value.agentService(db).remove(f.agent.id).then(value => ({ value }), error => ({ error }));
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select query from pg_stat_activity where datname = current_database()
and wait_event_type = 'Lock' and (query like '%companies%for no key update%' or query like 'delete from "heartbeat_runs"%')`);
expect(waiting.length).toBeGreaterThan(0);
}, { timeout: 3000 });
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id,
{ ...f.receipt, heartbeatRunId: f.run.id }, []);
});
// Existing charge FKs prohibit hard deletion. Preserve the settled charge
// and the atomic deletion failure, rather than deadlocking its writer.
expect(await removing!).toMatchObject({ error: { cause: { code: "23503" } } });
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
expect(await db.select().from(agents).where(eq(agents.id, f.agent.id))).toHaveLength(1);
expect(await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id))).toHaveLength(1);
} finally { await removing; }
});
it("preserves an in-flight native result when company deletion reaches its existing foreign-key restriction", async () => {
const f = await fixture();
await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id });
const contractHash = randomUUID();
const [contract] = await db.insert(completionContracts).values({ companyId: f.company.id, issueId: f.issue.id,
revision: 1, schemaVersion: "paperclip.completion-contract.v1", policyVersion: "test",
risk: "standard", completionAuthority: "server_arbiter", incompleteCriteriaPolicy: "preserve_non_terminal",
contractJson: {}, canonicalSha256: contractHash, createdByActorType: "system", createdByActorId: "test" }).returning();
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id,
completionContractId: contract.id, completionContractSha256: contractHash }).where(eq(heartbeatRuns.id, f.run.id));
let removing: Promise<{ value?: unknown; error?: unknown }> | undefined;
try {
await db.transaction(async tx => {
await tx.execute(sql`set local lock_timeout = '1s'`);
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update");
removing = services.value.companyService(db).remove(f.company.id).then(value => ({ value }), error => ({ error }));
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and wait_event_type = 'Lock' and query like 'delete from "heartbeat_runs"%'`);
expect(waiting.length).toBeGreaterThan(0);
}, { timeout: 3000 });
const store = new NativeRunCoordinatorStore(tx as unknown as ReturnType<typeof createDb>, {
companyId: f.company.id, issueId: f.issue.id, runId: f.run.id, agentId: f.agent.id,
normalizedSessionId: randomUUID(), runnerSourceInstanceId: randomUUID(),
completionContractId: contract.id, completionContractSha256: contractHash,
completionContractRevision: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.contractRevision,
completionContractCriterionIds: CONTROL_PLANE_CONFORMANCE_RESULT.completionClaim.criteria.map(c => c.criterionId),
});
expect(await store.completeRun({ result: CONTROL_PLANE_CONFORMANCE_RESULT, terminal: CONTROL_PLANE_CONFORMANCE_TERMINAL }))
.toMatchObject({ disposition: "committed" });
});
// Native artifacts already prevent hard deletion via restrictive FKs.
// Preserve that atomic failure; the accounting lock must not introduce a
// deadlock that aborts the native result instead (Postgres 40P01).
expect(await removing!).toMatchObject({ error: { cause: { code: "23503" } } });
expect(await db.select().from(nativeRunResults).where(eq(nativeRunResults.companyId, f.company.id))).toHaveLength(1);
expect(await db.select().from(companies).where(eq(companies.id, f.company.id))).toHaveLength(1);
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
} finally { await removing; }
});
it("counts cache reads once across mixed historical and receipt-backed reports without changing charges", async () => {
const f = await fixture();
const costs = costService(db);
const scope = { agentId: f.agent.id, issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id,
provider: "openai", biller: "openai", model: "gpt-test", occurredAt: new Date() };
// These two layouts both describe 100 input (80 cached) and 10 output.
for (const billingType of ["metered_api", "subscription_included"]) {
await db.insert(costEvents).values({ companyId: f.company.id, ...scope, billingType,
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
await costs.createEvent(f.company.id, { ...scope, billingType,
inputTokens: 20, cachedInputTokens: 80, outputTokens: 10, costCents: 11 });
}
const [provider, models] = await Promise.all([costs.byProvider(f.company.id), costs.byAgentModel(f.company.id)]);
for (const rows of [provider, models]) {
expect(rows).toHaveLength(2);
for (const row of rows) {
expect(row).toMatchObject({ inputTokens: 40, cachedInputTokens: 160, outputTokens: 20, costCents: 18 });
expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(220);
}
}
const [agent] = await costs.byAgent(f.company.id);
const [biller] = await costs.byBiller(f.company.id);
const [project] = await costs.byProject(f.company.id);
const issue = await costs.issueTreeSummary(f.company.id, f.issue.id);
const windows = await costs.windowSpend(f.company.id);
expect(windows).toHaveLength(3);
for (const row of [agent, biller, project, issue, ...windows]) {
expect(row).toMatchObject({ inputTokens: 80, cachedInputTokens: 320, outputTokens: 40, costCents: 36 });
expect(row.inputTokens + row.cachedInputTokens + row.outputTokens).toBe(440);
}
for (const row of [agent, biller, provider.find(row => row.billingType === "subscription_included")!]) {
expect(row).toMatchObject({ subscriptionInputTokens: 40, subscriptionCachedInputTokens: 160, subscriptionOutputTokens: 20 });
}
// Reading reports never rewrites historical evidence or retroactively bills.
const stored = await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id));
expect(stored.filter(row => row.receiptHash === null).map(row => row.inputTokens)).toEqual([100, 100]);
expect((await costs.summary(f.company.id)).spendCents).toBe(36);
// Legacy Anthropic input was already exclusive; retain its cache semantics.
await db.insert(costEvents).values({ companyId: f.company.id, ...scope, provider: "anthropic", billingType: "metered_api",
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 0 });
expect((await costs.byProvider(f.company.id)).find(row => row.provider === "anthropic")).toMatchObject({ inputTokens: 100, cachedInputTokens: 80, outputTokens: 10 });
});
it.each([
["openai/gpt-test", 100], ["openai/vendor/model", 100],
["gpt-test", 20], ["different/model", 20], ["openai/", 20],
] as const)("preserves historical input semantics for %s", async (model, inputTokens) => {
const f = await fixture(), costs = costService(db);
await db.insert(costEvents).values({ companyId: f.company.id, agentId: f.agent.id,
issueId: f.issue.id, projectId: f.project.id, heartbeatRunId: f.run.id,
provider: "openai", biller: "openai", model, occurredAt: new Date(),
inputTokens: 100, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
const rows = [
...await costs.byAgent(f.company.id), ...await costs.byProvider(f.company.id),
...await costs.byAgentModel(f.company.id), ...await costs.byBiller(f.company.id),
...await costs.byProject(f.company.id), ...await costs.windowSpend(f.company.id),
await costs.issueTreeSummary(f.company.id, f.issue.id),
...(await costs.byUser(f.company.id)).rows,
];
for (const row of rows) expect(row).toMatchObject({ inputTokens, cachedInputTokens: 80, outputTokens: 10, costCents: 7 });
expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))[0].inputTokens).toBe(100);
});
it.each((["company", "agent", "project"] as const).flatMap(scopeType => [0, 500].map(previousAmount => ({ scopeType, previousAmount }))))("edits $scopeType limits without confusing saved zero and disabled positive amounts ($previousAmount)", async ({ scopeType, previousAmount }) => {
const f = await fixture(), cancel = vi.fn(async () => {});
const budgets = budgetService(db, { cancelWorkForScope: cancel });
const scopeId = scopeType === "company" ? f.company.id : scopeType === "agent" ? f.agent.id : f.project.id;
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: previousAmount, isActive: false }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, projectId: f.project.id, costCents: 100 });
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ amount: 0, isActive: false });
const updated = await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 50 }, "board");
expect(updated).toMatchObject({ isActive: previousAmount === 0, paused: previousAmount === 0 });
expect(cancel).toHaveBeenCalledTimes(previousAmount === 0 ? 1 : 0);
expect((await budgets.overview(f.company.id)).activeIncidents).toHaveLength(previousAmount === 0 ? 1 : 0);
});
it.each(["issueId", "projectId", "goalId", "heartbeatRunId"] as const)("rejects a foreign-company %s without writing anything", async (field) => {
const a = await fixture(); const b = await fixture();
const foreignIds = { issueId: b.issue.id, projectId: b.project.id, goalId: b.goal.id, heartbeatRunId: b.run.id };
await expect(costService(db).createEvent(a.company.id, { ...a.receipt, [field]: foreignIds[field] })).rejects.toThrow(/company/);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, a.company.id))).toHaveLength(0);
});
it("allows admission inside a caller transaction holding a company foreign-key lock", async () => {
const f = await fixture();
// The recovery service dispatches through another connection while its
// transaction still owns FK KEY SHARE locks on the company. Accounting
// must serialize without requesting the conflicting FOR UPDATE mode.
await db.transaction(async (tx) => {
await tx.insert(issues).values({ companyId: f.company.id, title: "Concurrent recovery" });
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
});
}, 5000);
it.each([false, true])("takes the attributed issue lock before the run during native recovery (attribution changes: %s)", async (changeAttribution) => {
const f = await fixture();
const [nextIssue] = await db.insert(issues).values({ companyId: f.company.id, title: "New attribution" }).returning();
const usage = { provider: "openai", billingType: "metered_api", costUsd: 0.5, inputTokens: 10, outputTokens: 2,
accountingReceiptReady: true, ledgerScope: { issueId: f.issue.id } };
await db.update(heartbeatRuns).set({ runtimeMode: "native", nativeIssueId: f.issue.id, status: "failed", finishedAt: new Date(),
costAccountingPending: true, usageJson: usage }).where(eq(heartbeatRuns.id, f.run.id));
let accounting: Promise<{ value?: boolean; error?: unknown }> | undefined;
try {
await db.transaction(async tx => {
// recordRetryableFailure holds the issue while updating its run.
await tx.select().from(issues).where(eq(issues.id, f.issue.id)).for("update");
accounting = accountRunCost(db, f.run.id).then(value => ({ value }), error => ({ error }));
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and wait_event_type = 'Lock' and query like '%issues%for key share%'`);
expect(waiting.length).toBeGreaterThan(0);
}, { timeout: 3000 });
await tx.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)).for("update", { noWait: true });
await tx.update(heartbeatRuns).set({ resultJson: { prpRunTerminalState: "failed" },
...(changeAttribution ? { usageJson: { ...usage, ledgerScope: { issueId: nextIssue.id } } } : {}),
}).where(eq(heartbeatRuns.id, f.run.id));
});
if (changeAttribution) {
expect((await accounting!).error).toMatchObject({ status: 409, message: "Run cost attribution changed. Retry accounting." });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
expect(await accountRunCost(db, f.run.id)).toBe(true);
} else expect(await accounting!).toEqual({ value: true });
const charges = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
expect(charges).toHaveLength(1);
expect(charges[0]).toMatchObject({ issueId: changeAttribution ? nextIssue.id : f.issue.id, costCents: 50 });
} finally { await accounting; }
});
it("takes the company lock before retry issue/run locks during an attributed cost write", async () => {
const f = await fixture();
await db.update(issues).set({ status: "in_progress", assigneeAgentId: f.agent.id }).where(eq(issues.id, f.issue.id));
await db.update(heartbeatRuns).set({ status: "scheduled_retry", scheduledRetryAt: new Date(0), scheduledRetryReason: "workspace_busy",
contextSnapshot: { issueId: f.issue.id, projectId: f.project.id },
}).where(eq(heartbeatRuns.id, f.run.id));
let promotion: ReturnType<ReturnType<typeof createRunDispatch>["promoteScheduledRetry"]> | undefined;
await db.transaction(async (tx) => {
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
promotion = createRunDispatch(db).promoteScheduledRetry({ companyId: f.company.id, runId: f.run.id });
// Observe the actual blocked company-lock query before checking the issue;
// no timing guess or timeout is used as proof of lock ordering.
await vi.waitFor(async () => {
const result = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and pid <> pg_backend_pid() and wait_event_type = 'Lock' and query like '%companies%for no key update%'`);
expect(result.length).toBeGreaterThan(0);
}, { timeout: 3000 });
await tx.execute(sql`select id from issues where id = ${f.issue.id} for key share nowait`);
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id,
{ ...f.receipt, issueId: f.issue.id, projectId: f.project.id }, []);
});
expect((await promotion)?.outcome).toBe("promoted");
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
}, 10000);
it("locks company before activating an approved hire while a receipt writer is active", async () => {
const f = await fixture();
await db.update(agents).set({ status: "pending_approval" }).where(eq(agents.id, f.agent.id));
const [approval] = await db.insert(approvals).values({ companyId: f.company.id, type: "hire_agent", status: "pending",
payload: { agentId: f.agent.id, budgetMonthlyCents: 1000 } }).returning();
let approving: Promise<unknown> | undefined;
await db.transaction(async tx => {
await tx.select().from(companies).where(eq(companies.id, f.company.id)).for("no key update");
approving = services.value.approvalService(db).approve(approval.id, "board");
await vi.waitFor(async () => {
const waiting = await db.execute(sql`select pid from pg_stat_activity where datname = current_database()
and pid <> pg_backend_pid() and wait_event_type = 'Lock' and query like '%companies%for no key update%'`);
expect(waiting.length).toBeGreaterThan(0);
}, { timeout: 3000 });
await tx.execute(sql`select id from agents where id = ${f.agent.id} for no key update nowait`);
await createCostEventInTransaction(tx as unknown as ReturnType<typeof createDb>, f.company.id, f.receipt, []);
});
await expect(approving).resolves.toMatchObject({ applied: true });
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("idle");
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
}, 10000);
it("deduplicates concurrent retries and rejects changed receipt content", async () => {
const f = await fixture();
const receipt = { ...f.receipt, idempotencyKey: "provider-receipt-1" };
const events = await Promise.all(Array.from({ length: 8 }, () => costService(db).createEvent(f.company.id, receipt)));
expect(new Set(events.map((event) => event.id)).size).toBe(1);
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
await expect(costService(db).createEvent(f.company.id, { ...receipt, costCents: 200 })).rejects.toThrow(/different|conflict/i);
// Keys are scoped to the reporting company.
const other = await fixture();
await expect(costService(db).createEvent(other.company.id, { ...other.receipt, idempotencyKey: receipt.idempotencyKey })).resolves.toBeTruthy();
});
it("retains sub-cent spend and enforces the sum, not rounded individual receipts", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1 }, "board");
for (let i = 0; i < 3; i++) await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 0.4 });
expect((await costService(db).summary(f.company.id)).spendCents).toBe(1.2);
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" });
});
it("conserves project totals when a run touches multiple projects and includes unallocated spend", async () => {
const f = await fixture();
const [otherProject] = await db.insert(projects).values({ companyId: f.company.id, name: "Other" }).returning();
const [otherIssue] = await db.insert(issues).values({ companyId: f.company.id, projectId: otherProject.id, title: "Other" }).returning();
await db.insert(activityLog).values([f.issue, otherIssue].map((issue) => ({ companyId: f.company.id, actorType: "agent", actorId: f.agent.id, action: "issue.updated", entityType: "issue", entityId: issue.id, runId: f.run.id })));
await costService(db).createEvent(f.company.id, { ...f.receipt, heartbeatRunId: f.run.id, projectId: f.project.id });
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 30 });
const rows = await costService(db).byProject(f.company.id);
expect(rows.reduce((sum, row) => sum + row.costCents, 0)).toBe(130);
expect(rows.find((row) => row.projectId === f.project.id)?.costCents).toBe(100);
expect(rows.find((row) => row.projectId === null)?.costCents).toBe(30);
});
it("does not enforce inactive policies, and checks both monthly and lifetime limits", async () => {
const f = await fixture(); const budgets = budgetService(db);
await costService(db).createEvent(f.company.id, f.receipt);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1, isActive: false }, "board");
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1000, windowKind: "calendar_month_utc" }, "board");
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50, windowKind: "lifetime" }, "board");
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 2000, windowKind: "calendar_month_utc" }, "board");
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" });
});
it("creates one approval under concurrent evaluation and opens a new incident after a raised budget is reached", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
const event = await costService(db).createEvent(f.company.id, f.receipt);
await Promise.all(Array.from({ length: 8 }, () => budgets.evaluateCostEvent(event)));
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(1);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 150 }, "board");
await costService(db).createEvent(f.company.id, f.receipt);
expect(await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard"), eq(budgetIncidents.status, "open")))).toHaveLength(1);
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(2);
});
it("commits accounting despite cancellation delivery failure and retries delivery safely", async () => {
const f = await fixture();
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
const cancel = vi.fn().mockRejectedValueOnce(new Error("injected cancellation transport failure")).mockResolvedValue(undefined);
const costs = costService(db, { cancelWorkForScope: cancel });
const receipt = { ...f.receipt, idempotencyKey: "retry-after-cancel" };
await expect(costs.createEvent(f.company.id, receipt)).resolves.toBeTruthy();
await costs.createEvent(f.company.id, receipt);
expect(cancel).toHaveBeenCalledTimes(2);
expect((await costs.summary(f.company.id)).spendCents).toBe(100);
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "agent" });
});
it("recovers live scopes despite deleted agent and project policies and an isolated transaction failure", async () => {
const f = await fixture(); const other = await fixture(); const budgets = budgetService(db);
const [removedAgent] = await db.insert(agents).values({ companyId: f.company.id, name: "Removed", adapterType: "process" }).returning();
const [removedProject] = await db.insert(projects).values({ companyId: f.company.id, name: "Removed" }).returning();
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: removedAgent.id, amount: 100 }, "board");
await budgets.upsertPolicy(f.company.id, { scopeType: "project", scopeId: removedProject.id, amount: 100 }, "board");
await db.delete(agents).where(eq(agents.id, removedAgent.id));
await db.delete(projects).where(eq(projects.id, removedProject.id));
for (const live of [f, other]) {
await budgets.upsertPolicy(live.company.id, { scopeType: "agent", scopeId: live.agent.id, amount: 100 }, "board");
await db.update(agents).set({ status: "paused", pauseReason: "budget" }).where(eq(agents.id, live.agent.id));
}
// Restrict the scanned scope list so the injected first transaction fails
// on the stale scope, ahead of both live policies in this real database.
const query = vi.spyOn(db, "selectDistinct").mockReturnValueOnce({ from: () => ({ orderBy: async () => [
{ companyId: f.company.id, scopeType: "project", scopeId: removedProject.id },
{ companyId: f.company.id, scopeType: "agent", scopeId: removedAgent.id },
...[f, other].map(live => ({ companyId: live.company.id, scopeType: "agent", scopeId: live.agent.id })),
] }) } as never);
const transaction = vi.spyOn(db, "transaction").mockRejectedValueOnce(new Error("Injected scope recovery failure"));
try { await budgets.reconcilePolicies(); } finally { query.mockRestore(); transaction.mockRestore(); }
for (const live of [f, other]) {
const [agent] = await db.select().from(agents).where(eq(agents.id, live.agent.id));
expect(agent).toMatchObject({ status: "idle", pauseReason: null });
}
// A subsequent sweep uses actual policy discovery and skips both deleted scopes.
await expect(budgets.reconcilePolicies()).resolves.toBeUndefined();
});
it("recovers terminal run accounting exactly once across simultaneous live and recovery workers", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ status: "timed_out", finishedAt: new Date(), costAccountingPending: true,
usageJson: { inputTokens: 20, cachedInputTokens: 100, outputTokens: 10, costUsd: 0.004, provider: "test", model: "test", billingType: "metered_api", ledgerScope: { issueId: f.issue.id, projectId: f.project.id } },
}).where(eq(heartbeatRuns.id, f.run.id));
await Promise.all([accountRunCost(db, f.run.id), accountRunCost(db, f.run.id), reconcileRunCosts(db)]);
const [totals] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
expect(totals).toMatchObject({ totalInputTokens: 20, totalCachedInputTokens: 100, totalOutputTokens: 10, totalCostCents: 0.4 });
const events = await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id));
expect(events).toHaveLength(1);
expect(events[0]).toMatchObject({ costCents: 0.4, projectId: f.project.id });
const [run] = await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id));
expect(run.costAccountingPending).toBe(false);
expect(run.costAccountedAt).toBeInstanceOf(Date);
// Late terminal metadata may re-arm the pending flag. The durable
// acknowledgement, not the delivery flag, owns exactly-once totals.
await db.update(heartbeatRuns).set({ costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id));
await accountRunCost(db, f.run.id);
expect((await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)))[0].totalCostCents).toBe(0.4);
});
it("rolls back the ledger and counters on accounting failure, retaining a retryable run", async () => {
const f = await fixture(); const other = await fixture();
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, usageJson: {
inputTokens: 10, costUsd: 1, provider: "test", model: "test", ledgerScope: { projectId: other.project.id },
} }).where(eq(heartbeatRuns.id, f.run.id));
await expect(accountRunCost(db, f.run.id)).rejects.toThrow(/company/);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0);
expect(await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id))).toHaveLength(0);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].costAccountingPending).toBe(true);
});
it("rolls back a receipt, counters, and budget incidents if a later runtime write fails", async () => {
const f = await fixture();
await budgetService(db).upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50 }, "board");
await db.update(heartbeatRuns).set({ status: "failed", finishedAt: new Date(), costAccountingPending: true, usageJson: {
inputTokens: 10, costUsd: 1, provider: "test", billingType: "metered_api", model: "test",
} }).where(eq(heartbeatRuns.id, f.run.id));
await db.execute(sql`create function reject_runtime_accounting() returns trigger language plpgsql as $$ begin raise exception 'injected runtime write failure'; end $$`);
await db.execute(sql`create trigger reject_runtime_accounting before insert on agent_runtime_state for each row execute function reject_runtime_accounting()`);
try {
await expect(accountRunCost(db, f.run.id)).rejects.toThrow();
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0);
expect(await db.select().from(budgetIncidents).where(eq(budgetIncidents.companyId, f.company.id))).toHaveLength(0);
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(0);
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ spentMonthlyCents: 0, status: "active" });
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: true, costAccountedAt: null });
} finally {
await db.execute(sql`drop trigger reject_runtime_accounting on agent_runtime_state`);
await db.execute(sql`drop function reject_runtime_accounting()`);
}
await accountRunCost(db, f.run.id);
expect((await costService(db).summary(f.company.id)).spendCents).toBe(100);
});
it("releases an expired monthly budget pause while preserving manual pauses", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 50 }, "board");
const event = await costService(db).createEvent(f.company.id, f.receipt);
const now = new Date();
await db.update(costEvents).set({ occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), 0)) }).where(eq(costEvents.id, event.id));
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ status: "idle", pauseReason: null });
await db.update(agents).set({ status: "paused", pauseReason: "manual" }).where(eq(agents.id, f.agent.id));
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 200 }, "board");
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0]).toMatchObject({ status: "paused", pauseReason: "manual" });
});
it("uses live observed spend for incident resolution and synchronizes company budgets", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 50 }, "board");
await costService(db).createEvent(f.company.id, f.receipt);
const [incident] = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard")));
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 25 });
await expect(budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 125 }, "board")).rejects.toThrow("New budget must exceed current observed spend");
await budgets.resolveIncident(f.company.id, incident.id, { action: "raise_budget_and_resume", amount: 175 }, "board");
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0]).toMatchObject({ budgetMonthlyCents: 175, status: "active", pauseReason: null });
});
it("exposes missing prices and blocks strict budgets until explicitly allowed", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, inputTokens: 500, costCents: 0, costStatus: "unpriced" });
expect(await costService(db).summary(f.company.id)).toMatchObject({ spendCents: 0, unpricedEventCount: 1, pricingComplete: false });
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ reason: "Agent cannot start work because recorded usage has no reliable price." });
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100, unpricedUsagePolicy: "allow" }, "board");
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await budgets.overview(f.company.id)).policies[0]).toMatchObject({ unpricedEventCount: 1, unpricedUsagePolicy: "allow" });
});
it("keeps subscription-included usage outside monetary pricing gaps", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, inputTokens: 500, costCents: 0, costStatus: "unpriced", billingType: "subscription_included" });
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: true });
});
it("defaults reports to the UTC month and excludes future events from rolling spend", async () => {
const f = await fixture(); const costs = costService(db); const now = new Date();
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() - 1, 1)) });
await costs.createEvent(f.company.id, { ...f.receipt, occurredAt: new Date(now.getTime() + 86_400_000) });
expect((await costs.summary(f.company.id)).spendCents).toBe(0);
expect((await costs.summary(f.company.id, { allTime: true })).spendCents).toBe(200);
expect(await costs.windowSpend(f.company.id)).toEqual([]);
});
it("keeps finance currencies separate and deduplicates invoice-line retries", async () => {
const f = await fixture(); const finance = financeService(db);
const usd = { biller: "vendor", eventKind: "platform_fee", amountCents: 100, currency: "usd", occurredAt: new Date(), idempotencyKey: "invoice:line:1", metadataJson: { a: 1, b: 2 } };
await Promise.all(Array.from({ length: 6 }, () => finance.createEvent(f.company.id, usd)));
await finance.createEvent(f.company.id, { ...usd, metadataJson: { b: 2, a: 1 } });
await finance.createEvent(f.company.id, { ...usd, currency: "EUR", idempotencyKey: "invoice:line:2" });
const summary = await finance.summary(f.company.id);
expect(summary).toMatchObject({ currency: "USD", debitCents: 100, netCents: 100 });
expect(summary.currencies).toEqual(expect.arrayContaining([expect.objectContaining({ currency: "USD", netCents: 100 }), expect.objectContaining({ currency: "EUR", netCents: 100 })]));
expect(await finance.byBiller(f.company.id)).toHaveLength(2);
expect(await finance.byKind(f.company.id)).toHaveLength(2);
expect(await db.select().from(financeEvents).where(eq(financeEvents.companyId, f.company.id))).toHaveLength(2);
await expect(finance.createEvent(f.company.id, { ...usd, amountCents: 200 })).rejects.toThrow(/different/);
});
it("synchronizes budgets through generic agent and company updates", async () => {
const f = await fixture();
const { agentService, companyService } = services.value;
await agentService(db).update(f.agent.id, { budgetMonthlyCents: 50 });
await companyService(db).update(f.company.id, { budgetMonthlyCents: 75 });
await costService(db).createEvent(f.company.id, f.receipt);
const policies = await budgetService(db).listPolicies(f.company.id);
expect(policies).toEqual(expect.arrayContaining([expect.objectContaining({ scopeType: "agent", amount: 50 }), expect.objectContaining({ scopeType: "company", amount: 75 })]));
expect(await budgetService(db).getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ scopeType: "company" });
});
it("delivers cancellation after generic budget updates commit", async () => {
const f = await fixture();
await costService(db).createEvent(f.company.id, f.receipt);
const cancelWorkForScope = vi.fn(async (scope) => {
const policies = await budgetService(db).listPolicies(f.company.id);
expect(policies.some(policy => policy.scopeType === scope.scopeType && policy.amount < 100)).toBe(true);
});
await services.value.agentService(db, { cancelWorkForScope }).update(f.agent.id, { budgetMonthlyCents: 50 });
await services.value.companyService(db, { cancelWorkForScope }).update(f.company.id, { budgetMonthlyCents: 75 });
expect(cancelWorkForScope).toHaveBeenCalledWith(expect.objectContaining({ scopeType: "agent", scopeId: f.agent.id }));
expect(cancelWorkForScope).toHaveBeenCalledWith(expect.objectContaining({ scopeType: "company", scopeId: f.company.id }));
const rows = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, f.company.id));
expect(rows.every(row => row.enforcementVersion === row.enforcementDeliveredVersion)).toBe(true);
});
it("merges partial policy edits with current settings under the accounting lock", async () => {
const f = await fixture(); const budgets = budgetService(db);
const identity = { scopeType: "agent" as const, scopeId: f.agent.id };
await budgets.upsertPolicy(f.company.id, { ...identity, amount: 100, hardStopEnabled: true }, "board");
await budgets.upsertPolicy(f.company.id, { ...identity, amount: 200, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false }, "other-operator");
const saved = await budgets.upsertPolicy(f.company.id, upsertBudgetPolicySchema.parse({ ...identity, reservationCents: "10" }), "board");
expect(saved).toMatchObject({ amount: 200, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false, reservationCents: "10.0000000" });
await budgets.upsertPolicy(f.company.id, { ...identity, isActive: false }, "other-operator");
const disabled = await budgets.upsertPolicy(f.company.id, upsertBudgetPolicySchema.parse({ ...identity, amount: 300 }), "board");
expect(await db.select({ amount: budgetPolicies.amount, isActive: budgetPolicies.isActive }).from(budgetPolicies).where(eq(budgetPolicies.id, disabled.policyId))).toEqual([{ amount: 300, isActive: false }]);
expect(disabled).toMatchObject({ amount: 0, isActive: false, hardStopEnabled: false, warnPercent: 60, notifyEnabled: false, reservationCents: "10.0000000" });
await expect(budgets.upsertPolicy(f.company.id, { scopeType: "project", scopeId: f.project.id, reservationCents: "1" }, "board")).rejects.toThrow("Amount is required");
});
it("conserves both spend and tokens across complete per-model receipts", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ status: "succeeded", finishedAt: new Date(), costAccountingPending: true, usageJson: {
provider: "anthropic", billingType: "metered_api", model: "mixed", costUsd: 0.007,
inputTokens: 30, outputTokens: 4, cachedInputTokens: 100,
usageByModel: [
{ model: "large", costUsd: 0.005, usage: { inputTokens: 20, outputTokens: 3, cachedInputTokens: 100 } },
{ model: "small", costUsd: 0.002, usage: { inputTokens: 10, outputTokens: 1 } },
],
} }).where(eq(heartbeatRuns.id, f.run.id));
await accountRunCost(db, f.run.id);
const rows = await costService(db).byProvider(f.company.id);
expect(rows).toEqual(expect.arrayContaining([expect.objectContaining({ model: "large", costCents: 0.5, inputTokens: 20 }), expect.objectContaining({ model: "small", costCents: 0.2, inputTokens: 10 })]));
expect((await costService(db).summary(f.company.id)).spendCents).toBe(0.7);
const [totals] = await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id));
expect(totals).toMatchObject({ totalInputTokens: 30, totalCachedInputTokens: 100, totalOutputTokens: 4, totalCostCents: 0.7 });
});
it("blocks admission while a terminal run awaits accounting, even under an allow-unpriced policy", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 1000, unpricedUsagePolicy: "allow" }, "board");
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true }).where(eq(heartbeatRuns.id, f.run.id));
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toMatchObject({ reason: expect.stringContaining("await accounting") });
expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: false, pendingRunCount: 1 });
await accountRunCost(db, f.run.id);
expect(await costService(db).summary(f.company.id)).toMatchObject({ pricingComplete: false, pendingRunCount: 0, unpricedEventCount: 1 });
expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
});
it("waits for a late provider receipt after cancellation instead of acknowledging a zero charge", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ status: "cancelled", costAccountingPending: true, usageJson: { accountingReceiptReady: false } }).where(eq(heartbeatRuns.id, f.run.id));
expect(await accountRunCost(db, f.run.id)).toBe(false);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: true, costAccountedAt: null });
expect(await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id))).toHaveLength(0);
await db.update(heartbeatRuns).set({ usageJson: { accountingReceiptReady: true, costUsd: 0.25, inputTokens: 10 } }).where(eq(heartbeatRuns.id, f.run.id));
await Promise.all([accountRunCost(db, f.run.id), accountRunCost(db, f.run.id)]);
expect((await db.select().from(costEvents).where(eq(costEvents.heartbeatRunId, f.run.id)))[0]).toMatchObject({ costCents: 25, inputTokens: 10 });
expect((await db.select().from(agentRuntimeState).where(eq(agentRuntimeState.agentId, f.agent.id)))[0].totalCostCents).toBe(25);
});
it("acknowledges proven pre-provider failures without inventing a charge", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true,
resultJson: { executionRecovery: { providerWorkStarted: false } },
}).where(eq(heartbeatRuns.id, f.run.id));
await accountRunCost(db, f.run.id);
expect(await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id))).toHaveLength(0);
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ costAccountingPending: false, costAccountedAt: expect.any(Date) });
});
it("records spend after attribution targets are deleted", async () => {
const f = await fixture();
await db.update(heartbeatRuns).set({ status: "failed", costAccountingPending: true, usageJson: {
costUsd: 0.01, ledgerScope: { projectId: f.project.id, issueId: f.issue.id },
} }).where(eq(heartbeatRuns.id, f.run.id));
await db.delete(issues).where(eq(issues.id, f.issue.id));
await db.delete(projects).where(eq(projects.id, f.project.id));
await accountRunCost(db, f.run.id);
expect((await db.select().from(costEvents).where(eq(costEvents.companyId, f.company.id)))[0]).toMatchObject({ costCents: 1, issueId: null, projectId: null });
});
it("preserves dismissed incidents and warning incidents during admission checks", async () => {
const f = await fixture(); const budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "agent", scopeId: f.agent.id, amount: 100 }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 80 });
for (let i = 0; i < 3; i++) expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeNull();
expect((await budgets.overview(f.company.id)).activeIncidents).toHaveLength(1);
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 20 });
const [incident] = await db.select().from(budgetIncidents).where(and(eq(budgetIncidents.companyId, f.company.id), eq(budgetIncidents.thresholdType, "hard")));
await budgets.resolveIncident(f.company.id, incident.id, { action: "keep_paused" }, "board");
for (let i = 0; i < 3; i++) expect(await budgets.getInvocationBlock(f.company.id, f.agent.id)).toBeTruthy();
expect(await db.select().from(approvals).where(eq(approvals.companyId, f.company.id))).toHaveLength(1);
});
it("does not revive terminated agents or archived companies during budget reconciliation", async () => {
const f = await fixture(); const budgets = budgetService(db);
await db.update(agents).set({ status: "terminated", pauseReason: "budget" }).where(eq(agents.id, f.agent.id));
await db.update(companies).set({ status: "archived", pauseReason: "budget" }).where(eq(companies.id, f.company.id));
for (const [scopeType, scopeId] of [["agent", f.agent.id], ["company", f.company.id]] as const) {
await budgets.upsertPolicy(f.company.id, { scopeType, scopeId, amount: 1000 }, "board");
}
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2000 });
expect((await db.select().from(agents).where(eq(agents.id, f.agent.id)))[0].status).toBe("terminated");
expect((await db.select().from(companies).where(eq(companies.id, f.company.id)))[0].status).toBe("archived");
});
it.each(["agent", "company", "project"] as const)("limits delayed %s budget cancellation to work that preceded the policy check", async (scopeType) => {
const f = await fixture();
const { heartbeatService } = await import("../services/heartbeat.js");
const heartbeat = heartbeatService(db);
const scopeId = scopeType === "agent" ? f.agent.id : scopeType === "company" ? f.company.id : f.project.id;
const cutoff = new Date();
await db.update(heartbeatRuns).set({ status: "queued", createdAt: new Date(cutoff.getTime() - 1000), contextSnapshot: { projectId: f.project.id } }).where(eq(heartbeatRuns.id, f.run.id));
const [newRun] = await db.insert(heartbeatRuns).values({ companyId: f.company.id, agentId: f.agent.id, invocationSource: "on_demand", status: "scheduled_retry",
scheduledRetryAt: new Date(cutoff.getTime() + 60000), createdAt: new Date(cutoff.getTime() + 1000), contextSnapshot: { projectId: f.project.id },
}).returning();
await heartbeat.cancelBudgetScopeWork({ companyId: f.company.id, scopeType, scopeId, createdBefore: cutoff });
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0].status).toBe("cancelled");
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, newRun.id)))[0].status).toBe("scheduled_retry");
});
it.each(["agent", "company", "project"] as const)("rechecks delayed %s enforcement after a grant admits an old queued run", async (scopeType) => {
const f = await fixture();
const { heartbeatService } = await import("../services/heartbeat.js");
const heartbeat = heartbeatService(db);
const budgets = budgetService(db);
const scopeId = scopeType === "agent" ? f.agent.id : scopeType === "company" ? f.company.id : f.project.id;
const policyInput = { scopeType, scopeId, amount: 1 };
await db.update(heartbeatRuns).set({ status: "queued", contextSnapshot: { projectId: f.project.id } }).where(eq(heartbeatRuns.id, f.run.id));
const [wake] = await db.insert(agentWakeupRequests).values({ companyId: f.company.id, agentId: f.agent.id, source: "automation", payload: { projectId: f.project.id } }).returning();
await budgets.upsertPolicy(f.company.id, policyInput, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, projectId: f.project.id, costCents: 2 });
let release!: () => void;
const paused = new Promise<void>(resolve => { release = resolve; });
let snapshotReady!: (scope: BudgetEnforcementScope) => void;
const snapshot = new Promise<BudgetEnforcementScope>(resolve => { snapshotReady = resolve; });
const delivery = budgetService(db, { cancelWorkForScope: async (scope) => {
snapshotReady(scope); await paused; await heartbeat.cancelBudgetScopeWork(scope);
} }).deliverPendingEnforcement(f.company.id);
try {
expect((await snapshot).enforcement).toMatchObject({ policyId: expect.any(String), version: expect.any(Number) });
await budgets.upsertPolicy(f.company.id, { ...policyInput, amount: 100 }, "board");
await db.update(heartbeatRuns).set({ status: "running" }).where(eq(heartbeatRuns.id, f.run.id));
await reserveRunBudget(db, f.company.id, f.run.id, f.project.id);
} finally { release(); }
await delivery;
expect((await db.select().from(heartbeatRuns).where(eq(heartbeatRuns.id, f.run.id)))[0]).toMatchObject({ status: "running", resultJson: null });
expect((await db.select().from(agentWakeupRequests).where(eq(agentWakeupRequests.id, wake.id)))[0].status).toBe("queued");
});
it("keeps a durably claimed budget stop fenced after a later grant", async () => {
const f = await fixture(), budgets = budgetService(db);
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 1 }, "board");
await costService(db).createEvent(f.company.id, { ...f.receipt, costCents: 2 });
const [policy] = await db.select().from(budgetPolicies).where(eq(budgetPolicies.companyId, f.company.id));
const scope: BudgetEnforcementScope = { companyId: f.company.id, scopeType: "company", scopeId: f.company.id,
enforcement: { policyId: policy.id, version: policy.enforcementVersion } };
expect(await withCurrentBudgetEnforcement(db, scope, async (tx) => {
await tx.update(heartbeatRuns).set({ status: "running", resultJson: { cancellation: { reason: "Budget stop" } } }).where(eq(heartbeatRuns.id, f.run.id));
return true;
})).toBe(true);
await budgets.upsertPolicy(f.company.id, { scopeType: "company", scopeId: f.company.id, amount: 100 }, "board");
await expect(reserveRunBudget(db, f.company.id, f.run.id, null)).rejects.toThrow("cancellation already requested");
const effect = vi.fn(async () => true);
for (const invalid of [
scope,
{ ...scope, enforcement: { ...scope.enforcement!, policyId: randomUUID() } },
{ ...scope, scopeId: randomUUID() },
{ ...scope, scopeType: "agent" as const },
{ ...scope, enforcement: { ...scope.enforcement!, version: policy.enforcementVersion + 1 } },
]) expect(await withCurrentBudgetEnforcement(db, invalid, effect)).toBeNull();
expect(effect).not.toHaveBeenCalled();
});
});
@@ -0,0 +1,91 @@
import express from "express";
import request from "supertest";
import { randomUUID } from "node:crypto";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { agents, companies, createDb } from "@paperclipai/db";
import { costRoutes } from "../routes/costs.js";
import { errorHandler } from "../middleware/error-handler.js";
import { getEmbeddedPostgresTestSupport, startEmbeddedPostgresTestDatabase } from "./helpers/embedded-postgres.js";
const support = await getEmbeddedPostgresTestSupport();
(support.supported ? describe : describe.skip)("accounting operator authorization", () => {
let database: Awaited<ReturnType<typeof startEmbeddedPostgresTestDatabase>>;
let db: ReturnType<typeof createDb>;
let companyId: string, foreignId: string, agentId: string;
beforeAll(async () => {
database = await startEmbeddedPostgresTestDatabase("accounting-authz-"); db = createDb(database.connectionString);
const rows = await db.insert(companies).values([{ name: "A", issuePrefix: "AUTH_A" }, { name: "B", issuePrefix: "AUTH_B" }]).returning();
companyId = rows[0].id; foreignId = rows[1].id;
agentId = (await db.insert(agents).values({ companyId, name: "Worker", role: "engineer", adapterType: "process" }).returning())[0].id;
},30_000);
afterAll(async () => { await database?.cleanup(); });
function app(actor: Record<string, unknown>) {
const app = express(); app.use(express.json()); app.use((req,_res,next) => { req.actor = actor as typeof req.actor; next(); });
app.use("/api", costRoutes(db)); app.use(errorHandler); return app;
}
const operations = [
["get", "health", undefined], ["get", "inspect", undefined], ["get", "invoices", undefined],
["get", `invoices/${randomUUID()}`, undefined], ["get", `events/${randomUUID()}/adjustments`, undefined],
["post", "repair", { fingerprint: "a".repeat(64), reason: "test" }],
["post", "provider-costs/import", {}], ["post", "retry", { runId: randomUUID() }], ["post", "invoices", {}], ["post", `events/${randomUUID()}/adjustments`, {}],
] as const;
it.each(operations)("denies agents for %s %s before reading or validating operator payloads", async (method, route, body) => {
const response = await request(app({ type: "agent", companyId, agentId }))[method](`/api/companies/${companyId}/accounting/${route}`).send(body);
expect(response.status).toBe(403);
});
it.each(operations)("denies nonmembers for %s %s", async (method, route, body) => {
const response = await request(app({ type: "board", source: "session", userId: "operator", companyIds: [foreignId] }))[method](`/api/companies/${companyId}/accounting/${route}`).send(body);
expect(response.status).toBe(403);
});
it("allows operator inspection and rejects viewer mutations", async () => {
const board = { type: "board", source: "session", userId: "operator", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole: "viewer" }] };
const response = await request(app(board)).get(`/api/companies/${companyId}/accounting/inspect`);
expect(response.status).toBe(200); expect(response.body.findings).toEqual([]);
expect((await request(app(board)).post(`/api/companies/${companyId}/accounting/repair`).send({ fingerprint: response.body.fingerprint, reason: "test" })).status).toBe(403);
});
it.each(["member", "viewer", "unknown", undefined])("denies provider credential use by %s before payload validation", async (membershipRole) => {
const actor = { type: "board", source: "session", userId: "member", companyIds: [companyId], memberships: [{ companyId, status: "active", membershipRole }] };
const response = await request(app(actor)).post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({});
expect(response.status).toBe(403);
});
it.each([
{ source: "session", memberships: "owner" },
{ source: "session", memberships: "admin" },
{ source: "session", isInstanceAdmin: true },
{ source: "local_implicit" },
])("allows billing operators through to payload validation: %j", async (input) => {
const actor = { ...input, type: "board", userId: "operator", companyIds: [companyId], memberships: input.memberships ? [{ companyId, status: "active", membershipRole: input.memberships }] : [] };
const response = await request(app(actor)).post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({});
expect(response.status).toBe(400);
});
it("does not accept an inactive admin or an admin role from another company", async () => {
for (const membership of [
{ companyId, status: "suspended", membershipRole: "admin" },
{ companyId: foreignId, status: "active", membershipRole: "admin" },
]) {
const response = await request(app({ type: "board", source: "session", userId: "operator", companyIds: [companyId], memberships: [membership] }))
.post(`/api/companies/${companyId}/accounting/provider-costs/import`).send({});
expect(response.status).toBe(403);
}
});
it("authorizes the user cost report and validates its date range", async () => {
const path = `/api/companies/${companyId}/costs/by-user`;
const board = app({ type: "board", source: "local_implicit", userId: "operator" });
const response = await request(board).get(`${path}?period=all`);
expect(response.status).toBe(200);
expect(response.body).toEqual({ activeUserCount: 0, rows: [] });
for (const query of ["from=invalid", "from=2026-02-01&to=2026-01-01", "period=all&from=2026-01-01"]) {
expect((await request(board).get(`${path}?${query}`)).status).toBe(400);
}
for (const actor of [
{ type: "board", source: "session", userId: "outsider", companyIds: [foreignId] },
{ type: "agent", companyId: foreignId, agentId },
]) expect((await request(app(actor)).get(path)).status).toBe(403);
expect((await request(app({ type: "none" })).get(path)).status).toBe(401);
});
it("validates correction and repair payloads before mutating", async () => {
const board = app({ type: "board", source: "local_implicit", userId: "operator" });
expect((await request(board).post(`/api/companies/${companyId}/accounting/repair`).send({ fingerprint: "old", reason: "" })).status).toBe(400);
expect((await request(board).post(`/api/companies/${companyId}/accounting/events/${randomUUID()}/adjustments`).send({ idempotencyKey: "x", expectedCents: "0", correctedCents: "-0.000000001", reason: "Invalid", pricing: { source: "operator" } })).status).toBe(400);
});
});
@@ -944,6 +944,25 @@ describe("AgentMail durable email pipeline", () => {
);
});
it.each([
["1999999999.9999999", true],
["2000000000.0000000", false],
["2000000000.0000001", false],
])("compares exact monthly email spend %s at the budget boundary", async (spent, allowed) => {
const f = await fixture();
await f.receive(f.message());
const [conversation] = await db.select().from(chatConversations).where(eq(chatConversations.companyId, f.companyId));
const runId = randomUUID();
await db.insert(heartbeatRuns).values({ id: runId, companyId: f.companyId, agentId: f.agentId, status: "running", runtimeMode: "native", nativeIssueId: conversation.issueId, contextSnapshot: { issueId: conversation.issueId } });
await db.update(issues).set({ executionRunId: runId }).where(eq(issues.id, conversation.issueId));
await db.update(agents).set({ budgetMonthlyCents: 2000000000, spentMonthlyCents: sql`${spent}::numeric` }).where(eq(agents.id, f.agentId));
const request = emailSendSchema.parse({ endpointId: f.endpointId, conversationId: conversation.id, replyToMessageId: [...f.messages.keys()][0], text: "Exact budget reply", idempotencyKey: randomUUID() });
const queued = f.service.queueSend(f.companyId, request, { agentId: f.agentId, runId });
if (allowed) expect((await queued).outcome).toBe("queued");
else await expect(queued).rejects.toThrow("Agent is not available to send email");
expect(f.sends).toHaveLength(0);
});
it("enforces one inbox owner across companies and reconnects the same identity", async () => {
const f = await fixture();
const other = await fixture();
@@ -0,0 +1,20 @@
// Dedicated real-process fault fixture. Only invoked with a throwaway test DB.
import { createDb } from "@paperclipai/db";
import { costService } from "../../services/costs.js";
import { createRunUsageRecorder } from "../../services/usage-receipts.js";
const [mode, encoded] = process.argv.slice(2);
const input = JSON.parse(encoded);
if (!process.env.PAPERCLIP_ACCOUNTING_TEST_DATABASE) throw new Error("Explicit test database required");
const db = createDb(process.env.PAPERCLIP_ACCOUNTING_TEST_DATABASE);
try {
if (mode === "receipt") {
const recorder = await createRunUsageRecorder(db, { companyId: input.companyId, runId: input.runId, adapterType: "process" }, input.directory);
// The receipt is fsynced before persistUsageReceipt enters a transaction.
// Freeze at this boundary, so the parent can SIGKILL us before any DB row.
db.transaction = async () => { process.stdout.write("RECEIPT_DURABLE\n"); return new Promise(() => {}); };
await recorder.capture({ costUsdExact: "0.012345678", usage: { inputTokens: 7, cachedInputTokens: 11, outputTokens: 3 }, complete: true, billingType: "metered_api" });
} else {
await Promise.all(Array.from({ length: 4 }, () => costService(db).createEvent(input.companyId, { ...input.receipt, occurredAt: new Date(input.receipt.occurredAt) })));
process.stdout.write("WRITES_COMMITTED\n");
}
} finally { await db.$client.end(); }
@@ -1,5 +1,6 @@
import { describe, expect, it } from "vitest";
import {
normalizeAdapterRunUsage,
resolveCacheAdjustedCostUsd,
resolveLedgerCostStatus,
} from "../services/heartbeat.js";
@@ -35,6 +36,13 @@ describe("heartbeat cost accounting", () => {
})).toBe("unpriced");
});
it("distinguishes missing receipts, reported zeroes, and included subscription usage", () => {
const noTokens = { inputTokens: 0, cachedInputTokens: 0, outputTokens: 0 };
expect(resolveLedgerCostStatus({ ...noTokens, costUsd: null })).toBe("unpriced");
expect(resolveLedgerCostStatus({ ...noTokens, costUsd: 0 })).toBe("reported");
expect(resolveLedgerCostStatus({ ...noTokens, costUsd: null, billingType: "subscription_included" })).toBe("reported");
});
it("marks reported CLI cost as priced", () => {
expect(resolveLedgerCostStatus({
costUsd: 1.25,
@@ -86,3 +94,17 @@ describe("heartbeat cost accounting", () => {
})).toBe(1.5);
});
});
describe("adapter usage basis", () => {
const prior = { inputTokens: 100, cachedInputTokens: 500, outputTokens: 20 };
it.each([undefined, null, "per_run"] as const)("keeps equal consecutive run usage when the basis is %s", (basis) => {
expect(normalizeAdapterRunUsage(prior, prior, basis)).toEqual(prior);
});
it("subtracts session totals only when explicitly declared", () => {
expect(normalizeAdapterRunUsage({ inputTokens: 110, cachedInputTokens: 700, outputTokens: 25 }, prior, "session_cumulative"))
.toEqual({ inputTokens: 10, cachedInputTokens: 200, outputTokens: 5 });
expect(normalizeAdapterRunUsage({ inputTokens: 5, cachedInputTokens: 10, outputTokens: 2 }, prior, "session_cumulative"))
.toEqual({ inputTokens: 5, cachedInputTokens: 10, outputTokens: 2 });
});
});
@@ -7,6 +7,7 @@ import {
agentRuntimeState,
companies,
costEvents,
budgetReservations,
createDb,
documentRevisions,
documents,
@@ -827,6 +828,9 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
afterContinuationDispatchCheck = async ({ runId: guardedRunId, issueId: guardedIssueId }) => {
expect(guardedRunId).toBe(runId);
expect(guardedIssueId).toBe(issueId);
// Budget admission must finish before this last ownership check. Putting
// an awaited reservation inside dispatch lets work escape the row lock.
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("held");
ordering.push("validated");
parkPromise = Promise.resolve(
db
@@ -870,6 +874,8 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
expect(issue?.status).toBe("backlog");
expect(ordering).toEqual(["validated", "parked"]);
expect(countExecuteCallsForRun(runId)).toBe(0);
await heartbeat.drainActiveRunExecutions();
expect((await db.select().from(budgetReservations).where(eq(budgetReservations.runId, runId)))[0]?.state).toBe("released");
});
it("rate-limits skipped generic timer wakes by advancing the timer baseline", async () => {
@@ -1311,6 +1317,14 @@ describeEmbeddedPostgres("heartbeat stale queued-run invalidation", () => {
});
});
it.each(["1999999999.9999999", "2000000000.0000000"])("compares exact daily spend before admitting work (%s)", async costCents => {
const { companyId, agentId } = await seedCompanyAndAgent({ heartbeatConfig: { maxDailyCostCents: 2_000_000_000 } });
await db.insert(costEvents).values({ companyId, agentId, provider: "test", biller: "test", billingType: "metered_api", model: "test", costCents: costCents as unknown as number, occurredAt: new Date() });
const run = await heartbeat.wakeup(agentId, { source: "on_demand", triggerDetail: "manual" });
if (costCents === "1999999999.9999999") expect(run).not.toBeNull();
else { expect(run).toBeNull(); expect(mockAdapterExecute).not.toHaveBeenCalled(); }
});
it("treats zero daily cost cap as a hard stop", async () => {
const { agentId } = await seedCompanyAndAgent({
heartbeatConfig: {
@@ -796,6 +796,36 @@ describe("openapi routes", () => {
});
});
it("documents exact money, reviewed correction inputs, and board-only accounting access", () => {
const { spec } = loadSpecRoutes();
const base = "/api/companies/{companyId}/accounting/";
for (const [suffix, methods] of [
["health", ["get"]], ["inspect", ["get"]], ["repair", ["post"]], ["retry", ["post"]],
["invoices", ["get", "post"]], ["invoices/{invoiceId}", ["get"]], ["events/{eventId}/adjustments", ["get", "post"]],
] as const) {
for (const method of methods) {
const operation = spec.paths[`${base}${suffix}`][method];
expect(operation["x-paperclip-authorization"]).toEqual({ actor: "board" });
expect(operation.security).toEqual([{ BoardSessionAuth: [] }, { BoardApiKeyAuth: [] }]);
expect(operation.responses["403"]).toBeDefined();
}
}
const adjustment = spec.paths[`${base}events/{eventId}/adjustments`].post;
expect(adjustment.responses["201"]).toBeDefined();
expect(adjustment.responses["409"]).toBeDefined();
expect(adjustment.responses["422"]).toBeDefined();
expect(adjustment.requestBody.content["application/json"].schema.required).toEqual(expect.arrayContaining([
"idempotencyKey", "expectedCents", "correctedCents", "reason", "pricing",
]));
for (const [endpoint, field] of [["cost-events", "costCents"], ["finance-events", "amountCents"]]) {
const operation = spec.paths[`/api/companies/{companyId}/${endpoint}`].post;
expect(operation.requestBody.content["application/json"].schema.properties[field]).toMatchObject({ oneOf: [{ type: "string" }, { type: "number" }] });
expect(operation.responses["409"]).toBeDefined();
}
const parameters = spec.paths["/api/companies/{companyId}/costs/summary"].get.parameters;
expect(parameters).toEqual(expect.arrayContaining([expect.objectContaining({ name: "period", in: "query", schema: expect.objectContaining({ enum: ["month", "all"] }) })]));
});
it("documents board-only repository discovery and selection", () => {
const { spec } = loadSpecRoutes();
const discovery = spec.paths["/api/companies/{companyId}/project-repositories"].get;
@@ -0,0 +1,236 @@
import { describe, expect, it, vi } from "vitest";
import { fetchProviderDailyCosts } from "../services/provider-billing-import.js";
import type { ImportProviderCosts } from "@paperclipai/shared";
const input: ImportProviderCosts = {
provider: "openai",
secretId: "00000000-0000-4000-8000-000000000001",
accountId: "org_test",
scopeIds: ["proj_test"],
from: "2026-09-01",
to: "2026-09-02",
};
const start = Date.parse(input.from) / 1000;
function page(value = 0.06) {
return {
data: [
{
start_time: start,
end_time: start + 86400,
results: [
{ project_id: "proj_test", amount: { currency: "usd", value } },
],
},
],
has_more: false,
next_page: null,
};
}
const response = (body: unknown) =>
new Response(JSON.stringify(body), { status: 200 });
describe("provider cost report ingestion", () => {
it("uses fixed endpoints, admin identity and scoped daily amounts in the right currency units", async () => {
const fetcher = vi.fn(async () => response(page()));
expect(
await fetchProviderDailyCosts(input, "private-admin", fetcher),
).toEqual([
{ day: "2026-09-01", scopeId: "proj_test", amountCents: "6.0000000" },
]);
const [url, options] = fetcher.mock.calls[0] as unknown as [
URL,
RequestInit,
];
expect(url.origin).toBe("https://api.openai.com");
expect(url.searchParams.get("group_by[]")).toBe("project_id");
expect(options.redirect).toBe("error");
expect(options.headers).toMatchObject({
"OpenAI-Organization": "org_test",
});
});
it("verifies Anthropic organization identity and treats its amount as cents", async () => {
const fetcher = vi
.fn()
.mockResolvedValueOnce(response({ id: "org_test" }))
.mockResolvedValueOnce(
response({
data: [
{
starting_at: "2026-09-01T00:00:00Z",
ending_at: "2026-09-02T00:00:00Z",
results: [
{
workspace_id: "proj_test",
amount: "123.78912",
currency: "USD",
},
],
},
],
has_more: false,
}),
);
expect(
(
await fetchProviderDailyCosts(
{ ...input, provider: "anthropic" },
"private-admin",
fetcher,
)
)[0].amountCents,
).toBe("123.7891200");
await expect(
fetchProviderDailyCosts(
{ ...input, provider: "anthropic" },
"private-admin",
vi.fn(async () => response({ id: "other" })),
),
).rejects.toThrow("different provider organization");
});
it("excludes unrelated projects and supports an explicit empty daily bucket", async () => {
const body = page();
body.data[0].results[0].project_id = "other-company-project";
expect(
(
await fetchProviderDailyCosts(
input,
"private",
vi.fn(async () => response(body)),
)
)[0].amountCents,
).toBe("0.0000000");
});
it.each([
"missing-day",
"missing-group",
"overlap",
"currency",
"negative",
"period",
"cursor",
"shape",
])(
"rejects %s rather than importing partial or invented totals",
async (kind) => {
const body: any = page();
if (kind === "missing-day") body.data = [];
if (kind === "missing-group") delete body.data[0].results[0].project_id;
if (kind === "overlap") body.data.push(body.data[0]);
if (kind === "currency") body.data[0].results[0].amount.currency = "EUR";
if (kind === "negative") body.data[0].results[0].amount.value = -1;
if (kind === "period") body.data[0].end_time++;
if (kind === "cursor") body.has_more = true;
if (kind === "shape") body.data[0].results[0].amount = {};
await expect(
fetchProviderDailyCosts(
input,
"private",
vi.fn(async () => response(body)),
),
).rejects.toThrow();
},
);
it("does not expose upstream errors or credentials", async () => {
await expect(
fetchProviderDailyCosts(
input,
"private-secret",
vi.fn(
async () =>
new Response("private-secret raw diagnostics", { status: 403 }),
),
),
).rejects.toThrow("Check the admin credential");
});
it("bounds response size and does not follow redirects", async () => {
await expect(
fetchProviderDailyCosts(
input,
"private",
vi.fn(async () => new Response("x".repeat(2_000_001))),
),
).rejects.toThrow("too large");
});
it("collects all pages before returning and rejects a repeated cursor", async () => {
const first = { ...page(), has_more: true, next_page: "next" };
const second = page(0.01);
second.data[0].start_time += 86400;
second.data[0].end_time += 86400;
const fetcher = vi
.fn()
.mockResolvedValueOnce(response(first))
.mockResolvedValueOnce(response(second));
expect(
await fetchProviderDailyCosts(
{ ...input, to: "2026-09-03" },
"private",
fetcher,
),
).toHaveLength(2);
const empty = { data: [], has_more: true, next_page: "same" };
await expect(
fetchProviderDailyCosts(
input,
"private",
vi.fn(async () => response(empty)),
),
).rejects.toThrow("cursor");
});
it("preserves a valid report if closing its already-consumed response stream fails", async () => {
let sent = false;
const reader = {
read: async () =>
sent
? { done: true }
: ((sent = true),
{ done: false, value: Buffer.from(JSON.stringify(page())) }),
cancel: async () => {
throw new Error("closed transport");
},
};
const fetcher = vi
.fn<typeof fetch>()
.mockResolvedValue({
ok: true,
body: { getReader: () => reader },
} as unknown as Response);
expect(
(await fetchProviderDailyCosts(input, "private", fetcher))[0].amountCents,
).toBe("6.0000000");
});
it("includes unallocated costs only when the default scope was explicitly selected", async () => {
const body = page();
(body.data[0].results[0] as { project_id: string | null }).project_id =
null;
expect(
await fetchProviderDailyCosts(
{ ...input, scopeIds: ["default"] },
"private",
vi.fn(async () => response(body)),
),
).toEqual([
{ day: "2026-09-01", scopeId: "default", amountCents: "6.0000000" },
]);
});
it("rejects a provider amount with the other provider's unit/shape and missing period boundaries", async () => {
const wrong: any = page();
wrong.data[0].results[0].amount = "6";
wrong.data[0].results[0].currency = "USD";
await expect(
fetchProviderDailyCosts(
input,
"private",
vi.fn(async () => response(wrong)),
),
).rejects.toThrow("invalid billing amount");
const missing: any = page();
delete missing.data[0].start_time;
delete missing.data[0].end_time;
await expect(
fetchProviderDailyCosts(
input,
"private",
vi.fn(async () => response(missing)),
),
).rejects.toThrow("unexpected billing period");
});
});
+4 -4
View File
@@ -1078,7 +1078,7 @@ describeEmbeddedPostgres("status card routes", () => {
model: "gpt-5.4",
inputTokens: 5200,
outputTokens: 980,
costCents: 2,
costCents: 0.1234567,
occurredAt: new Date(),
});
const summaryWrite = await request(writerApp).put(`/api/status-cards/${created.body.id}/summary`).send({
@@ -1111,7 +1111,7 @@ describeEmbeddedPostgres("status card routes", () => {
summaryBody: "**Decide:** unblock launch approval.\n\n**Recent work:** launch review is waiting.",
watchedIssueCount: 1,
todayTokens: 6180,
todayCostCents: 2,
todayCostCents: 0.1234567,
};
const detail = await request(boardApp).get(`/api/status-cards/${created.body.id}`);
expect(detail.status).toBe(200);
@@ -1143,7 +1143,7 @@ describeEmbeddedPostgres("status card routes", () => {
model: "gpt-5.4",
inputTokens: 1300,
outputTokens: 410,
costCents: 1,
costCents: 3_000_000_000.25,
occurredAt: new Date(),
});
const incrementalWrite = await request(createApp(db, agentActor(company.id, summarizer.id, updateRun.id)))
@@ -1155,7 +1155,7 @@ describeEmbeddedPostgres("status card routes", () => {
model: "gpt-5.4",
});
expect(incrementalWrite.status).toBe(200);
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "incremental"))).toMatchObject({ inputTokens: 1300, outputTokens: 410 });
expect(await db.select().from(statusCardUpdates).then((rows) => rows.find((row) => row.kind === "incremental"))).toMatchObject({ inputTokens: 1300, outputTokens: 410, costCents: 3_000_000_000.25 });
const revisions = await request(boardApp).get(`/api/status-cards/${created.body.id}/summary-revisions`);
expect(revisions.status).toBe(200);
expect(revisions.body.map((row: { revisionNumber: number }) => row.revisionNumber)).toEqual([2, 1]);
+19
View File
@@ -0,0 +1,19 @@
{
"extends": "./tsconfig.json",
"compilerOptions": {
"rootDir": "..",
"noEmit": true
},
"include": [
"src/__tests__/cost-accounting-*.test.ts",
"src/__tests__/codex-pricing.test.ts",
"src/__tests__/provider-billing-import.test.ts",
"src/__tests__/company-quota-windows.test.ts",
"src/__tests__/fixtures/accounting-writer.ts",
"src/**/*.d.ts",
"vitest.accounting-mutations.config.ts",
"vitest.accounting.config.ts",
"../tests/e2e/cost-accounting.spec.ts"
],
"exclude": []
}
@@ -0,0 +1,27 @@
import { fileURLToPath } from "node:url";
import { defineConfig } from "vitest/config";
import base from "./vitest.config.js";
export const mutations = {
deduplication: { file: "/services/costs.ts", from: "idempotencyKey: parsed.data.idempotencyKey ?? null,", to: "idempotencyKey: null,", test: "deduplicates a retried receipt" },
company: { file: "/services/costs.ts", from: "const conditions: ReturnType<typeof eq>[] = [eq(costEvents.companyId, companyId)];", to: "const conditions: ReturnType<typeof eq>[] = [sql`true`];", test: "isolates company reporting" },
projection: { file: "/services/costs.ts", from: "+ ${delta}::numeric", to: "+ 0::numeric", test: "conserves agent projections" },
threshold: { file: "/services/budgets.ts", from: "compareCents(observed.totalExact, policy.amount) >= 0", to: "compareCents(observed.totalExact, policy.amount) > 0", test: "stops at the exact budget boundary" },
};
const name = process.env.PAPERCLIP_ACCOUNTING_MUTATION;
if (name && !(name in mutations)) throw new Error(`Unknown accounting mutation: ${name}`);
const mutation = name ? mutations[name as keyof typeof mutations] : null;
export default defineConfig({
...base, root: fileURLToPath(new URL(".", import.meta.url)),
plugins: [...(base.plugins ?? []), {
name: "accounting-mutation-test-only", enforce: "pre",
transform(code, id) {
if (!mutation || !id.split("?")[0].endsWith(mutation.file)) return;
if (!code.includes(mutation.from)) throw new Error(`Mutation anchor missing: ${name}`);
// Replace precisely one site in Vite's in-memory module. Workspace source
// stays untouched, including while another test process is running.
console.error(`ACCOUNTING_MUTATION_APPLIED ${name}`);
return code.replace(mutation.from, mutation.to);
},
}],
test: { ...base.test, include: ["src/__tests__/cost-accounting-mutation-targets.test.ts"], coverage: { enabled: false } },
});
+43
View File
@@ -0,0 +1,43 @@
import { fileURLToPath } from "node:url";
import { defineConfig } from "vitest/config";
import base from "./vitest.config.js";
export default defineConfig({
...base,
root: fileURLToPath(new URL(".", import.meta.url)),
test: {
...base.test,
include: [
"src/__tests__/cost-accounting-*.test.ts",
"src/__tests__/budgets-service.test.ts",
"src/__tests__/costs-service.test.ts",
"src/__tests__/codex-pricing.test.ts",
"src/__tests__/provider-billing-import.test.ts",
],
coverage: {
enabled: true,
provider: "v8",
reportOnFailure: true,
reportsDirectory: "../coverage/accounting",
reporter: ["text", "json", "json-summary", "html"],
// Every module must retain its own floor; a well-covered helper cannot
// hide a regression in ingestion, recovery, finance, or enforcement.
thresholds: { perFile: true, lines: 98, branches: 90, functions: 96, statements: 96 },
include: [
"src/services/accounting-transaction.ts",
"src/services/accounting-integrity.ts",
"src/services/billing-reconciliation.ts",
"src/services/budget-reservations.ts",
"src/services/usage-receipts.ts",
"src/services/receipt-fingerprint.ts",
"src/services/cost-date-range.ts",
"src/services/run-cost-accounting.ts",
"src/services/costs.ts",
"src/services/finance.ts",
"src/services/budgets.ts",
"src/services/codex-pricing.ts",
"src/services/provider-billing-import.ts",
],
},
},
});